From 9e8ce51f930e66fbbad053c18946046a4415e0e7 Mon Sep 17 00:00:00 2001 From: wolfplus Date: Sun, 6 Apr 2025 13:52:33 +0100 Subject: [PATCH] init --- a | 18 - android/app/src/main/AndroidManifest.xml | 9 + .../yunqiinnovation/deepsound/MainActivity.kt | 104 ++++- .../deepsound/SystemFunctionHandler.kt | 225 ++++++++++ .../deepsound/VoiceFunctionHandler.kt | 336 ++++++++++++++- .../deepsound/VoiceInteractionHandler.kt | 19 +- .../deepsound/VoiceInteractionService.kt | 7 +- lib/core/bindings/initial_binding.dart | 9 + .../events/voice_interaction_event.dart | 13 + lib/data/services/ai_service.dart | 2 + .../services/open_ai_service_adapter.dart | 247 +++++++++++ .../services/voice_interaction_service.dart | 24 ++ .../chat/controllers/chat_controller.dart | 12 +- .../azure_speech/AzureAsrHelper.kt | 4 +- local_plugins/open_ai_service/README.md | 405 +++++++----------- .../open_ai_service/OpenAIService.kt | 155 ++++--- .../open_ai_service/OpenAIServicePlugin.kt | 7 +- .../open_ai_service/lib/open_ai_service.dart | 38 -- test.json | 1 + 19 files changed, 1225 insertions(+), 410 deletions(-) delete mode 100644 a create mode 100644 android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt create mode 100644 lib/data/services/open_ai_service_adapter.dart create mode 100644 test.json diff --git a/a b/a deleted file mode 100644 index 7133736ae..000000000 --- a/a +++ /dev/null @@ -1,18 +0,0 @@ -curl 'https://ark.cn-beijing.volces.com/api/v3/bots/chat/completions' \ --H "Authorization: Bearer 168deb3d-fd0c-4912-b9f1-aaee5c6743e6" \ --H 'Content-Type: application/json' \ --d '{ - "model": "bot-20250405211523-l7c9r", - "stream": true, - "stream_options": {"include_usage": true}, - "messages": [ - { - "role": "system", - "content": "You are a helpful assistant." - }, - { - "role": "user", - "content": "Hello!" - } - ] -}' \ No newline at end of file diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index f120cc1f7..23c380fb0 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -27,6 +27,15 @@ + + + + + + + + = Build.VERSION_CODES.S) { + arrayOf( + Manifest.permission.RECORD_AUDIO, + Manifest.permission.BLUETOOTH_CONNECT, + Manifest.permission.BLUETOOTH_SCAN, + Manifest.permission.ACCESS_FINE_LOCATION, + Manifest.permission.SEND_SMS, + Manifest.permission.READ_CONTACTS, + Manifest.permission.CALL_PHONE, + Manifest.permission.POST_NOTIFICATIONS + ) + } else { + arrayOf( + Manifest.permission.RECORD_AUDIO, + Manifest.permission.BLUETOOTH, + Manifest.permission.BLUETOOTH_ADMIN, + Manifest.permission.ACCESS_FINE_LOCATION, + Manifest.permission.SEND_SMS, + Manifest.permission.READ_CONTACTS, + Manifest.permission.CALL_PHONE + ) + } + // 广播接收器 private val voiceInteractionReceiver = object : BroadcastReceiver() { override fun onReceive(context: Context?, intent: Intent?) { - Log.d(TAG, "收到广播: ${intent?.action}") + FileLogger.d(TAG, "收到广播: ${intent?.action}") when (intent?.action) { VoiceInteractionService.ACTION_RECOGNITION_STARTED -> { val timestamp = intent.getLongExtra("timestamp", 0) - Log.d(TAG, "收到语音识别启动广播: timestamp=$timestamp") sendVoiceInteractionEvent(mapOf( "type" to "recognition_started", "timestamp" to timestamp @@ -54,6 +82,12 @@ class MainActivity: FlutterActivity() { sendChatHistoryEvent(agentId, userMessage, assistantMessage, timestamp) } + VoiceInteractionService.ACTION_ENTER_TRANSLATION_MODE -> { + sendVoiceInteractionEvent(mapOf( + "type" to "enter_translation_mode", + "timestamp" to System.currentTimeMillis() + )) + } } } } @@ -249,6 +283,9 @@ class MainActivity: FlutterActivity() { // 初始化 FileLogger FileLogger.init(applicationContext) + // 请求必要权限 + requestRequiredPermissions() + // 初始化 Azure 语音服务 classicBluetoothHelper = ClassicBluetoothHelper(applicationContext) @@ -267,7 +304,14 @@ class MainActivity: FlutterActivity() { val voiceInteractionFilter = IntentFilter().apply { addAction(VoiceInteractionService.ACTION_RECOGNITION_STARTED) addAction(VoiceInteractionService.ACTION_CHAT_HISTORY_UPDATED) + addAction(VoiceInteractionService.ACTION_ENTER_TRANSLATION_MODE) } + + // 打印已注册的广播 + FileLogger.d(TAG, "已注册语音交互广播：${VoiceInteractionService.ACTION_RECOGNITION_STARTED}, " + + "${VoiceInteractionService.ACTION_CHAT_HISTORY_UPDATED}, " + + "${VoiceInteractionService.ACTION_ENTER_TRANSLATION_MODE}") + if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { registerReceiver(voiceInteractionReceiver, voiceInteractionFilter, Context.RECEIVER_NOT_EXPORTED) } else { @@ -550,14 +594,19 @@ class MainActivity: FlutterActivity() { private fun sendVoiceInteractionEvent(event: Map) { if (voiceInteractionEventSink == null) { Log.e(TAG, "无法发送语音交互事件：事件通道未准备好") + FileLogger.e(TAG, "无法发送语音交互事件：事件通道未准备好") return } + FileLogger.d(TAG, "准备发送事件到Flutter: ${event["type"]}") + runOnUiThread { try { voiceInteractionEventSink?.success(event) + FileLogger.d(TAG, "成功发送事件到Flutter: ${event["type"]}") } catch (e: Exception) { - Log.e(TAG, "发送语音交互事件到Flutter失败: ${e.message}", e) + Log.e(TAG, "发送语音交互事件到Flutter失败: ${e.message}") + FileLogger.e(TAG, "发送语音交互事件到Flutter失败: ${e.message}") } } } @@ -598,5 +647,54 @@ class MainActivity: FlutterActivity() { super.onDestroy() } + + /** + * 请求必要权限 + */ + private fun requestRequiredPermissions() { + val permissionsToRequest = ArrayList() + + for (permission in REQUIRED_PERMISSIONS) { + if (ContextCompat.checkSelfPermission(this, permission) != PackageManager.PERMISSION_GRANTED) { + permissionsToRequest.add(permission) + } + } + + if (permissionsToRequest.isNotEmpty()) { + ActivityCompat.requestPermissions( + this, + permissionsToRequest.toTypedArray(), + PERMISSION_REQUEST_CODE + ) + } + } + + /** + * 处理权限请求结果 + */ + override fun onRequestPermissionsResult( + requestCode: Int, + permissions: Array, + grantResults: IntArray + ) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults) + + if (requestCode == PERMISSION_REQUEST_CODE) { + val deniedPermissions = ArrayList() + + for (i in permissions.indices) { + if (grantResults[i] != PackageManager.PERMISSION_GRANTED) { + deniedPermissions.add(permissions[i]) + } + } + + if (deniedPermissions.isNotEmpty()) { + // 记录未授权的权限 + FileLogger.w(TAG, "未授权的权限: ${deniedPermissions.joinToString()}") + } else { + FileLogger.d(TAG, "所有必要权限已授权") + } + } + } } diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt new file mode 100644 index 000000000..4e28f26aa --- /dev/null +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt @@ -0,0 +1,225 @@ +package com.yunqiinnovation.deepsound + +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.net.Uri +import android.provider.ContactsContract +import android.telephony.SmsManager +import com.yunqiinnovation.deepsound.core.utils.FileLogger + +/** + * 系统功能处理器 - 负责调用系统API执行各种功能 + */ +class SystemFunctionHandler(private val context: Context) { + companion object { + private const val TAG = "SystemFunctionHandler" + } + + /** + * 启动翻译模式 + * @return 操作结果 + */ + fun enterTranslationMode(): Result { + FileLogger.d(TAG, "进入翻译模式") + // 此处应添加实际的翻译模式实现 + // 可能需要启动一个专门的翻译服务或活动 + + // 广播通知APP的其他部分进入翻译模式 + // 使用VoiceInteractionService中定义的常量 + val intent = Intent(VoiceInteractionService.ACTION_ENTER_TRANSLATION_MODE).apply { + + putExtra("timestamp", System.currentTimeMillis()) + } + + // 发送广播 + context.sendBroadcast(intent) + FileLogger.d(TAG, "已发送广播: ${VoiceInteractionService.ACTION_ENTER_TRANSLATION_MODE}") + + return Result( + success = true, + message = "已进入翻译模式" + ) + } + + /** + * 发送短信 + * @param contact 联系人（名称或电话号码） + * @param message 短信内容 + * @return 操作结果 + */ + fun sendTextMessage(contact: String, message: String): Result { + FileLogger.d(TAG, "发送短信给 $contact: $message") + + try { + val phoneNumber = getPhoneNumberForContact(contact) + if (phoneNumber.isEmpty()) { + return Result( + success = false, + message = "未找到联系人信息: $contact" + ) + } + + // 使用SmsManager发送短信 + val smsManager = SmsManager.getDefault() + smsManager.sendTextMessage(phoneNumber, null, message, null, null) + + return Result( + success = true, + message = "已向${contact}发送短信: $message" + ) + } catch (e: Exception) { + FileLogger.e(TAG, "发送短信失败: ${e.message}", e) + return Result( + success = false, + message = "发送短信失败: ${e.message}" + ) + } + } + + /** + * 拨打电话 + * @param contact 联系人（名称或电话号码） + * @return 操作结果 + */ + fun makePhoneCall(contact: String): Result { + FileLogger.d(TAG, "拨打电话给 $contact") + + try { + val phoneNumber = getPhoneNumberForContact(contact) + if (phoneNumber.isEmpty()) { + return Result( + success = false, + message = "未找到联系人信息: $contact" + ) + } + + // 创建拨号意图 + val intent = Intent(Intent.ACTION_CALL) + intent.data = Uri.parse("tel:$phoneNumber") + intent.flags = Intent.FLAG_ACTIVITY_NEW_TASK + context.startActivity(intent) + + return Result( + success = true, + message = "正在拨打${contact}的电话" + ) + } catch (e: Exception) { + FileLogger.e(TAG, "拨打电话失败: ${e.message}", e) + return Result( + success = false, + message = "拨打电话失败: ${e.message}" + ) + } + } + + /** + * 打开应用程序 + * @param appName 应用名称 + * @return 操作结果 + */ + fun openApp(appName: String): Result { + FileLogger.d(TAG, "打开应用: $appName") + + try { + // 查找应用的包名 + val packageManager = context.packageManager + val intent = Intent(Intent.ACTION_MAIN, null) + intent.addCategory(Intent.CATEGORY_LAUNCHER) + + val apps = packageManager.queryIntentActivities(intent, 0) + + for (resolveInfo in apps) { + val appLabel = resolveInfo.loadLabel(packageManager).toString() + if (appLabel.equals(appName, ignoreCase = true) || + appLabel.contains(appName, ignoreCase = true)) { + + // 找到匹配的应用 + val packageName = resolveInfo.activityInfo.packageName + val launchIntent = packageManager.getLaunchIntentForPackage(packageName) + + if (launchIntent != null) { + launchIntent.flags = Intent.FLAG_ACTIVITY_NEW_TASK + context.startActivity(launchIntent) + + return Result( + success = true, + message = "正在打开${appName}应用" + ) + } + } + } + + return Result( + success = false, + message = "未找到应用: $appName" + ) + } catch (e: Exception) { + FileLogger.e(TAG, "打开应用失败: ${e.message}", e) + return Result( + success = false, + message = "打开应用失败: ${e.message}" + ) + } + } + + /** + * 根据联系人名称获取电话号码 + * @param contact 联系人名称或电话号码 + * @return 电话号码 + */ + private fun getPhoneNumberForContact(contact: String): String { + // 如果输入的是电话号码（简单判断，包含数字和特殊符号） + if (contact.matches(Regex("^[+\\d\\s\\-()]*\$")) && contact.any { it.isDigit() }) { + // 移除非数字字符 + return contact.replace(Regex("[^+\\d]"), "") + } + + // 从联系人数据库查询 + try { + val contentResolver = context.contentResolver + val uri = Uri.withAppendedPath( + ContactsContract.Contacts.CONTENT_FILTER_URI, + Uri.encode(contact) + ) + + val projection = arrayOf( + ContactsContract.Contacts._ID, + ContactsContract.Contacts.DISPLAY_NAME, + ContactsContract.Contacts.HAS_PHONE_NUMBER + ) + + contentResolver.query(uri, projection, null, null, null)?.use { cursor -> + if (cursor.moveToFirst()) { + val hasPhoneNumber = cursor.getInt(cursor.getColumnIndexOrThrow(ContactsContract.Contacts.HAS_PHONE_NUMBER)) + + if (hasPhoneNumber > 0) { + val contactId = cursor.getString(cursor.getColumnIndexOrThrow(ContactsContract.Contacts._ID)) + val phoneUri = ContactsContract.CommonDataKinds.Phone.CONTENT_URI + val phoneProjection = arrayOf(ContactsContract.CommonDataKinds.Phone.NUMBER) + val phoneSelection = "${ContactsContract.CommonDataKinds.Phone.CONTACT_ID} = ?" + val phoneSelectionArgs = arrayOf(contactId) + + contentResolver.query(phoneUri, phoneProjection, phoneSelection, phoneSelectionArgs, null)?.use { phoneCursor -> + if (phoneCursor.moveToFirst()) { + return phoneCursor.getString(phoneCursor.getColumnIndexOrThrow(ContactsContract.CommonDataKinds.Phone.NUMBER)) + } + } + } + } + } + } catch (e: Exception) { + FileLogger.e(TAG, "查询联系人信息失败: ${e.message}", e) + } + + return "" + } + + /** + * 操作结果数据类 + */ + data class Result( + val success: Boolean, + val message: String + ) +} \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt index 78a55b642..c15ca7e17 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt @@ -4,18 +4,22 @@ import org.json.JSONArray import org.json.JSONObject import com.yunqiinnovation.open_ai_service.OpenAIService import com.yunqiinnovation.deepsound.core.utils.FileLogger +import android.content.Context /** * 语音功能处理器 - 处理AI函数调用 */ class VoiceFunctionHandler( private val openAIService: OpenAIService, - private val systemPrompt: String + private val context: Context ) { companion object { private const val TAG = "VoiceFunctionHandler" } + // 添加系统功能处理器 + private val systemFunctionHandler = SystemFunctionHandler(context) + /** * 初始化并注册所有可用的函数 */ @@ -24,6 +28,18 @@ class VoiceFunctionHandler( // 注册退出交互函数 registerExitInteractionFunction() + // 注册翻译模式函数 + registerTranslationModeFunction() + + // 注册发送短信函数 + registerSendTextMessageFunction() + + // 注册拨打电话函数 + registerMakePhoneCallFunction() + + // 注册打开应用函数 + registerOpenAppFunction() + // 在这里可以注册更多函数 } catch (e: Exception) { @@ -53,6 +69,113 @@ class VoiceFunctionHandler( } } + /** + * 注册翻译模式函数 + */ + private fun registerTranslationModeFunction() { + try { + openAIService.registerFunction( + "enter_translation_mode", + "进入翻译模式", + JSONObject(""" + { + "type": "object", + "properties": {}, + "required": [] + } + """) + ) + FileLogger.d(TAG, "翻译模式功能已注册") + } catch (e: Exception) { + FileLogger.e(TAG, "注册翻译模式函数失败: ${e.message}", e) + } + } + + /** + * 注册发送短信函数 + */ + private fun registerSendTextMessageFunction() { + try { + openAIService.registerFunction( + "send_text_message", + "发送短信", + JSONObject(""" + { + "type": "object", + "properties": { + "contact": { + "type": "string", + "description": "联系人姓名或电话号码" + }, + "message": { + "type": "string", + "description": "短信内容" + } + }, + "required": ["contact", "message"] + } + """) + ) + FileLogger.d(TAG, "发送短信功能已注册") + } catch (e: Exception) { + FileLogger.e(TAG, "注册发送短信函数失败: ${e.message}", e) + } + } + + /** + * 注册拨打电话函数 + */ + private fun registerMakePhoneCallFunction() { + try { + openAIService.registerFunction( + "make_phone_call", + "拨打电话", + JSONObject(""" + { + "type": "object", + "properties": { + "contact": { + "type": "string", + "description": "联系人姓名或电话号码" + } + }, + "required": ["contact"] + } + """) + ) + FileLogger.d(TAG, "拨打电话功能已注册") + } catch (e: Exception) { + FileLogger.e(TAG, "注册拨打电话函数失败: ${e.message}", e) + } + } + + /** + * 注册打开应用函数 + */ + private fun registerOpenAppFunction() { + try { + openAIService.registerFunction( + "open_app", + "打开应用", + JSONObject(""" + { + "type": "object", + "properties": { + "app_name": { + "type": "string", + "description": "应用名称" + } + }, + "required": ["app_name"] + } + """) + ) + FileLogger.d(TAG, "打开应用功能已注册") + } catch (e: Exception) { + FileLogger.e(TAG, "注册打开应用函数失败: ${e.message}", e) + } + } + /** * 处理函数调用 * @@ -74,6 +197,22 @@ class VoiceFunctionHandler( handleExitInteraction(functionCall, messages, callback) true } + "enter_translation_mode" -> { + handleEnterTranslationMode(functionCall, messages, callback) + true + } + "send_text_message" -> { + handleSendTextMessage(functionCall, messages, callback) + true + } + "make_phone_call" -> { + handleMakePhoneCall(functionCall, messages, callback) + true + } + "open_app" -> { + handleOpenApp(functionCall, messages, callback) + true + } else -> { // 未知函数，返回默认结果 handleUnknownFunction(functionCall, messages, callback) @@ -96,7 +235,6 @@ class VoiceFunctionHandler( openAIService.sendFunctionCallResult( messages = messages, - systemPrompt = systemPrompt, functionCall = functionCall, functionResult = "{\"result\": \"已退出语音交互\"}", callback = object : OpenAIService.StreamCallback { @@ -126,6 +264,199 @@ class VoiceFunctionHandler( ) } + /** + * 处理进入翻译模式函数 + */ + private fun handleEnterTranslationMode( + functionCall: JSONObject, + messages: JSONArray, + callback: FunctionCallCallback + ) { + FileLogger.d(TAG, "处理进入翻译模式函数") + + // 调用系统功能处理器执行翻译模式操作 + val result = systemFunctionHandler.enterTranslationMode() + + val responseBuilder = StringBuilder() + + openAIService.sendFunctionCallResult( + messages = messages, + functionCall = functionCall, + functionResult = "{\"result\": \"${result.message}\"}", + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + responseBuilder.append(token) + callback.onTokenReceived(token) + } + + override fun onComplete() { + FileLogger.d(TAG, "handleEnterTranslationMode onComplete: ${responseBuilder.toString()}") + callback.onComplete() + } + + override fun onError(e: Exception) { + FileLogger.e(TAG, "处理进入翻译模式函数调用出错: ${e.message}") + callback.onError("进入翻译模式时出错") + } + + override fun onFunctionCall(nestedCall: JSONObject) { + callback.onFunctionCall(nestedCall) + } + } + ) + } + + /** + * 处理发送短信函数 + */ + private fun handleSendTextMessage( + functionCall: JSONObject, + messages: JSONArray, + callback: FunctionCallCallback + ) { + FileLogger.d(TAG, "处理发送短信函数") + + try { + val arguments = JSONObject(functionCall.getString("arguments")) + val contact = arguments.getString("contact") + val message = arguments.getString("message") + + // 调用系统功能处理器发送短信 + val result = systemFunctionHandler.sendTextMessage(contact, message) + + val responseBuilder = StringBuilder() + + openAIService.sendFunctionCallResult( + messages = messages, + functionCall = functionCall, + functionResult = "{\"result\": \"${result.message}\"}", + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + responseBuilder.append(token) + callback.onTokenReceived(token) + } + + override fun onComplete() { + FileLogger.d(TAG, "handleSendTextMessage onComplete: ${responseBuilder.toString()}") + callback.onComplete() + } + + override fun onError(e: Exception) { + FileLogger.e(TAG, "处理发送短信函数调用出错: ${e.message}") + callback.onError("发送短信时出错") + } + + override fun onFunctionCall(nestedCall: JSONObject) { + callback.onFunctionCall(nestedCall) + } + } + ) + } catch (e: Exception) { + FileLogger.e(TAG, "解析发送短信参数失败: ${e.message}") + callback.onError("发送短信参数错误") + } + } + + /** + * 处理拨打电话函数 + */ + private fun handleMakePhoneCall( + functionCall: JSONObject, + messages: JSONArray, + callback: FunctionCallCallback + ) { + FileLogger.d(TAG, "处理拨打电话函数") + + try { + val arguments = JSONObject(functionCall.getString("arguments")) + val contact = arguments.getString("contact") + + // 调用系统功能处理器拨打电话 + val result = systemFunctionHandler.makePhoneCall(contact) + + val responseBuilder = StringBuilder() + + openAIService.sendFunctionCallResult( + messages = messages, + functionCall = functionCall, + functionResult = "{\"result\": \"${result.message}\"}", + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + responseBuilder.append(token) + callback.onTokenReceived(token) + } + + override fun onComplete() { + FileLogger.d(TAG, "handleMakePhoneCall onComplete: ${responseBuilder.toString()}") + callback.onComplete() + } + + override fun onError(e: Exception) { + FileLogger.e(TAG, "处理拨打电话函数调用出错: ${e.message}") + callback.onError("拨打电话时出错") + } + + override fun onFunctionCall(nestedCall: JSONObject) { + callback.onFunctionCall(nestedCall) + } + } + ) + } catch (e: Exception) { + FileLogger.e(TAG, "解析拨打电话参数失败: ${e.message}") + callback.onError("拨打电话参数错误") + } + } + + /** + * 处理打开应用函数 + */ + private fun handleOpenApp( + functionCall: JSONObject, + messages: JSONArray, + callback: FunctionCallCallback + ) { + FileLogger.d(TAG, "处理打开应用函数") + + try { + val arguments = JSONObject(functionCall.getString("arguments")) + val appName = arguments.getString("app_name") + + // 调用系统功能处理器打开应用 + val result = systemFunctionHandler.openApp(appName) + + val responseBuilder = StringBuilder() + + openAIService.sendFunctionCallResult( + messages = messages, + functionCall = functionCall, + functionResult = "{\"result\": \"${result.message}\"}", + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + responseBuilder.append(token) + callback.onTokenReceived(token) + } + + override fun onComplete() { + FileLogger.d(TAG, "handleOpenApp onComplete: ${responseBuilder.toString()}") + callback.onComplete() + } + + override fun onError(e: Exception) { + FileLogger.e(TAG, "处理打开应用函数调用出错: ${e.message}") + callback.onError("打开应用时出错") + } + + override fun onFunctionCall(nestedCall: JSONObject) { + callback.onFunctionCall(nestedCall) + } + } + ) + } catch (e: Exception) { + FileLogger.e(TAG, "解析打开应用参数失败: ${e.message}") + callback.onError("打开应用参数错误") + } + } + /** * 处理未知函数调用 */ @@ -139,7 +470,6 @@ class VoiceFunctionHandler( try { openAIService.sendFunctionCallResult( messages = messages, - systemPrompt = systemPrompt, functionCall = functionCall, functionResult = "{\"result\": \"处理函数调用中\"}", callback = object : OpenAIService.StreamCallback { diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt index fa179c279..1d6a528b3 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt @@ -30,21 +30,9 @@ class VoiceInteractionHandler( // OpenAI服务 private val openAIService = OpenAIService() - // 系统提示词 - private val systemPrompt = """ - 你是一个智能语音助手，能够简洁明了地回答用户的问题。 -时刻关心用户的情绪和需求，主动提供鼓励和温暖。 -语言风格活泼、亲切，能够幽默地互动，陪伴用户，缓解压力，增添生活乐趣。 - -请始终以用户为中心，保持回应的高效性、准确性和温暖体贴，成为用户真正的灵魂伴侣。 - - 当用户说"退出"、"再见"、"结束对话"等类似意图时，你应该使用exit_interaction函数来结束对话， - 并在结束前说一句友好的告别语，例如"再见，有需要随时找我"。 - """.trimIndent() - // 语音功能处理 - private val voiceFunctionHandler = VoiceFunctionHandler(openAIService, systemPrompt) + private val voiceFunctionHandler = VoiceFunctionHandler(openAIService, context) // 当前用户输入 private var currentUserInput = "" @@ -127,12 +115,14 @@ class VoiceInteractionHandler( override fun onResult(result: String, detectedLanguage: String) { if (result.isNotEmpty()) { + notifyStateChanged() + processWithOpenAI(result) + } // 重置状态，继续识别 hasSpeechDetected = false - notifyStateChanged() } override fun onSessionStarted() { @@ -227,7 +217,6 @@ class VoiceInteractionHandler( openAIService.sendMessageStream( messages = messages, - systemPrompt = systemPrompt, callback = object : OpenAIService.StreamCallback { override fun onToken(token: String) { // 累加响应内容 diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt index 76faa6f33..1cb132e1a 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt @@ -46,7 +46,7 @@ class VoiceInteractionService : Service() { private const val CHANNEL_ID = "voice_interaction_channel" // 语音识别超时时间（毫秒） - private const val RECOGNITION_TIMEOUT = 8000L + private const val RECOGNITION_TIMEOUT = 10000L // 用于跟踪服务是否正在运行 private val isRunning = AtomicBoolean(false) @@ -58,6 +58,7 @@ class VoiceInteractionService : Service() { const val ACTION_RECOGNITION_STARTED = "com.yunqiinnovation.deepsound.ACTION_RECOGNITION_STARTED" const val ACTION_PAUSE_VOICE_INTERACTION = "com.yunqiinnovation.deepsound.ACTION_PAUSE_VOICE_INTERACTION" const val ACTION_CHAT_HISTORY_UPDATED = "com.yunqiinnovation.deepsound.ACTION_CHAT_HISTORY_UPDATED" + const val ACTION_ENTER_TRANSLATION_MODE = "com.yunqiinnovation.deepsound.ACTION_ENTER_TRANSLATION_MODE" } // 服务状态 @@ -293,12 +294,12 @@ class VoiceInteractionService : Service() { if (!isActive) { isActive = true } - + // FileLogger.d(TAG, "服务状态: isActive=${isActive}, isRecognitionActive=${voiceInteractionHandler.isRecognitionActive}, isTimeoutPaused=${isTimeoutPaused}") // 检查语音识别状态 if (voiceInteractionHandler.isRecognitionActive) { val currentTime = System.currentTimeMillis() val elapsedTime = currentTime - lastActivityTime - + FileLogger.d(TAG, "hasSpeechDetected=${voiceInteractionHandler.hasSpeechDetected}, isTtsSpeaking=${voiceInteractionHandler.isTtsSpeaking}, elapsedTime=${elapsedTime}") // 如果超过指定时间没有检测到语音，且不在TTS播放中，暂停语音识别 if (!voiceInteractionHandler.hasSpeechDetected && !voiceInteractionHandler.isTtsSpeaking && diff --git a/lib/core/bindings/initial_binding.dart b/lib/core/bindings/initial_binding.dart index 3f42d9560..748589f63 100644 --- a/lib/core/bindings/initial_binding.dart +++ b/lib/core/bindings/initial_binding.dart @@ -8,6 +8,7 @@ import '../../data/services/classic_bluetooth_service.dart'; import '../../data/services/bluetooth_media_button_service.dart'; import '../../core/utils/logger.dart'; import '../../data/services/voice_interaction_service.dart'; +import '../../data/services/open_ai_service_adapter.dart'; /// 初始绑定，用于管理全局依赖 class InitialBinding extends Bindings { @@ -29,6 +30,13 @@ class InitialBinding extends Bindings { Get.lazyPut(() => VolcanoTranslationService(), fenix: true); + // OpenAI服务适配器 + Get.lazyPut(() { + final adapter = OpenAIServiceAdapter(); + adapter.initialize(); + return adapter; + }, fenix: true); + // 火山AI服务 Get.lazyPut(() => VolcanoAIService(), fenix: true); @@ -41,6 +49,7 @@ class InitialBinding extends Bindings { () => BluetoothMediaButtonService(), fenix: true); + // 语音交互服务 Get.lazyPut(() => VoiceInteractionService(), fenix: true); diff --git a/lib/data/models/events/voice_interaction_event.dart b/lib/data/models/events/voice_interaction_event.dart index 4bfa636df..81860de23 100644 --- a/lib/data/models/events/voice_interaction_event.dart +++ b/lib/data/models/events/voice_interaction_event.dart @@ -26,4 +26,17 @@ class RecognitionStartedEvent extends VoiceInteractionEvent { RecognitionStartedEvent({ required int timestamp, }) : super(timestamp: timestamp); +} + +/// 通用语音交互事件 +/// 用于处理其他类型的事件 +class GenericVoiceInteractionEvent extends VoiceInteractionEvent { + final String type; + final Map? data; + + GenericVoiceInteractionEvent({ + required this.type, + this.data, + required int timestamp, + }) : super(timestamp: timestamp); } \ No newline at end of file diff --git a/lib/data/services/ai_service.dart b/lib/data/services/ai_service.dart index 37a1f5df0..c1a0a1bcd 100644 --- a/lib/data/services/ai_service.dart +++ b/lib/data/services/ai_service.dart @@ -1,5 +1,7 @@ /// AI回复服务接口 abstract class AiService { + + /// 非流式输出方法 Future sendMessage({ required List> messages, diff --git a/lib/data/services/open_ai_service_adapter.dart b/lib/data/services/open_ai_service_adapter.dart new file mode 100644 index 000000000..2c0da53ac --- /dev/null +++ b/lib/data/services/open_ai_service_adapter.dart @@ -0,0 +1,247 @@ +import 'dart:async'; +import 'package:flutter_dotenv/flutter_dotenv.dart'; +import 'package:open_ai_service/open_ai_service.dart'; +import 'package:get/get.dart'; +import 'ai_service.dart'; + +/// OpenAI服务适配器 - 连接AiService接口与OpenAIService插件 +class OpenAIServiceAdapter implements AiService { + final OpenAIService _openAIService = OpenAIService(); + StreamSubscription? _eventSubscription; + final StreamController _tokenStreamController = StreamController.broadcast(); + bool _isProcessingStream = false; + + /// 构造函数 + OpenAIServiceAdapter() { + printInfo(info: '创建OpenAIServiceAdapter实例'); + _setupEventListener(); + } + + /// 设置事件监听器 + void _setupEventListener() { + try { + // 首先确保访问eventStream以初始化底层事件通道 + _openAIService.eventStream; + + // 设置事件处理 + _eventSubscription = _openAIService.eventStream.listen( + (event) { + if (!_isProcessingStream) return; + + try { + switch (event.type) { + case OpenAIEventType.token: + if (event.content is String) { + _tokenStreamController.add(event.content as String); + } else { + printInfo(info: '收到非字符串类型的token: ${event.content}'); + } + break; + case OpenAIEventType.complete: + _isProcessingStream = false; + break; + case OpenAIEventType.error: + if (event.content is String) { + _tokenStreamController.addError(event.content as String); + } else { + _tokenStreamController.addError('未知错误: ${event.content}'); + } + _isProcessingStream = false; + break; + case OpenAIEventType.functionCall: + try { + if (event.content is Map) { + _tokenStreamController.addError('收到函数调用，该流仅支持文本响应'); + } else { + _tokenStreamController.addError('收到未知格式的函数调用'); + printError(info: '函数调用格式错误: ${event.content}'); + } + } catch (e) { + printError(info: '处理函数调用事件出错: $e'); + _tokenStreamController.addError('处理函数调用失败: $e'); + } + _isProcessingStream = false; + break; + } + } catch (e) { + printError(info: '处理事件出错: $e'); + _tokenStreamController.addError('处理事件失败: $e'); + _isProcessingStream = false; + } + }, + onError: (error) { + printError(info: '事件流错误: $error'); + _tokenStreamController.addError('事件流错误: $error'); + _isProcessingStream = false; + }, + onDone: () { + printInfo(info: '事件流已关闭'); + _isProcessingStream = false; + }, + ); + } catch (e) { + printError(info: '设置事件监听器失败: $e'); + } + } + + /// 初始化OpenAI服务 + Future initialize() async { + try { + // 从.env文件中读取配置 + final apiKey = dotenv.env['OPENAI_API_KEY'] ?? ''; + final baseUrl = dotenv.env['OPENAI_BASE_URL'] ?? ''; + final model = dotenv.env['OPENAI_MODEL'] ?? ''; + + printInfo(info: '从.env读取OpenAI配置'); + printInfo(info: '基础URL: $baseUrl'); + printInfo(info: '模型名称: $model'); + + if (apiKey.isEmpty) { + printError(info: '错误: OpenAI API密钥未配置，请在.env文件中设置OPENAI_API_KEY'); + return false; + } + + // 初始化OpenAI服务 + final result = await _openAIService.initialize( + apiKey: apiKey, + baseUrl: baseUrl, + model: model, + ); + + if (result) { + printInfo(info: 'OpenAI服务初始化成功'); + } else { + printError(info: 'OpenAI服务初始化失败'); + } + + return result; + } catch (e) { + printError(info: 'OpenAI服务初始化异常: $e'); + return false; + } + } + + /// 发送消息并获取回复 + @override + Future sendMessage({ + required List> messages, + required String systemPrompt, + }) async { + try { + // 在方法内直接转换 + final convertedMessages = messages.map((m) => + Map.from(m)).toList(); + + // 发送消息并获取回复 + final response = await _openAIService.sendMessage( + messages: convertedMessages, + ); + + return response; + } catch (e) { + printError(info: 'OpenAI发送消息失败: $e'); + throw '发送消息失败: $e'; + } + } + + /// 发送消息并获取流式回复 + @override + Stream sendMessageStream({ + required List> messages, + required String systemPrompt, + }) async* { + try { + // 在方法内直接转换 + final convertedMessages = messages.map((m) => + Map.from(m)).toList(); + + // 创建用于接收token的控制器 + final localController = StreamController(); + + // 标记开始处理流 + _isProcessingStream = true; + + // 添加从广播流到本地流的订阅 + final subscription = _tokenStreamController.stream.listen( + (token) => localController.add(token), + onError: (error) { + printError(info: '令牌流错误: $error'); + localController.addError(error); + localController.close(); + }, + onDone: () { + printInfo(info: '令牌流已完成'); + localController.close(); + } + ); + + // 当本地控制器关闭时，取消订阅 + localController.onCancel = () { + subscription.cancel(); + }; + + // 启动流式消息请求 + bool started = false; + try { + started = await _openAIService.sendMessageStream( + messages: convertedMessages, + ); + } catch (e) { + printError(info: '启动消息流失败: $e'); + localController.addError('启动消息流失败: $e'); + localController.close(); + _isProcessingStream = false; + throw '启动消息流失败: $e'; + } + + if (!started) { + printError(info: '无法启动消息流'); + localController.addError('无法启动消息流'); + localController.close(); + _isProcessingStream = false; + throw '无法启动消息流'; + } + + // 通过yield*将controller的流转发 + yield* localController.stream; + } catch (e) { + printError(info: 'OpenAI流式消息处理失败: $e'); + throw '流式消息处理失败: $e'; + } + } + + /// 注册函数 + Future registerFunction(String name, String description, Map parameters) async { + try { + final result = await _openAIService.registerFunction( + name: name, + description: description, + parameters: parameters, + ); + + if (result) { + printInfo(info: '函数 "$name" 注册成功'); + } else { + printError(info: '函数 "$name" 注册失败'); + } + + return result; + } catch (e) { + printError(info: '注册函数失败: $e'); + return false; + } + } + + /// 释放资源 + void dispose() { + try { + _isProcessingStream = false; + _eventSubscription?.cancel(); + _tokenStreamController.close(); + printInfo(info: 'OpenAIServiceAdapter资源已释放'); + } catch (e) { + printError(info: '释放资源时出错: $e'); + } + } + +} \ No newline at end of file diff --git a/lib/data/services/voice_interaction_service.dart b/lib/data/services/voice_interaction_service.dart index 792e94912..cce506a58 100644 --- a/lib/data/services/voice_interaction_service.dart +++ b/lib/data/services/voice_interaction_service.dart @@ -5,6 +5,7 @@ import 'package:flutter_dotenv/flutter_dotenv.dart'; import '../models/events/voice_interaction_event.dart'; import '../../core/utils/logger.dart'; import '../../modules/chat/models/message_model.dart'; +import '../../routes/app_routes.dart'; import 'chat_history_service.dart'; /// 语音交互服务接口 @@ -104,6 +105,29 @@ class VoiceInteractionService extends GetxService { // 保存聊天历史到ChatHistoryService _saveChatHistory(agentId, userMessage, assistantMessage, DateTime.now().millisecondsSinceEpoch); break; + + case 'enter_translation_mode': + // 进入翻译模式事件 + Logger.info('收到进入翻译模式事件，正在导航到翻译界面'); + _navigateToTranslation(); + + final translationModeEvent = GenericVoiceInteractionEvent( + type: 'enter_translation_mode', + timestamp: DateTime.now().millisecondsSinceEpoch, + ); + _eventStreamController.add(translationModeEvent); + break; + } + } + + /// 导航到翻译界面 + void _navigateToTranslation() { + try { + // 使用GetX导航到翻译页面 + Get.toNamed(Routes.translation); + Logger.info('已导航到翻译界面'); + } catch (e) { + Logger.error('导航到翻译界面失败: $e'); } } diff --git a/lib/modules/chat/controllers/chat_controller.dart b/lib/modules/chat/controllers/chat_controller.dart index 4d0457c98..77fa84a9f 100644 --- a/lib/modules/chat/controllers/chat_controller.dart +++ b/lib/modules/chat/controllers/chat_controller.dart @@ -16,6 +16,7 @@ import '../../../data/services/asr_service.dart'; import '../../../data/services/chat_history_service.dart'; import '../../../data/models/events/voice_interaction_event.dart'; import '../../../data/services/voice_interaction_service.dart'; +import '../../../data/services/open_ai_service_adapter.dart'; class ChatController extends GetxController { // 服务 @@ -92,6 +93,7 @@ class ChatController extends GetxController { agent = foundAgent; + // 根据Agent ID选择不同的AI服务 switch (agent.id) { case 'cyber_girlfriend': // 亲子陪伴 @@ -104,7 +106,15 @@ class ChatController extends GetxController { _aiService = Get.find(); break; default: - _aiService = Get.find(); + // 默认使用OpenAIServiceAdapter + try { + _aiService = Get.find(); + Logger.info('使用OpenAIServiceAdapter'); + } catch (e) { + // 如果找不到OpenAIServiceAdapter，则回退到VolcanoAIService + Logger.info('未找到OpenAIServiceAdapter，回退使用VolcanoAIService: $e'); + _aiService = Get.find(); + } } // 使用克隆音色语音合成 diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt index ac6697cdb..1dd7cf624 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt @@ -25,7 +25,7 @@ class AzureAsrHelper(private val context: Context) { private var subscriptionKey = "" private var region = "" private var isAutoDetectLanguage = false - private var supportedLanguages = arrayOf("zh-CN", "en-US") + private var supportedLanguages = arrayOf("zh-CN") // 是否使用回音消除 - 内部控制常量 private val useEchoCancellation = false @@ -37,7 +37,7 @@ class AzureAsrHelper(private val context: Context) { // 初始化SDK并创建recognizer fun initialize(subscriptionKey: String, region: String, - supportedLanguages: Array = arrayOf("zh-CN", "en-US")): Boolean { + supportedLanguages: Array = arrayOf("zh-CN")): Boolean { try { FileLogger.d(TAG, "初始化 Azure 语音服务") diff --git a/local_plugins/open_ai_service/README.md b/local_plugins/open_ai_service/README.md index d7ccb709d..351eee84f 100644 --- a/local_plugins/open_ai_service/README.md +++ b/local_plugins/open_ai_service/README.md @@ -1,80 +1,78 @@ -# OpenAI服务插件 +# OpenAI Service Plugin -提供与OpenAI API的原生交互功能，支持流式输出和函数调用，同时支持Android和iOS平台。 +一个用于Flutter应用的OpenAI服务插件，支持Android和iOS平台。 -## 功能特点 +## 功能 -- 原生实现，性能更好 -- 支持流式输出，实时显示AI回复 -- 支持函数调用（工具调用） +- 支持文本生成（completions） +- 支持流式输出（streaming） +- 支持函数调用（function calling） - 支持自定义API基础URL -- 支持Android和iOS平台 -- 提供简洁易用的Dart接口 +- 支持自定义模型选择 ## 安装 -将该插件添加到您的`pubspec.yaml`文件中： +在你的`pubspec.yaml`文件中添加以下依赖： ```yaml dependencies: open_ai_service: - path: local_plugins/open_ai_service + path: 本地路径/open_ai_service ``` ## 使用方法 -### 初始化 - -首先需要初始化OpenAI服务，提供API密钥和可选的基础URL和模型： +### 初始化服务 ```dart import 'package:open_ai_service/open_ai_service.dart'; final openAIService = OpenAIService(); -// 使用OpenAI官方API和默认模型 -await openAIService.initialize(apiKey: 'your_openai_api_key'); - -// 或使用自定义API端点和模型 +// 初始化服务 await openAIService.initialize( - apiKey: 'your_api_key', - baseUrl: 'https://your-custom-endpoint.com/v1/chat/completions', - model: 'gpt-4-0125-preview', // 自定义模型 + apiKey: 'your_openai_api_key', + baseUrl: 'https://api.openai.com/v1/chat/completions', // 可选 + model: 'gpt-4-turbo', // 可选 ); ``` -### 普通聊天 - -发送消息并获取非流式回复： +### 发送非流式请求 ```dart -// 创建用户消息 +// 创建消息 final userMessage = await openAIService.createUserMessage('你好，请介绍一下自己'); -// 发送消息 +// 发送请求 final response = await openAIService.sendMessage( messages: [userMessage], - systemPrompt: '你是一个有帮助的AI助手，请简洁地回答问题。', + systemPrompt: '你是一个有用的AI助手', ); print('AI回复: $response'); ``` -### 流式输出 - -使用流式输出获取实时回复： +### 发送流式请求（回调方式） ```dart -// 注册事件监听 +// 创建消息 +final userMessage = await openAIService.createUserMessage('写一个短故事'); + +// 发送流式请求 +await openAIService.sendMessageStream( + messages: [userMessage], + systemPrompt: '你是一个善于讲故事的AI助手', +); + +// 处理事件 final subscription = openAIService.processEvents( onToken: (token) { - // 收到部分回复 - print('收到部分回复: $token'); - // 更新UI... + // 处理每个返回的token + print(token); }, onComplete: () { - // 回复完成 - print('回复完成'); + // 处理完成事件 + print('生成完成'); }, onError: (error) { // 处理错误 @@ -82,27 +80,50 @@ final subscription = openAIService.processEvents( }, onFunctionCall: (functionCall) { // 处理函数调用 - print('函数调用: $functionCall'); - handleFunctionCall(functionCall); + print('函数调用: ${functionCall['name']}'); }, ); -// 发送流式消息 -await openAIService.sendMessageStream( +// 在不需要时取消订阅 +subscription.cancel(); +``` + +### 发送流式请求（Stream方式） + +```dart +// 创建消息 +final userMessage = await openAIService.createUserMessage('写一个短故事'); + +// 获取字符串流 +final stream = openAIService.streamMessage( messages: [userMessage], - systemPrompt: '你是一个有帮助的AI助手，请简洁地回答问题。', + systemPrompt: '你是一个善于讲故事的AI助手', ); -// 完成后，记得取消订阅 -// subscription.cancel(); -``` +// 使用流 +final StringBuilder responseBuilder = StringBuilder(); -### 函数调用 +stream.listen( + (token) { + // 处理每个token + responseBuilder.write(token); + print(token); // 实时输出 + }, + onDone: () { + // 流结束 + print('完整回复: ${responseBuilder.toString()}'); + }, + onError: (error) { + // 错误处理 + print('错误: $error'); + } +); +``` -注册函数并处理函数调用： +### 注册函数 ```dart -// 注册一个天气查询函数 +// 注册一个函数 await openAIService.registerFunction( name: 'get_weather', description: '获取指定城市的天气信息', @@ -111,238 +132,122 @@ await openAIService.registerFunction( 'properties': { 'city': { 'type': 'string', - 'description': '城市名称，如北京、上海等', + 'description': '城市名称', }, 'date': { 'type': 'string', - 'description': '查询日期，默认为今天', + 'description': '日期，格式为YYYY-MM-DD', }, }, 'required': ['city'], }, ); +``` -// 处理函数调用 -void handleFunctionCall(Map functionCall) { - final name = functionCall['name']; - final arguments = jsonDecode(functionCall['arguments']); - final id = functionCall['id']; - - if (name == 'get_weather') { - final city = arguments['city']; - final date = arguments['date'] ?? '今天'; +### 处理函数调用 + +```dart +// 创建消息 +final userMessage = await openAIService.createUserMessage('明天北京的天气如何？'); + +// 发送流式请求 +await openAIService.sendMessageStream( + messages: [userMessage], + systemPrompt: '你是一个有用的AI助手', +); + +// 处理事件 +openAIService.processEvents( + onToken: (token) { + print(token); + }, + onComplete: () { + print('生成完成'); + }, + onError: (error) { + print('错误: $error'); + }, + onFunctionCall: (functionCall) { + // 处理函数调用 + final name = functionCall['name']; + final arguments = functionCall['arguments']; + + print('收到函数调用: $name, 参数: $arguments'); - // 获取天气信息（实际应用中应调用天气API） - final weatherResult = '{"temperature": 25, "condition": "晴天", "humidity": 40}'; + // 假设处理了函数调用并获得结果 + final result = '{"temperature": 25, "condition": "sunny"}'; // 发送函数调用结果 openAIService.sendFunctionCallResult( - messages: messages, // 当前的消息历史 - systemPrompt: systemPrompt, + messages: [userMessage], + systemPrompt: '你是一个有用的AI助手', functionCall: functionCall, - functionResult: weatherResult, + functionResult: result, ); - } -} + }, +); ``` -## 示例应用 - -一个简单的聊天机器人示例： +### 使用Stream API处理函数调用 ```dart -import 'package:flutter/material.dart'; -import 'package:open_ai_service/open_ai_service.dart'; +// 创建消息和响应处理器 +final userMessage = await openAIService.createUserMessage('明天北京的天气如何？'); +final responseBuilder = StringBuilder(); -void main() { - runApp(MyApp()); -} - -class MyApp extends StatelessWidget { - @override - Widget build(BuildContext context) { - return MaterialApp( - title: 'OpenAI 聊天', - theme: ThemeData(primarySwatch: Colors.blue), - home: ChatScreen(), - ); - } -} - -class ChatScreen extends StatefulWidget { - @override - _ChatScreenState createState() => _ChatScreenState(); -} - -class _ChatScreenState extends State { - final OpenAIService _openAIService = OpenAIService(); - final TextEditingController _textController = TextEditingController(); - final List> _messages = []; - StreamSubscription? _subscription; - String _currentResponse = ''; - bool _isLoading = false; - - @override - void initState() { - super.initState(); - _initializeOpenAI(); - } - - Future _initializeOpenAI() async { - try { - await _openAIService.initialize( - apiKey: 'your_api_key_here', - ); - - // 注册事件监听 - _subscription = _openAIService.processEvents( - onToken: (token) { - setState(() { - _currentResponse += token; - }); - }, - onComplete: () { - _addAIMessage(_currentResponse); - setState(() { - _currentResponse = ''; - _isLoading = false; - }); - }, - onError: (error) { - ScaffoldMessenger.of(context).showSnackBar( - SnackBar(content: Text('错误: $error')), - ); - setState(() { - _isLoading = false; - }); - }, - onFunctionCall: (functionCall) { - // 处理函数调用... - }, - ); - } catch (e) { - ScaffoldMessenger.of(context).showSnackBar( - SnackBar(content: Text('初始化失败: $e')), - ); - } - } - - void _sendMessage() async { - if (_textController.text.isEmpty) return; +// 处理事件流以捕获函数调用 +final subscription = openAIService.processEvents( + onFunctionCall: (functionCall) async { + // 取消当前事件监听 + subscription.cancel(); - final userMessage = _textController.text; - _addUserMessage(userMessage); - _textController.clear(); + // 处理函数调用 + final name = functionCall['name']; + final arguments = functionCall['arguments']; - try { - setState(() { - _isLoading = true; - }); - - final message = await _openAIService.createUserMessage(userMessage); - - await _openAIService.sendMessageStream( - messages: _messages, - systemPrompt: '你是一个有帮助的AI助手，请简洁地回答问题。', - ); - } catch (e) { - ScaffoldMessenger.of(context).showSnackBar( - SnackBar(content: Text('发送消息失败: $e')), - ); - setState(() { - _isLoading = false; - }); - } - } - - void _addUserMessage(String content) { - setState(() { - _messages.add({ - 'role': 'user', - 'content': content, - }); - }); - } - - void _addAIMessage(String content) { - setState(() { - _messages.add({ - 'role': 'assistant', - 'content': content, - }); - }); - } - - @override - void dispose() { - _subscription?.cancel(); - _textController.dispose(); - super.dispose(); - } - - @override - Widget build(BuildContext context) { - return Scaffold( - appBar: AppBar(title: Text('OpenAI 聊天')), - body: Column( - children: [ - Expanded( - child: ListView.builder( - itemCount: _messages.length + (_currentResponse.isNotEmpty ? 1 : 0), - itemBuilder: (context, index) { - if (index < _messages.length) { - final message = _messages[index]; - final isUser = message['role'] == 'user'; - - return ListTile( - leading: Icon(isUser ? Icons.person : Icons.computer), - title: Text(message['content']), - ); - } else { - // 显示当前正在生成的回复 - return ListTile( - leading: Icon(Icons.computer), - title: Text(_currentResponse), - ); - } - }, - ), - ), - if (_isLoading) - LinearProgressIndicator(), - Padding( - padding: const EdgeInsets.all(8.0), - child: Row( - children: [ - Expanded( - child: TextField( - controller: _textController, - decoration: InputDecoration( - hintText: '输入消息...', - border: OutlineInputBorder(), - ), - ), - ), - IconButton( - icon: Icon(Icons.send), - onPressed: _sendMessage, - ), - ], - ), - ), - ], - ), + print('收到函数调用: $name, 参数: $arguments'); + + // 假设处理了函数调用并获得结果 + final result = '{"temperature": 25, "condition": "sunny"}'; + + // 使用Stream API发送函数调用结果 + final resultStream = openAIService.streamFunctionResult( + messages: [userMessage], + systemPrompt: '你是一个有用的AI助手', + functionCall: functionCall, + functionResult: result, + ); + + // 处理结果流 + resultStream.listen( + (token) { + responseBuilder.write(token); + print(token); // 实时输出 + }, + onDone: () { + print('完整回复: ${responseBuilder.toString()}'); + }, + onError: (error) { + print('错误: $error'); + } ); } -} +); + +// 启动请求 +await openAIService.sendMessageStream( + messages: [userMessage], + systemPrompt: '你是一个有用的AI助手', +); ``` ## 注意事项 -- 请确保您的API密钥安全存储，不要直接硬编码在应用程序中 -- 流式输出需要在调用`sendMessageStream`之前注册事件监听 -- 使用完毕后记得取消事件订阅 +1. 确保在使用前已正确初始化服务 +2. 对于流式请求，确保在不需要时取消订阅 +3. 处理函数调用时，确保提供有效的结果格式 +4. 网络请求可能会失败，请确保加入适当的错误处理 ## 许可证 -此插件遵循MIT许可证。 \ No newline at end of file +[MIT License](LICENSE) \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt index 7b9213046..907be93a2 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt @@ -28,6 +28,35 @@ class OpenAIService() { // 用于存储注册的函数 private val registeredFunctions = mutableListOf() + /** + * 构建curl命令用于测试 + */ + private fun buildCurlCommand(request: Request, body: String): String { + val command = StringBuilder("curl -v -X ${request.method}") + + // 添加请求头 + request.headers.forEach { header -> + // 敏感信息处理：不显示真实的API Key + if (header.first == "Authorization") { + command.append(" -H '${header.first}: Bearer $apiKey'") + } else { + command.append(" -H '${header.first}: ${header.second}'") + } + } + + // 添加请求体 + if (request.method == "POST" || request.method == "PUT") { + // 转义JSON中的单引号，确保curl命令正确 + val escapedBody = body.replace("'", "\\'") + command.append(" -d '${escapedBody}'") + } + + // 添加URL + command.append(" '${request.url}'") + + return command.toString() + } + /** * 创建用户消息 */ @@ -97,24 +126,14 @@ class OpenAIService() { * 发送消息（非流式输出） */ @Throws(OpenAIException::class) - fun sendMessage(messages: JSONArray, systemPrompt: String): String { + fun sendMessage(messages: JSONArray): String { if (!isInitialized || apiKey.isEmpty()) { throw OpenAIException("OpenAI服务未初始化") } - val fullMessages = JSONArray().apply { - put(JSONObject().apply { - put("role", "system") - put("content", systemPrompt) - }) - for (i in 0 until messages.length()) { - put(messages.getJSONObject(i)) - } - } - val requestBody = JSONObject().apply { put("model", model) - put("messages", fullMessages) + put("messages", messages) put("temperature", 0.7) put("max_tokens", 2000) put("stream", false) @@ -142,7 +161,10 @@ class OpenAIService() { .build() try { - Log.d(TAG, "request: $request") + // 输出用于测试的curl命令 + // val curlCommand = buildCurlCommand(request, requestBody.toString()) + // Log.d(TAG, "curl command: \n$curlCommand") + client.newCall(request).execute().use { response -> if (!response.isSuccessful) { throw OpenAIException("API调用失败: ${response.code}") @@ -196,25 +218,15 @@ class OpenAIService() { /** * 发送消息（流式输出） */ - fun sendMessageStream(messages: JSONArray, systemPrompt: String, callback: StreamCallback) { + fun sendMessageStream(messages: JSONArray, callback: StreamCallback) { if (!isInitialized || apiKey.isEmpty()) { callback.onError(OpenAIException("OpenAI服务未初始化")) return } - val fullMessages = JSONArray().apply { - put(JSONObject().apply { - put("role", "system") - put("content", systemPrompt) - }) - for (i in 0 until messages.length()) { - put(messages.getJSONObject(i)) - } - } - val requestBody = JSONObject().apply { put("model", model) - put("messages", fullMessages) + put("messages", messages) put("temperature", 0.7) put("max_tokens", 2000) put("stream", true) @@ -242,7 +254,10 @@ class OpenAIService() { .post(requestBody.toString().toRequestBody(mediaType)) .build() - Log.d(TAG, "request: $request") + // 输出用于测试的curl命令 + // val curlCommand = buildCurlCommand(request, requestBody.toString()) + // Log.d(TAG, "curl command: $curlCommand") + client.newCall(request).enqueue(object : Callback { override fun onFailure(call: Call, e: IOException) { callback.onError(OpenAIException(e.message ?: "请求失败")) @@ -267,14 +282,13 @@ class OpenAIService() { val finalToolCalls = mutableMapOf() while (!bufferedSource.exhausted()) { - val line = bufferedSource.readUtf8Line() ?: continue - val trimmedLine = line.trim() - if (trimmedLine.isEmpty()) continue - if (trimmedLine.startsWith("data:")) { - val data = trimmedLine.substring(5) + val line = bufferedSource.readUtf8Line()?.trim() ?: continue + if (line.isEmpty()) continue + if (line.startsWith("data:")) { + val data = line.substring(5).trim() // 处理[DONE]消息 - if (data == "[DONE]") { + if (data == "[DONE]" || data == "[\"DONE\"]") { processToolCalls(finalToolCalls, callback) callback.onComplete() break @@ -350,54 +364,53 @@ class OpenAIService() { */ fun sendFunctionCallResult( messages: JSONArray, - systemPrompt: String, functionCall: JSONObject, functionResult: String, callback: StreamCallback ) { try { - val fullMessages = JSONArray().apply { - // 添加系统提示 - put(JSONObject().apply { - put("role", "system") - put("content", systemPrompt) - }) - - // 添加用户消息 - for (i in 0 until messages.length()) { - put(messages.getJSONObject(i)) - } + val fullMessages = JSONArray() + + // 添加用户消息 + for (i in 0 until messages.length()) { + fullMessages.put(messages.getJSONObject(i)) + } + + // 添加函数调用消息 + fullMessages.put(JSONObject().apply { + put("role", "assistant") + put("content", "") - // 添加函数调用消息 - put(JSONObject().apply { - put("role", "assistant") - put("content", null) - - // 添加工具调用 - val toolCalls = JSONArray().apply { - val toolCall = JSONObject().apply { - put("id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) - put("type", "function") - put("function", JSONObject().apply { - put("name", functionCall.getString("name")) - put("arguments", functionCall.getString("arguments")) - }) - } - put(toolCall) + // 添加工具调用 + val toolCalls = JSONArray().apply { + val toolCall = JSONObject().apply { + put("id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) + put("type", "function") + put("function", JSONObject().apply { + put("name", functionCall.getString("name")) + put("arguments", functionCall.getString("arguments")) + }) } - put("tool_calls", toolCalls) - }) - - // 添加函数调用结果 - put(JSONObject().apply { - put("role", "tool") - put("content", functionResult) - put("tool_call_id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) - }) - } + put(toolCall) + } + put("tool_calls", toolCalls) + }) + + // 添加函数调用结果 + fullMessages.put(JSONObject().apply { + put("role", "tool") + put("content", functionResult) + put("tool_call_id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) + }) + + // 添加一个带有content的assistant消息，确保API请求不会因为缺少content而失败 + fullMessages.put(JSONObject().apply { + put("role", "assistant") + put("content", "") // 空内容，让模型生成新的回复 + }) // 发送完整对话 - sendMessageStream(fullMessages, systemPrompt, callback) + sendMessageStream(fullMessages, callback) } catch (e: Exception) { callback.onError(OpenAIException("发送函数调用结果失败: ${e.message}")) diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt index 5c93e5be6..70af36c13 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt @@ -83,7 +83,6 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { "sendMessage" -> { val messagesRaw = call.argument>>("messages") ?: emptyList() - val systemPrompt = call.argument("systemPrompt") ?: "" // 转换消息格式 val messages = JSONArray() @@ -94,7 +93,7 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { // 在后台线程执行请求 executor.execute { try { - val response = openAIService.sendMessage(messages, systemPrompt) + val response = openAIService.sendMessage(messages) // 在主线程返回结果 android.os.Handler(android.os.Looper.getMainLooper()).post { result.success(response) @@ -110,7 +109,6 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { "sendMessageStream" -> { val messagesRaw = call.argument>>("messages") ?: emptyList() - val systemPrompt = call.argument("systemPrompt") ?: "" // 检查事件接收器 if (eventSink == null) { @@ -129,7 +127,6 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { try { openAIService.sendMessageStream( messages = messages, - systemPrompt = systemPrompt, callback = object : OpenAIService.StreamCallback { override fun onToken(token: String) { // 发送token事件 @@ -177,7 +174,6 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { "sendFunctionCallResult" -> { val messagesRaw = call.argument>>("messages") ?: emptyList() - val systemPrompt = call.argument("systemPrompt") ?: "" val functionCallRaw = call.argument>("functionCall") ?: emptyMap() val functionResult = call.argument("functionResult") ?: "" @@ -201,7 +197,6 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { try { openAIService.sendFunctionCallResult( messages = messages, - systemPrompt = systemPrompt, functionCall = functionCall, functionResult = functionResult, callback = object : OpenAIService.StreamCallback { diff --git a/local_plugins/open_ai_service/lib/open_ai_service.dart b/local_plugins/open_ai_service/lib/open_ai_service.dart index b946d1bff..7c6fe8024 100644 --- a/local_plugins/open_ai_service/lib/open_ai_service.dart +++ b/local_plugins/open_ai_service/lib/open_ai_service.dart @@ -184,17 +184,14 @@ class OpenAIService { /// 发送消息（非流式输出） /// /// [messages] 消息列表 - /// [systemPrompt] 系统提示词 Future sendMessage({ required List> messages, - required String systemPrompt, }) async { try { final result = await _channel.invokeMethod( 'sendMessage', { 'messages': messages, - 'systemPrompt': systemPrompt, }, ); @@ -211,19 +208,16 @@ class OpenAIService { /// 发送消息（流式输出） /// /// [messages] 消息列表 - /// [systemPrompt] 系统提示词 /// /// 返回一个布尔值，表示请求是否已开始 Future sendMessageStream({ required List> messages, - required String systemPrompt, }) async { try { final result = await _channel.invokeMethod( 'sendMessageStream', { 'messages': messages, - 'systemPrompt': systemPrompt, }, ); @@ -236,14 +230,12 @@ class OpenAIService { /// 发送函数调用结果 /// /// [messages] 消息列表 - /// [systemPrompt] 系统提示词 /// [functionCall] 函数调用信息 /// [functionResult] 函数调用结果 /// /// 返回一个布尔值，表示请求是否已开始 Future sendFunctionCallResult({ required List> messages, - required String systemPrompt, required Map functionCall, required String functionResult, }) async { @@ -252,7 +244,6 @@ class OpenAIService { 'sendFunctionCallResult', { 'messages': messages, - 'systemPrompt': systemPrompt, 'functionCall': functionCall, 'functionResult': functionResult, }, @@ -264,35 +255,6 @@ class OpenAIService { } } - /// 处理流式事件 - /// - /// [onToken] 收到token时调用 - /// [onComplete] 完成时调用 - /// [onError] 错误时调用 - /// [onFunctionCall] 函数调用时调用 - StreamSubscription processEvents({ - void Function(String token)? onToken, - void Function()? onComplete, - void Function(String error)? onError, - void Function(Map functionCall)? onFunctionCall, - }) { - return eventStream.listen((event) { - switch (event.type) { - case OpenAIEventType.token: - onToken?.call(event.content as String); - break; - case OpenAIEventType.complete: - onComplete?.call(); - break; - case OpenAIEventType.error: - onError?.call(event.content as String); - break; - case OpenAIEventType.functionCall: - onFunctionCall?.call(Map.from(event.content as Map)); - break; - } - }); - } /// 从JSON字符串解析函数调用 Map parseFunctionCall(String functionCallJson) { diff --git a/test.json b/test.json new file mode 100644 index 000000000..703948620 --- /dev/null +++ b/test.json @@ -0,0 +1 @@ +curl -v -X POST -H 'Content-Type: application/json' -H 'Authorization: Bearer 168deb3d-fd0c-4912-b9f1-aaee5c6743e6' -H 'Accept: text/event-stream' -d '{"model":"bot-20250405211523-l7c9r","messages":[{"role":"system","content":" 你是一个智能语音助手，能够简洁明了地回答用户的问题。\n时刻关心用户的情绪和需求，主动提供鼓励和温暖。\n\n语言风格活泼、亲切，能够幽默地互动，陪伴用户，缓解压力，增添生活乐趣。\n\n请始终以用户为中心，保持回应的高效性、准确性和温暖体贴，成为用户真正的灵魂伴侣。\n \n 当用户说\"退出\"、\"再见\"、\"结束对话\"等类似意图时，你应该使用exit_interaction函数来结束对话，\n 并在结束前说一句友好的告别语，例如\"再见，有需要随时找我\"。"},{"role":"system","content":" 你是一个智能语音助手，能够简洁明了地回答用户的问题。\n时刻关心用户的情绪和需求，主动提供鼓励和温暖。\n\n语言风格活泼、亲切，能够幽默地互动，陪伴用户，缓解压力，增添生活乐趣。\n\n请始终以用户为中心，保持回应的高效性、准确性和温暖体贴，成为用户真正的灵魂伴侣。\n \n 当用户说\"退出\"、\"再见\"、\"结束对话\"等类似意图时，你应该使用exit_interaction函数来结束对话，\n 并在结束前说一句友好的告别语，例如\"再见，有需要随时找我\"。"},{"role":"user","content":"退下吧。"},{"role":"assistant","content":"","tool_calls":[{"id":"call_8k680azmfc4thqrrnwpwqxah","type":"function","function":{"name":"exit_interaction","arguments":" {}"}}]},{"role":"tool","content":"{\"result\": \"已退出语音交互\"}","tool_call_id":"call_8k680azmfc4thqrrnwpwqxah"}],"temperature":0.7,"max_tokens":2000,"stream":true,"tools":[{"type":"function","function":{"name":"exit_interaction","description":"退出当前语音交互","parameters":{"type":"object","properties":{},"required":[]}}}]}' 'https://ark.cn-beijing.volces.com/api/v3/bots/chat/completions' \ No newline at end of file