From 08471f2d3907590ae32e98f379ae537631156bf2 Mon Sep 17 00:00:00 2001 From: wolfplus Date: Sat, 5 Apr 2025 21:29:31 +0100 Subject: [PATCH] add --- a | 18 + android/app/build.gradle.kts | 1 + .../yunqiinnovation/deepsound/MainActivity.kt | 10 +- .../deepsound/OpenAIService.kt | 606 ------------------ .../deepsound/VoiceFunctionHandler.kt | 2 +- .../deepsound/VoiceInteractionHandler.kt | 450 +++++++++++++ .../deepsound/VoiceInteractionService.kt | 438 ++----------- android/build.gradle.kts | 1 + android/settings.gradle.kts | 2 + .../services/voice_interaction_service.dart | 6 +- .../azure_speech/android/build.gradle.kts | 9 +- local_plugins/open_ai_service/README.md | 348 ++++++++++ .../open_ai_service/android/build.gradle.kts | 37 ++ .../android/settings.gradle.kts | 1 + .../android/src/main/AndroidManifest.xml | 11 + .../open_ai_service/OpenAIService.kt | 456 +++++++++++++ .../open_ai_service/OpenAIServicePlugin.kt | 322 ++++++++++ .../ios/Classes/OpenAIService.swift | 473 ++++++++++++++ .../ios/Classes/OpenAIServicePlugin.swift | 215 +++++++ .../open_ai_service/lib/open_ai_service.dart | 305 +++++++++ local_plugins/open_ai_service/pubspec.yaml | 26 + pubspec.yaml | 2 + 22 files changed, 2744 insertions(+), 995 deletions(-) create mode 100644 a delete mode 100644 android/app/src/main/kotlin/com/yunqiinnovation/deepsound/OpenAIService.kt create mode 100644 android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt create mode 100644 local_plugins/open_ai_service/README.md create mode 100644 local_plugins/open_ai_service/android/build.gradle.kts create mode 100644 local_plugins/open_ai_service/android/settings.gradle.kts create mode 100644 local_plugins/open_ai_service/android/src/main/AndroidManifest.xml create mode 100644 local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt create mode 100644 local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt create mode 100644 local_plugins/open_ai_service/ios/Classes/OpenAIService.swift create mode 100644 local_plugins/open_ai_service/ios/Classes/OpenAIServicePlugin.swift create mode 100644 local_plugins/open_ai_service/lib/open_ai_service.dart create mode 100644 local_plugins/open_ai_service/pubspec.yaml diff --git a/a b/a new file mode 100644 index 000000000..7133736ae --- /dev/null +++ b/a @@ -0,0 +1,18 @@ +curl 'https://ark.cn-beijing.volces.com/api/v3/bots/chat/completions' \ +-H "Authorization: Bearer 168deb3d-fd0c-4912-b9f1-aaee5c6743e6" \ +-H 'Content-Type: application/json' \ +-d '{ + "model": "bot-20250405211523-l7c9r", + "stream": true, + "stream_options": {"include_usage": true}, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Hello!" + } + ] +}' \ No newline at end of file diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 78b816b75..b0a39cbd4 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -92,6 +92,7 @@ dependencies { // 添加本地插件模块依赖 implementation(project(":azure_speech")) + implementation(project(":open_ai_service")) // 添加OkHttp依赖 implementation("com.squareup.okhttp3:okhttp:4.9.3") diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt index df36efb44..2367f8241 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt @@ -135,6 +135,7 @@ class MainActivity: FlutterActivity() { var volcanoAiApiKey: String = "" var openaiApiKey: String = "" var openaiBaseUrl: String? = null + var openaiModel: String? = null // 安全存储相关常量 private const val SECURE_PREFS_FILENAME = "deep_voice_secure_prefs" @@ -143,7 +144,9 @@ class MainActivity: FlutterActivity() { private const val KEY_VOLCANO_AI_API_KEY = "volcano_ai_api_key" private const val KEY_OPENAI_API_KEY = "openai_api_key" private const val KEY_OPENAI_BASE_URL = "openai_base_url" + private const val KEY_OPENAI_MODEL = "openai_model" private const val KEY_MCP_SERVER_ENDPOINT = "mcp_server_endpoint" + // 会话管理 private const val KEY_SESSION_ID = "session_id" private var currentSessionId = "" @@ -180,6 +183,7 @@ class MainActivity: FlutterActivity() { .putString(KEY_OPENAI_API_KEY, openaiApiKey) .putString(KEY_OPENAI_BASE_URL, openaiBaseUrl) .putString(KEY_SESSION_ID, currentSessionId) + .putString(KEY_OPENAI_MODEL, openaiModel) .apply() FileLogger.d("MainActivity", "密钥已安全保存到加密存储中") @@ -225,6 +229,7 @@ class MainActivity: FlutterActivity() { volcanoAiApiKey = sharedPreferences.getString(KEY_VOLCANO_AI_API_KEY, "") ?: "" openaiApiKey = sharedPreferences.getString(KEY_OPENAI_API_KEY, "") ?: "" openaiBaseUrl = sharedPreferences.getString(KEY_OPENAI_BASE_URL, null) + openaiModel = sharedPreferences.getString(KEY_OPENAI_MODEL, null) FileLogger.d("MainActivity", "已从加密存储加载密钥") // 检查是否成功获取所有必要密钥 @@ -460,7 +465,7 @@ class MainActivity: FlutterActivity() { val region = call.argument("azure_speech_region") ?: "" val openaiKey = call.argument("openai_api_key") ?: "" val baseUrl = call.argument("openai_base_url") - + val model = call.argument("openai_model") // 设置 Azure Speech 和 AI 配置 azureSpeechKey = key azureSpeechRegion = region @@ -468,6 +473,9 @@ class MainActivity: FlutterActivity() { if (baseUrl != null) { openaiBaseUrl = baseUrl } + if (model != null) { + openaiModel = model + } // 保存密钥到安全存储 saveKeysToSecureStorage(applicationContext) diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/OpenAIService.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/OpenAIService.kt deleted file mode 100644 index 39907e6d5..000000000 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/OpenAIService.kt +++ /dev/null @@ -1,606 +0,0 @@ -package com.yunqiinnovation.deepsound - -import android.util.Log -import okhttp3.* -import okhttp3.MediaType.Companion.toMediaTypeOrNull -import okhttp3.RequestBody.Companion.toRequestBody -import org.json.JSONArray -import org.json.JSONObject -import java.io.IOException -import java.util.concurrent.TimeUnit -import com.yunqiinnovation.deepsound.core.utils.FileLogger - -/** - * OpenAI服务的原生实现 - */ -class OpenAIService() { - private val TAG = "OpenAIService" - private var baseUrl = "https://api.openai.com/v1/chat/completions" - private val client = OkHttpClient.Builder() - .connectTimeout(30, TimeUnit.SECONDS) - .readTimeout(30, TimeUnit.SECONDS) - .writeTimeout(30, TimeUnit.SECONDS) - .build() - - private var apiKey: String = "" - private var isInitialized = false - private var model: String = "doubao-1-5-lite-32k-250115" // 默认模型 - - // 用于存储注册的函数 - private val registeredFunctions = mutableListOf() - - - /** - * 初始化OpenAI服务 - */ - fun initialize(apiKey: String, baseUrl: String = ""): Boolean { - this.apiKey = apiKey - if (baseUrl.isNotEmpty()) { - this.baseUrl = baseUrl - } - isInitialized = apiKey.isNotEmpty() - return isInitialized - } - - /** - * 注册函数 - */ - fun registerFunction(name: String, description: String, parameters: JSONObject): Boolean { - try { - val function = JSONObject().apply { - put("name", name) - put("description", description) - put("parameters", parameters) - } - - // 检查是否已存在相同名称的函数 - val existingIndex = registeredFunctions.indexOfFirst { - it.getString("name") == name - } - - if (existingIndex >= 0) { - // 如果已存在，则替换 - registeredFunctions[existingIndex] = function - } else { - // 如果不存在，则添加 - registeredFunctions.add(function) - } - - return true - } catch (e: Exception) { - return false - } - } - - /** - * 发送消息（非流式输出） - */ - @Throws(OpenAIException::class) - fun sendMessage(messages: JSONArray, systemPrompt: String): String { - if (!isInitialized || apiKey.isEmpty()) { - throw OpenAIException("OpenAI服务未初始化") - } - - val fullMessages = JSONArray().apply { - put(JSONObject().apply { - put("role", "system") - put("content", systemPrompt) - }) - for (i in 0 until messages.length()) { - put(messages.getJSONObject(i)) - } - } - - val requestBody = JSONObject().apply { - put("model", model) - put("messages", fullMessages) - put("temperature", 0.7) - put("max_tokens", 2000) - put("stream", false) - - // 如果有注册的函数，则添加到请求中 - if (registeredFunctions.isNotEmpty()) { - val tools = JSONArray() - for (function in registeredFunctions) { - val tool = JSONObject().apply { - put("type", "function") - put("function", function) - } - tools.put(tool) - } - put("tools", tools) - } - } - - val mediaType = "application/json".toMediaTypeOrNull() - val request = Request.Builder() - .url(baseUrl) - .addHeader("Content-Type", "application/json") - .addHeader("Authorization", "Bearer $apiKey") - .post(requestBody.toString().toRequestBody(mediaType)) - .build() - - try { - client.newCall(request).execute().use { response -> - if (!response.isSuccessful) { - throw OpenAIException("API调用失败: ${response.code}") - } - - val responseBody = response.body?.string() ?: throw OpenAIException("Empty response") - val jsonResponse = JSONObject(responseBody) - - // 检查是否有函数调用 - if (jsonResponse.has("choices") && - jsonResponse.getJSONArray("choices").length() > 0) { - - val choice = jsonResponse.getJSONArray("choices").getJSONObject(0) - - // 检查是否是函数调用 - if (choice.has("message")) { - val message = choice.getJSONObject("message") - - // 检查是否有工具调用 - if (message.has("tool_calls")) { - val toolCalls = message.getJSONArray("tool_calls") - if (toolCalls.length() > 0) { - val toolCall = toolCalls.getJSONObject(0) - if (toolCall.has("function")) { - val function = toolCall.getJSONObject("function") - val functionCall = JSONObject().apply { - put("name", function.getString("name")) - put("arguments", function.getString("arguments")) - put("id", toolCall.getString("id")) - } - return functionCall.toString() - } - } - } - - // 如果没有工具调用，返回消息内容 - if (message.has("content")) { - return message.getString("content") - } - } - } - - throw OpenAIException("Invalid response format") - } - } catch (e: Exception) { - if (e is OpenAIException) throw e - throw OpenAIException("Failed to communicate with AI service: ${e.message}") - } - } - - /** - * 发送消息（流式输出） - */ - fun sendMessageStream(messages: JSONArray, systemPrompt: String, callback: StreamCallback) { - if (!isInitialized || apiKey.isEmpty()) { - callback.onError(OpenAIException("OpenAI服务未初始化")) - return - } - - val fullMessages = JSONArray().apply { - put(JSONObject().apply { - put("role", "system") - put("content", systemPrompt) - }) - for (i in 0 until messages.length()) { - put(messages.getJSONObject(i)) - } - } - - val requestBody = JSONObject().apply { - put("model", model) - put("messages", fullMessages) - put("temperature", 0.7) - put("max_tokens", 2000) - put("stream", true) - - // 如果有注册的函数，则添加到请求中 - if (registeredFunctions.isNotEmpty()) { - val tools = JSONArray() - for (function in registeredFunctions) { - val tool = JSONObject().apply { - put("type", "function") - put("function", function) - } - tools.put(tool) - } - put("tools", tools) - } - } - - val mediaType = "application/json".toMediaTypeOrNull() - val request = Request.Builder() - .url(baseUrl) - .addHeader("Content-Type", "application/json") - .addHeader("Authorization", "Bearer $apiKey") - .addHeader("Accept", "text/event-stream") - .post(requestBody.toString().toRequestBody(mediaType)) - .build() - - client.newCall(request).enqueue(object : Callback { - override fun onFailure(call: Call, e: IOException) { - callback.onError(OpenAIException(e.message ?: "请求失败")) - } - - override fun onResponse(call: Call, response: Response) { - if (!response.isSuccessful) { - callback.onError(OpenAIException("API调用失败: ${response.code}")) - return - } - - val responseBody = response.body ?: return - val source = responseBody.source() - - try { - // 预取数据到缓冲区 - source.request(Long.MAX_VALUE) - val bufferedSource = source.buffer - - // 用于存储函数调用的各个部分 - val finalToolCalls = mutableMapOf() - - while (!bufferedSource.exhausted()) { - val line = bufferedSource.readUtf8Line() ?: continue - - if (line.isEmpty()) continue - if (line.startsWith("data: ")) { - val data = line.substring(6) - if (data == "[DONE]") { - callback.onComplete() - break - } - - try { - val jsonData = JSONObject(data) - if (jsonData.has("choices") && - jsonData.getJSONArray("choices").length() > 0) { - - val choice = jsonData.getJSONArray("choices").getJSONObject(0) - - // 检查是否有delta - if (choice.has("delta")) { - val delta = choice.getJSONObject("delta") - - // 检查是否有工具调用 - if (delta.has("tool_calls")) { - val toolCalls = delta.getJSONArray("tool_calls") - for (i in 0 until toolCalls.length()) { - val toolCall = toolCalls.getJSONObject(i) - val index = toolCall.optInt("index", i) - - // 如果是新的工具调用，初始化 - if (!finalToolCalls.containsKey(index)) { - finalToolCalls[index] = ToolCallInfo() - } - - // 获取ID - if (toolCall.has("id")) { - finalToolCalls[index]?.id = toolCall.getString("id") - } - - // 处理函数信息 - if (toolCall.has("function")) { - val function = toolCall.getJSONObject("function") - - if (function.has("name")) { - finalToolCalls[index]?.name = function.getString("name") - } - - if (function.has("arguments")) { - finalToolCalls[index]?.arguments += function.getString("arguments") - } - } - } - continue - } - - // 如果有内容，发送给回调 - if (delta.has("content") && !delta.isNull("content")) { - val content = delta.getString("content") - callback.onToken(content) - } - } - } - } catch (e: Exception) { - // 忽略无效的JSON - continue - } - } - } - - // 处理完整的函数调用 - for ((_, toolCallInfo) in finalToolCalls) { - if (toolCallInfo.name.isNotEmpty()) { - try { - // 创建函数调用对象 - val functionCall = JSONObject().apply { - put("id", toolCallInfo.id) - put("name", toolCallInfo.name) - put("arguments", toolCallInfo.arguments.trim()) - } - - callback.onFunctionCall(functionCall) - } catch (e: Exception) { - // 出错时使用空参数 - val functionCall = JSONObject().apply { - put("id", toolCallInfo.id) - put("name", toolCallInfo.name) - put("arguments", "{}") - } - callback.onFunctionCall(functionCall) - } - } - } - } catch (e: Exception) { - callback.onError(OpenAIException("处理流式响应出错: ${e.message}")) - } finally { - response.close() - } - } - }) - } - - /** - * 发送函数调用结果 - */ - fun sendFunctionCallResult( - messages: JSONArray, - systemPrompt: String, - functionCall: JSONObject, - functionResult: String, - callback: StreamCallback - ) { - if (!isInitialized || apiKey.isEmpty()) { - callback.onError(OpenAIException("OpenAI服务未初始化")) - return - } - - // 构建完整的消息历史 - val fullMessages = JSONArray().apply { - // 添加系统提示 - put(JSONObject().apply { - put("role", "system") - put("content", systemPrompt) - }) - - // 添加历史消息 - for (i in 0 until messages.length()) { - put(messages.getJSONObject(i)) - } - - // 添加函数调用信息 - put(JSONObject().apply { - put("role", "assistant") - put("content", null) - put("tool_calls", JSONArray().apply { - put(JSONObject().apply { - put("id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) - put("type", "function") - put("function", JSONObject().apply { - put("name", functionCall.getString("name")) - put("arguments", functionCall.getString("arguments")) - }) - }) - }) - }) - - // 添加函数返回结果 - put(JSONObject().apply { - put("role", "tool") - put("tool_call_id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) - put("content", functionResult) - }) - } - - // 构建请求 - val requestBody = JSONObject().apply { - put("model", model) - put("messages", fullMessages) - put("temperature", 0.7) - put("max_tokens", 2000) - put("stream", true) - - // 如果有注册的函数，则添加到请求中 - if (registeredFunctions.isNotEmpty()) { - val tools = JSONArray() - for (function in registeredFunctions) { - val tool = JSONObject().apply { - put("type", "function") - put("function", function) - } - tools.put(tool) - } - put("tools", tools) - } - } - - val mediaType = "application/json".toMediaTypeOrNull() - val request = Request.Builder() - .url(baseUrl) - .addHeader("Content-Type", "application/json") - .addHeader("Authorization", "Bearer $apiKey") - .addHeader("Accept", "text/event-stream") - .post(requestBody.toString().toRequestBody(mediaType)) - .build() - - // 发送请求 - client.newCall(request).enqueue(object : Callback { - override fun onFailure(call: Call, e: IOException) { - callback.onError(OpenAIException(e.message ?: "请求失败")) - } - - override fun onResponse(call: Call, response: Response) { - if (!response.isSuccessful) { - callback.onError(OpenAIException("API调用失败: ${response.code}")) - return - } - - val responseBody = response.body ?: return - val source = responseBody.source() - - try { - // 预取数据到缓冲区 - source.request(Long.MAX_VALUE) - val bufferedSource = source.buffer - - // 用于存储函数调用的各个部分 - val finalToolCalls = mutableMapOf() - - while (!bufferedSource.exhausted()) { - val line = bufferedSource.readUtf8Line() ?: continue - - if (line.isEmpty()) continue - if (line.startsWith("data: ")) { - val data = line.substring(6) - if (data == "[DONE]") { - callback.onComplete() - break - } - - try { - val jsonData = JSONObject(data) - if (jsonData.has("choices") && - jsonData.getJSONArray("choices").length() > 0) { - - val choice = jsonData.getJSONArray("choices").getJSONObject(0) - - // 检查是否有delta - if (choice.has("delta")) { - val delta = choice.getJSONObject("delta") - - // 检查是否有工具调用 - if (delta.has("tool_calls")) { - val toolCalls = delta.getJSONArray("tool_calls") - for (i in 0 until toolCalls.length()) { - val toolCall = toolCalls.getJSONObject(i) - val index = toolCall.optInt("index", i) - - // 如果是新的工具调用，初始化 - if (!finalToolCalls.containsKey(index)) { - finalToolCalls[index] = ToolCallInfo() - } - - // 获取ID - if (toolCall.has("id")) { - finalToolCalls[index]?.id = toolCall.getString("id") - } - - // 处理函数信息 - if (toolCall.has("function")) { - val function = toolCall.getJSONObject("function") - - if (function.has("name")) { - finalToolCalls[index]?.name = function.getString("name") - } - - if (function.has("arguments")) { - finalToolCalls[index]?.arguments += function.getString("arguments") - } - } - } - continue - } - - // 如果有内容，发送给回调 - if (delta.has("content") && !delta.isNull("content")) { - val content = delta.getString("content") - callback.onToken(content) - } - } - } - } catch (e: Exception) { - // 忽略无效的JSON - continue - } - } - } - - // 处理完整的函数调用 - for ((_, toolCallInfo) in finalToolCalls) { - if (toolCallInfo.name.isNotEmpty()) { - try { - // 创建函数调用对象 - val functionCall = JSONObject().apply { - put("id", toolCallInfo.id) - put("name", toolCallInfo.name) - put("arguments", toolCallInfo.arguments.trim()) - } - - callback.onFunctionCall(functionCall) - } catch (e: Exception) { - // 出错时使用空参数 - val functionCall = JSONObject().apply { - put("id", toolCallInfo.id) - put("name", toolCallInfo.name) - put("arguments", "{}") - } - callback.onFunctionCall(functionCall) - } - } - } - } catch (e: Exception) { - callback.onError(OpenAIException("处理流式响应出错: ${e.message}")) - } finally { - response.close() - } - } - }) - } - - /** - * 创建用户消息 - */ - fun createUserMessage(content: String): JSONObject { - return JSONObject().apply { - put("role", "user") - put("content", content) - } - } - - /** - * 创建系统消息 - */ - fun createSystemMessage(content: String): JSONObject { - return JSONObject().apply { - put("role", "system") - put("content", content) - } - } - - /** - * 创建助手消息 - */ - fun createAssistantMessage(content: String): JSONObject { - return JSONObject().apply { - put("role", "assistant") - put("content", content) - } - } - - /** - * 流式输出回调接口 - */ - interface StreamCallback { - fun onToken(token: String) - fun onComplete() - fun onError(e: Exception) - fun onFunctionCall(functionCall: JSONObject) {} - } - - /** - * 用于存储工具调用信息的辅助类 - */ - private class ToolCallInfo { - var id: String = "" - var name: String = "" - var arguments: String = "" - } -} - -/** - * OpenAI异常 - */ -class OpenAIException(message: String) : Exception(message) \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt index a89441506..78a55b642 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt @@ -2,7 +2,7 @@ package com.yunqiinnovation.deepsound import org.json.JSONArray import org.json.JSONObject -import com.yunqiinnovation.deepsound.OpenAIService +import com.yunqiinnovation.open_ai_service.OpenAIService import com.yunqiinnovation.deepsound.core.utils.FileLogger /** diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt new file mode 100644 index 000000000..fa179c279 --- /dev/null +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt @@ -0,0 +1,450 @@ +package com.yunqiinnovation.deepsound + +import android.content.Context +import org.json.JSONArray +import org.json.JSONObject +import android.util.Log +import com.yunqiinnovation.deepsound.core.utils.FileLogger +import com.yunqiinnovation.azure_speech.AzureAsrHelper +import com.yunqiinnovation.azure_speech.AzureTtsHelper +import com.yunqiinnovation.open_ai_service.OpenAIService + + +/** + * 语音交互处理器 - 处理语音识别、TTS和AI对话相关逻辑 + */ +class VoiceInteractionHandler( + private val context: Context, + private val azureSpeechKey: String, + private val azureSpeechRegion: String, + private val openaiApiKey: String, + private val openaiBaseUrl: String = "", + private val openaiModel: String = "" +) { + private val TAG = "VoiceInteractionHandler" + + // Azure服务 + private var azureAsrHelper: AzureAsrHelper? = null + private var azureTtsHelper: AzureTtsHelper? = null + + // OpenAI服务 + private val openAIService = OpenAIService() + + // 系统提示词 + private val systemPrompt = """ + 你是一个智能语音助手，能够简洁明了地回答用户的问题。 +时刻关心用户的情绪和需求，主动提供鼓励和温暖。 + +语言风格活泼、亲切，能够幽默地互动，陪伴用户，缓解压力，增添生活乐趣。 + +请始终以用户为中心，保持回应的高效性、准确性和温暖体贴，成为用户真正的灵魂伴侣。 + + 当用户说"退出"、"再见"、"结束对话"等类似意图时，你应该使用exit_interaction函数来结束对话， + 并在结束前说一句友好的告别语，例如"再见，有需要随时找我"。 + """.trimIndent() + + // 语音功能处理 + private val voiceFunctionHandler = VoiceFunctionHandler(openAIService, systemPrompt) + + // 当前用户输入 + private var currentUserInput = "" + + // 状态 + private var isInitialized = false + var isRecognitionActive = false + private set + var isTtsSpeaking = false + private set + var hasSpeechDetected = false + private set + + // 回调 + private var callback: InteractionCallback? = null + + /** + * 初始化 + */ + fun initialize(): Boolean { + if (isInitialized) return true + + try { + // 初始化Azure ASR + azureAsrHelper = AzureAsrHelper(context).apply { + initialize(azureSpeechKey, azureSpeechRegion) + } + + // 初始化Azure TTS + azureTtsHelper = AzureTtsHelper(context).apply { + initialize(azureSpeechKey, azureSpeechRegion) + } + + // 初始化OpenAI服务 + openAIService.initialize(openaiApiKey, openaiBaseUrl, openaiModel) + + // 初始化语音功能处理器 + voiceFunctionHandler.initialize() + + isInitialized = true + return true + } catch (e: Exception) { + FileLogger.e(TAG, "初始化失败: ${e.message}", e) + return false + } + } + + /** + * 设置回调 + */ + fun setCallback(callback: InteractionCallback) { + this.callback = callback + } + + /** + * 开始语音识别 + */ + fun startRecognition() { + if (isRecognitionActive) return + + // 检查录音权限 + if (!checkRecordAudioPermission()) { + callback?.onError("需要录音权限，请在设置中授予权限") + return + } + + isRecognitionActive = true + hasSpeechDetected = false + notifyStateChanged() + + try { + azureAsrHelper?.startContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { + override fun onRecognizing(recognizing: String, detectedLanguage: String) { + if (recognizing.isNotEmpty()) { + hasSpeechDetected = true + stopTts() + notifyStateChanged() + } + } + + override fun onResult(result: String, detectedLanguage: String) { + if (result.isNotEmpty()) { + processWithOpenAI(result) + } + + // 重置状态，继续识别 + hasSpeechDetected = false + notifyStateChanged() + } + + override fun onSessionStarted() { + notifyStateChanged() + } + + override fun onSessionStopped() { + isRecognitionActive = false + notifyStateChanged() + } + + override fun onCanceled(reason: String, errorDetails: String) { + isRecognitionActive = false + notifyStateChanged() + } + + override fun onError(error: String) { + isRecognitionActive = false + callback?.onError("语音识别出错") + notifyStateChanged() + } + + override fun onSuccess(message: String) { + // 处理成功事件 + } + }) + } catch (e: Exception) { + isRecognitionActive = false + FileLogger.e(TAG, "启动语音识别失败: ${e.message}", e) + callback?.onError("启动语音识别失败") + notifyStateChanged() + } + } + + /** + * 停止语音识别 + */ + fun stopRecognition() { + if (!isRecognitionActive) return + + FileLogger.d(TAG, "停止语音识别") + + try { + azureAsrHelper?.stopContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { + override fun onResult(result: String, detectedLanguage: String) {} + override fun onRecognizing(recognizing: String, detectedLanguage: String) {} + override fun onSessionStarted() {} + override fun onSessionStopped() { + isRecognitionActive = false + FileLogger.d(TAG, "语音识别会话已停止") + notifyStateChanged() + } + override fun onCanceled(reason: String, errorDetails: String) { + isRecognitionActive = false + FileLogger.d(TAG, "语音识别已取消: $reason") + notifyStateChanged() + } + override fun onError(error: String) { + isRecognitionActive = false + FileLogger.e(TAG, "停止语音识别时出错: $error") + notifyStateChanged() + } + override fun onSuccess(message: String) { + isRecognitionActive = false + FileLogger.d(TAG, "语音识别已停止: $message") + notifyStateChanged() + } + }) + } catch (e: Exception) { + FileLogger.e(TAG, "停止语音识别异常: ${e.message}", e) + // 确保状态一致性 + isRecognitionActive = false + notifyStateChanged() + } + } + + /** + * 使用OpenAI处理语音识别结果 + */ + private fun processWithOpenAI(text: String) { + // 保存当前用户输入，用于后续同步聊天记录 + currentUserInput = text + + Thread { + try { + val messages = JSONArray().apply { + put(openAIService.createUserMessage(text)) + } + + // 创建响应构建器 + val responseBuilder = StringBuilder() + + openAIService.sendMessageStream( + messages = messages, + systemPrompt = systemPrompt, + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + // 累加响应内容 + responseBuilder.append(token) + } + + override fun onComplete() { + // 处理完整响应 + val response = responseBuilder.toString() + if (response.isNotEmpty()) { + // 播放AI回复 + Log.d(TAG, "AI 回复: $response") + + speakAIResponse(response) + + // 同步聊天记录到Flutter端 + sendChatHistoryUpdate("personal_assistant", text, response) + } + } + + override fun onError(e: Exception) { + FileLogger.e(TAG, "AI流式处理出错: ${e.message}", e) + callback?.onError("AI处理出错") + } + + override fun onFunctionCall(call: JSONObject) { + FileLogger.d(TAG, "收到函数调用请求: ${call.getString("name")}") + + // 使用函数处理器处理函数调用 + val handled = voiceFunctionHandler.handleFunctionCall( + functionCall = call, + messages = messages, + callback = object : VoiceFunctionHandler.FunctionCallCallback { + override fun onTokenReceived(token: String) { + responseBuilder.append(token) + } + + override fun onComplete() { + val response = responseBuilder.toString() + if (response.isNotEmpty()) { + // 播放AI回复 + Log.d(TAG, "AI Function Call 回复: $response") + speakAIResponse(response) + + // 同步聊天记录到Flutter端 + sendChatHistoryUpdate("personal_assistant", text, response) + } + notifyStateChanged() + } + + override fun onError(message: String) { + FileLogger.e(TAG, "函数处理出错: $message") + callback?.onError(message) + } + + override fun onFunctionCall(nestedCall: JSONObject) { + FileLogger.d(TAG, "收到嵌套函数调用: ${nestedCall.getString("name")}") + // 递归处理嵌套函数调用 + voiceFunctionHandler.handleFunctionCall( + functionCall = nestedCall, + messages = messages, + callback = this + ) + } + + override fun onExitWithMessage(farewell: String) { + // 播放退出消息 + speakAIResponse(farewell) + + // 同步聊天记录 + sendChatHistoryUpdate("personal_assistant", text, farewell) + + // 停止语音识别 + stopRecognition() + } + } + ) + + if (!handled) { + // 如果函数没有被处理，作为普通文本处理 + FileLogger.d(TAG, "函数未处理，作为普通文本处理") + speakAIResponse("我无法处理这个请求") + sendChatHistoryUpdate("personal_assistant", text, "我无法处理这个请求") + } + } + } + ) + + } catch (e: Exception) { + FileLogger.e(TAG, "AI处理出错: ${e.message}", e) + callback?.onError("AI处理出错") + } + }.start() + } + + /** + * 播放TTS + */ + fun playTts(text: String, callback: TtsCallback? = null) { + isTtsSpeaking = true + notifyStateChanged() + + azureTtsHelper?.speakText(text, object : AzureTtsHelper.TTSCallback { + override fun onSuccess(message: String) { + isTtsSpeaking = false + notifyStateChanged() + callback?.onComplete() + } + + override fun onError(error: String) { + isTtsSpeaking = false + notifyStateChanged() + callback?.onError(error) + } + }) + } + + /** + * 停止TTS播放 + */ + fun stopTts() { + if (isTtsSpeaking) { + azureTtsHelper?.stopSpeaking() + isTtsSpeaking = false + notifyStateChanged() + } + } + + /** + * 播放AI回复 + */ + private fun speakAIResponse(text: String) { + isTtsSpeaking = true + notifyStateChanged() + + azureTtsHelper?.speakText(text, object : AzureTtsHelper.TTSCallback { + override fun onSuccess(message: String) { + isTtsSpeaking = false + notifyStateChanged() + } + + override fun onError(error: String) { + isTtsSpeaking = false + notifyStateChanged() + } + }) + } + + /** + * 释放资源 + */ + fun dispose() { + // 停止语音识别 + stopRecognition() + + // 停止TTS播放 + stopTts() + + // 释放Azure资源 + azureAsrHelper?.let { + FileLogger.d(TAG, "关闭Azure ASR服务") + it.dispose() + } + + azureTtsHelper?.let { + FileLogger.d(TAG, "关闭Azure TTS服务") + it.dispose() + } + + FileLogger.d(TAG, "语音交互处理器资源已释放") + } + + /** + * 检查录音权限 + */ + private fun checkRecordAudioPermission(): Boolean { + val permission = android.Manifest.permission.RECORD_AUDIO + val result = context.checkCallingOrSelfPermission(permission) + return result == android.content.pm.PackageManager.PERMISSION_GRANTED + } + + /** + * 通知状态变化 + */ + private fun notifyStateChanged() { + callback?.onStateChanged(isRecognitionActive, isTtsSpeaking, hasSpeechDetected) + } + + /** + * 发送聊天历史更新 + */ + private fun sendChatHistoryUpdate(agentId: String, userMessage: String, assistantMessage: String) { + val intent = android.content.Intent(VoiceInteractionService.ACTION_CHAT_HISTORY_UPDATED).apply { + putExtra("agentId", agentId) + putExtra("userMessage", userMessage) + putExtra("assistantMessage", assistantMessage) + putExtra("timestamp", System.currentTimeMillis()) + } + + // 发送广播 + context.sendBroadcast(intent) + } + + /** + * 交互回调接口 + */ + interface InteractionCallback { + fun onStateChanged(isRecognitionActive: Boolean, isTtsSpeaking: Boolean, hasSpeechDetected: Boolean) + fun onError(message: String) + fun onPromptRequest(message: String) + } + + /** + * TTS回调接口 + */ + interface TtsCallback { + fun onComplete() + fun onError(error: String) + } +} \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt index ad20ab6b7..76faa6f33 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt @@ -29,14 +29,14 @@ import android.net.Uri import com.yunqiinnovation.deepsound.core.utils.FileLogger import com.yunqiinnovation.azure_speech.AzureAsrHelper import com.yunqiinnovation.azure_speech.AzureTtsHelper -import com.yunqiinnovation.deepsound.OpenAIService +import com.yunqiinnovation.open_ai_service.OpenAIService /** * 后台语音交互 Service： * 1) 前台服务，确保不会被系统轻易杀死 * 2) MediaSession 捕获蓝牙耳机按键 - * 3) 处理录音/语音识别 + * 3) 负责唤醒控制和服务生命周期管理 */ class VoiceInteractionService : Service() { @@ -62,10 +62,7 @@ class VoiceInteractionService : Service() { // 服务状态 private var isActive = false // 服务是否活跃 - private var isRecognitionActive = false // 语音识别是否活跃 private var isTimeoutPaused = false // 是否因超时暂停 - private var hasSpeechDetected = false // 是否检测到语音 - private var isTtsSpeaking = false // 是否正在播放TTS // 按键处理 private var lastKeyEventTime = 0L @@ -74,17 +71,12 @@ class VoiceInteractionService : Service() { // 活动时间 private var lastActivityTime = 0L - // 当前用户输入 - private var currentUserInput = "" - - // 服务组件 private lateinit var mediaSession: MediaSessionCompat private lateinit var audioManager: AudioManager - private lateinit var azureAsrHelper: AzureAsrHelper - private lateinit var azureTtsHelper: AzureTtsHelper - private lateinit var openAIService: OpenAIService - private lateinit var functionHandler: VoiceFunctionHandler + + // 语音交互处理器 + private lateinit var voiceInteractionHandler: VoiceInteractionHandler // 定时器 private val handler = Handler(Looper.getMainLooper()) @@ -94,18 +86,6 @@ class VoiceInteractionService : Service() { handler.postDelayed(this, 1000) // 每秒执行一次 } } - - // 系统提示词 - private val systemPrompt = """ - 你是一个智能语音助手，能够简洁明了地回答用户的问题。 - 请保持回答简短、准确，避免过长的解释。 - 如果用户的问题不清楚，请礼貌地请求澄清。 - 不要使用复杂的术语，除非用户明确要求。 - 用户用语音和你交互。 - - 当用户说"退出"、"再见"、"结束对话"等类似意图时，你应该使用exit_interaction函数来结束对话， - 并在结束前说一句友好的告别语，例如"再见，有需要随时找我"。 - """.trimIndent() // 添加媒体播放器 private var audioPlayer: MiniMediaPlayer? = null @@ -124,7 +104,9 @@ class VoiceInteractionService : Service() { audioManager = getSystemService(Context.AUDIO_SERVICE) as AudioManager FileLogger.d(TAG, "AudioManager初始化完成") - initServices() + // 初始化语音交互处理器 + initVoiceInteractionHandler() + initMediaSession() registerMediaButtonReceiver() @@ -134,7 +116,6 @@ class VoiceInteractionService : Service() { // 设置为媒体播放状态 setPlaybackState(PlaybackStateCompat.STATE_PAUSED) - // FileLogger.d(TAG, "设置播放状态为STATE_PAUSED") // 启动监控和前台服务 startMonitoring() @@ -148,26 +129,21 @@ class VoiceInteractionService : Service() { */ private fun resetState() { isActive = false - isRecognitionActive = false isTimeoutPaused = false - hasSpeechDetected = false - isTtsSpeaking = false } /** - * 初始化所有服务 + * 初始化语音交互处理器 */ - private fun initServices() { - // 创建新的Azure服务实例 - FileLogger.d(TAG, "创建新的Azure服务实例") - azureAsrHelper = AzureAsrHelper(this) - azureTtsHelper = AzureTtsHelper(this) + private fun initVoiceInteractionHandler() { + FileLogger.d(TAG, "初始化语音交互处理器") // 尝试从静态变量获取配置 var subscriptionKey = MainActivity.azureSpeechKey var serviceRegion = MainActivity.azureSpeechRegion var openaiKey = MainActivity.openaiApiKey var openaiBaseUrl = MainActivity.openaiBaseUrl ?: "" // OpenAI API基本URL + var openaiModel = MainActivity.openaiModel ?: "" // OpenAI模型 // 如果静态变量中没有配置，尝试从加密存储中加载 if (subscriptionKey.isEmpty() || serviceRegion.isEmpty() || openaiKey.isEmpty()) { @@ -182,6 +158,7 @@ class VoiceInteractionService : Service() { serviceRegion = MainActivity.azureSpeechRegion openaiKey = MainActivity.openaiApiKey openaiBaseUrl = MainActivity.openaiBaseUrl ?: "" + openaiModel = MainActivity.openaiModel ?: "" FileLogger.d(TAG, "已从加密存储加载配置信息") } else { @@ -189,42 +166,31 @@ class VoiceInteractionService : Service() { } } - // 初始化语音服务 - if (subscriptionKey.isNotEmpty() && serviceRegion.isNotEmpty()) { - // 初始化ASR - azureAsrHelper.initialize(subscriptionKey, serviceRegion, arrayOf("zh-CN")) - - // 初始化TTS - azureTtsHelper.initialize(subscriptionKey, serviceRegion, "zh-CN") - - FileLogger.d(TAG, "Azure语音服务已初始化") - } else { - FileLogger.e(TAG, "Azure配置信息不完整，无法初始化Azure服务") - } - - // 初始化OpenAI服务 - openAIService = OpenAIService() + // 初始化语音交互处理器 + voiceInteractionHandler = VoiceInteractionHandler(applicationContext, subscriptionKey, serviceRegion, openaiKey, openaiBaseUrl, openaiModel) - // 初始化OpenAI服务 - if (openaiKey.isNotEmpty()) { - val initialized = if (openaiBaseUrl.isNotEmpty()) { - openAIService.initialize(openaiKey, openaiBaseUrl) - } else { - openAIService.initialize(openaiKey) + // 初始化回调 + voiceInteractionHandler.setCallback(object : VoiceInteractionHandler.InteractionCallback { + override fun onStateChanged(isRecognitionActive: Boolean, isTtsSpeaking: Boolean, hasSpeechDetected: Boolean) { + // 更新活动时间 + updateLastActivityTime() } - if (initialized) { - FileLogger.d(TAG, "OpenAI服务已初始化") - - // 初始化函数处理器 - functionHandler = VoiceFunctionHandler(openAIService, systemPrompt) - functionHandler.initialize() - - } else { - FileLogger.e(TAG, "OpenAI服务初始化失败") + override fun onError(message: String) { + playNotification(message) } + + override fun onPromptRequest(message: String) { + playPrompt(message) + } + }) + + // 初始化处理器 + val initialized = voiceInteractionHandler.initialize() + if (initialized) { + FileLogger.d(TAG, "语音交互处理器初始化成功") } else { - FileLogger.e(TAG, "OpenAI配置信息不完整，无法初始化OpenAI服务") + FileLogger.e(TAG, "语音交互处理器初始化失败") } } @@ -329,22 +295,21 @@ class VoiceInteractionService : Service() { } // 检查语音识别状态 - if (isRecognitionActive) { + if (voiceInteractionHandler.isRecognitionActive) { val currentTime = System.currentTimeMillis() val elapsedTime = currentTime - lastActivityTime - // 如果超过5秒没有检测到语音，且不在TTS播放中，暂停语音识别 - if (!hasSpeechDetected && !isTtsSpeaking && elapsedTime >= RECOGNITION_TIMEOUT) { + // 如果超过指定时间没有检测到语音，且不在TTS播放中，暂停语音识别 + if (!voiceInteractionHandler.hasSpeechDetected && + !voiceInteractionHandler.isTtsSpeaking && + elapsedTime >= RECOGNITION_TIMEOUT) { + FileLogger.d(TAG, "超过${RECOGNITION_TIMEOUT/1000}秒未检测到语音，停止识别") isTimeoutPaused = true playNotification("没有听到您说话，已暂停对话。双击耳机按钮可重新开始。") - // 停止语音识别并确保资源完全释放 - stopVoiceRecognition() - - // 清理识别状态 - isRecognitionActive = false - hasSpeechDetected = false + // 停止语音识别 + voiceInteractionHandler.stopRecognition() } } } @@ -365,14 +330,12 @@ class VoiceInteractionService : Service() { lastKeyEventTime = currentTime // 停止当前TTS播放 - stopCurrentTTS() + voiceInteractionHandler.stopTts() setPlaybackState(PlaybackStateCompat.STATE_PLAYING) // 播放提示音 playPrompt("我在!") - // audioPlayer?.play(R.raw.listening) - // 设置为播放状态 setPlaybackState(PlaybackStateCompat.STATE_PAUSED) @@ -380,308 +343,48 @@ class VoiceInteractionService : Service() { // 重置超时暂停标志 isTimeoutPaused = false - // 启动或重置语音识别 - if (!isRecognitionActive) { + // 启动语音识别 + if (!voiceInteractionHandler.isRecognitionActive) { FileLogger.d(TAG, "语音识别未激活，开始启动") - // 如果之前是因为超时暂停，重新初始化语音识别组件 - // if (isTimeoutPaused) { - // Log.d(TAG, "之前因超时暂停，重新初始化Azure服务") - // restartAsr() - // } + // 通知 Flutter 语音识别已启动 + notifyRecognitionStarted() - startVoiceRecognition() + // 启动语音识别 + voiceInteractionHandler.startRecognition() } else { FileLogger.d(TAG, "语音识别已激活，更新活动时间") updateLastActivityTime() - hasSpeechDetected = false } } - /** - * 开始语音识别 - */ - private fun startVoiceRecognition() { - if (isRecognitionActive) return - - // 检查录音权限 - if (!checkRecordAudioPermission()) { - playNotification("需要录音权限，请在设置中授予权限") - return - } - // 通知 Flutter 语音识别已启动 - notifyRecognitionStarted() - - isActive = true - isRecognitionActive = true - hasSpeechDetected = false - updateLastActivityTime() - - try { - azureAsrHelper.startContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { - override fun onRecognizing(recognizing: String, detectedLanguage: String) { - if (recognizing.isNotEmpty()) { - hasSpeechDetected = true - stopCurrentTTS() - updateLastActivityTime() - } - } - - override fun onResult(result: String, detectedLanguage: String) { - if (result.isNotEmpty()) { - processWithOpenAI(result) - } - - // 重置状态，继续识别 - hasSpeechDetected = false - updateLastActivityTime() - } - - override fun onSessionStarted() { - updateLastActivityTime() - - } - - override fun onSessionStopped() { - isRecognitionActive = false - - - } - - override fun onCanceled(reason: String, errorDetails: String) { - isRecognitionActive = false - - - } - - override fun onError(error: String) { - isRecognitionActive = false - playNotification("语音识别出错") - - } - }) - } catch (e: Exception) { - isRecognitionActive = false - FileLogger.e(TAG, "启动语音识别失败: ${e.message}", e) - playNotification("启动语音识别失败") - - - } - } - - /** - * 停止语音识别 - */ - private fun stopVoiceRecognition() { - if (!isRecognitionActive) return - - FileLogger.d(TAG, "停止语音识别") - - try { - azureAsrHelper.stopContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { - override fun onResult(result: String, detectedLanguage: String) {} - override fun onRecognizing(recognizing: String, detectedLanguage: String) {} - override fun onSessionStarted() {} - override fun onSessionStopped() { - isRecognitionActive = false - FileLogger.d(TAG, "语音识别会话已停止") - } - override fun onCanceled(reason: String, errorDetails: String) { - isRecognitionActive = false - FileLogger.d(TAG, "语音识别已取消: $reason") - } - override fun onError(error: String) { - isRecognitionActive = false - FileLogger.e(TAG, "停止语音识别时出错: $error") - } - }) - } catch (e: Exception) { - FileLogger.e(TAG, "停止语音识别异常: ${e.message}", e) - // 确保状态一致性 - isRecognitionActive = false - } - - setPlaybackState(PlaybackStateCompat.STATE_PAUSED) - isRecognitionActive = false - hasSpeechDetected = false - // 不重置 isTimeoutPaused，保留暂停原因 - } - - /** - * 使用OpenAI处理语音识别结果 - */ - private fun processWithOpenAI(text: String) { - // 保存当前用户输入，用于后续同步聊天记录 - currentUserInput = text - - Thread { - try { - val messages = JSONArray().apply { - put(openAIService.createUserMessage(text)) - } - - // 创建响应构建器 - val responseBuilder = StringBuilder() - - openAIService.sendMessageStream( - messages = messages, - systemPrompt = systemPrompt, - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - // 累加响应内容 - responseBuilder.append(token) - } - - override fun onComplete() { - // 处理完整响应 - val response = responseBuilder.toString() - if (response.isNotEmpty()) { - // 播放AI回复 - Log.d(TAG, "AI 回复: $response") - - speakAIResponse(response) - - // 同步聊天记录到Flutter端 - notifyChatHistoryUpdated("personal_assistant", text, response) - } - } - - override fun onError(e: Exception) { - FileLogger.e(TAG, "AI流式处理出错: ${e.message}", e) - playNotification("AI处理出错") - } - - override fun onFunctionCall(call: JSONObject) { - FileLogger.d(TAG, "收到函数调用请求: ${call.getString("name")}") - - // 使用函数处理器处理函数调用 - val handled = functionHandler.handleFunctionCall( - functionCall = call, - messages = messages, - callback = object : VoiceFunctionHandler.FunctionCallCallback { - override fun onTokenReceived(token: String) { - responseBuilder.append(token) - } - - override fun onComplete() { - val response = responseBuilder.toString() - if (response.isNotEmpty()) { - // 播放AI回复 - Log.d(TAG, "AI Function Call 回复: $response") - speakAIResponse(response) - - // 同步聊天记录到Flutter端 - notifyChatHistoryUpdated("personal_assistant", text, response) - } - updateLastActivityTime() - } - - override fun onError(message: String) { - FileLogger.e(TAG, "函数处理出错: $message") - playNotification(message) - } - - override fun onFunctionCall(nestedCall: JSONObject) { - FileLogger.d(TAG, "收到嵌套函数调用: ${nestedCall.getString("name")}") - // 递归处理嵌套函数调用 - functionHandler.handleFunctionCall( - functionCall = nestedCall, - messages = messages, - callback = this - ) - } - - override fun onExitWithMessage(farewell: String) { - // 播放退出消息 - speakAIResponse(farewell) - - // 同步聊天记录 - notifyChatHistoryUpdated("personal_assistant", text, farewell) - - // 停止语音识别 - stopVoiceRecognition() - } - } - ) - - if (!handled) { - // 如果函数没有被处理，作为普通文本处理 - FileLogger.d(TAG, "函数未处理，作为普通文本处理") - speakAIResponse("我无法处理这个请求") - notifyChatHistoryUpdated("personal_assistant", text, "我无法处理这个请求") - } - } - } - ) - - } catch (e: Exception) { - FileLogger.e(TAG, "AI处理出错: ${e.message}", e) - playNotification("AI处理出错") - } - }.start() - } - - /** - * 播放AI回复 - */ - private fun speakAIResponse(text: String) { - isTtsSpeaking = true - azureTtsHelper.speakText(text, object : AzureTtsHelper.TTSCallback { - override fun onSuccess(message: String) { - isTtsSpeaking = false - updateLastActivityTime() - } - - override fun onError(error: String) { - isTtsSpeaking = false - } - }) - } - /** * 播放提示音 */ private fun playPrompt(message: String) { - isTtsSpeaking = true - azureTtsHelper.speakText(message, object : AzureTtsHelper.TTSCallback { - override fun onSuccess(message: String) { isTtsSpeaking = false } - override fun onError(error: String) { isTtsSpeaking = false } - }) + voiceInteractionHandler.playTts(message) } /** * 播放通知提示音 */ private fun playNotification(message: String) { - isTtsSpeaking = true // 更新播放状态为播放中，增加接收蓝牙按键事件的几率 setPlaybackState(PlaybackStateCompat.STATE_PLAYING) // 确保媒体会话处于活跃状态 mediaSession.isActive = true - azureTtsHelper.speakText(message, object : AzureTtsHelper.TTSCallback { - override fun onSuccess(message: String) { - isTtsSpeaking = false + voiceInteractionHandler.playTts(message, object : VoiceInteractionHandler.TtsCallback { + override fun onComplete() { setPlaybackState(PlaybackStateCompat.STATE_PAUSED) } override fun onError(error: String) { - isTtsSpeaking = false setPlaybackState(PlaybackStateCompat.STATE_PAUSED) } }) } - /** - * 停止当前TTS播放 - */ - private fun stopCurrentTTS() { - if (isTtsSpeaking) { - azureTtsHelper.stopSpeaking() - isTtsSpeaking = false - } - } - /** * 更新最后活动时间 */ @@ -715,15 +418,6 @@ class VoiceInteractionService : Service() { } } - /** - * 检查录音权限 - */ - private fun checkRecordAudioPermission(): Boolean { - val permission = android.Manifest.permission.RECORD_AUDIO - val result = applicationContext.checkCallingOrSelfPermission(permission) - return result == android.content.pm.PackageManager.PERMISSION_GRANTED - } - /** * 启动前台服务 */ @@ -865,30 +559,17 @@ class VoiceInteractionService : Service() { stopMonitoring() // 停止语音识别 - stopVoiceRecognition() + voiceInteractionHandler.stopRecognition() // 停止媒体会话 mediaSession.release() FileLogger.d(TAG, "媒体会话已释放") // 停止TTS - stopCurrentTTS() + voiceInteractionHandler.stopTts() - // 关闭Azure语音服务 - if (::azureAsrHelper.isInitialized) { - FileLogger.d(TAG, "关闭Azure语音服务") - azureAsrHelper.dispose() - } - - if (::azureTtsHelper.isInitialized) { - FileLogger.d(TAG, "关闭Azure TTS服务") - azureTtsHelper.dispose() - } - - // 关闭OpenAI服务 - if (::openAIService.isInitialized) { - FileLogger.d(TAG, "关闭OpenAI服务") - } + // 释放语音交互处理器资源 + voiceInteractionHandler.dispose() // 关闭音频播放器 audioPlayer?.release() @@ -907,11 +588,11 @@ class VoiceInteractionService : Service() { FileLogger.d(TAG, "暂停后台语音交互（来自Flutter的请求）") // 停止当前TTS播放 - stopCurrentTTS() + voiceInteractionHandler.stopTts() // 停止语音识别 - if (isRecognitionActive) { - stopVoiceRecognition() + if (voiceInteractionHandler.isRecognitionActive) { + voiceInteractionHandler.stopRecognition() } // 设置为暂停状态，但保持服务活跃 @@ -923,8 +604,6 @@ class VoiceInteractionService : Service() { * 通知 Flutter 端聊天记录已更新 */ private fun notifyChatHistoryUpdated(agentId: String, userMessage: String, assistantMessage: String) { - - // 创建广播 Intent val intent = Intent(ACTION_CHAT_HISTORY_UPDATED).apply { putExtra("agentId", agentId) @@ -1025,5 +704,4 @@ class VoiceInteractionService : Service() { */ fun isPlaying() = mediaPlayer?.isPlaying == true } - } \ No newline at end of file diff --git a/android/build.gradle.kts b/android/build.gradle.kts index 35365df1a..f8f47c600 100644 --- a/android/build.gradle.kts +++ b/android/build.gradle.kts @@ -2,6 +2,7 @@ allprojects { repositories { google() mavenCentral() + maven { url = uri("https://storage.googleapis.com/download.flutter.io") } } } diff --git a/android/settings.gradle.kts b/android/settings.gradle.kts index 6a3483c9c..6ca3ffd16 100644 --- a/android/settings.gradle.kts +++ b/android/settings.gradle.kts @@ -30,7 +30,9 @@ plugins { include(":app") include(":azure_speech") +include(":open_ai_service") // 设置azure_speech项目的路径 project(":azure_speech").projectDir = file("../local_plugins/azure_speech/android") +project(":open_ai_service").projectDir = file("../local_plugins/open_ai_service/android") diff --git a/lib/data/services/voice_interaction_service.dart b/lib/data/services/voice_interaction_service.dart index 1bb2406f5..792e94912 100644 --- a/lib/data/services/voice_interaction_service.dart +++ b/lib/data/services/voice_interaction_service.dart @@ -40,7 +40,7 @@ class VoiceInteractionService extends GetxService { late String _azureSpeechRegion; late String _openaiApiKey; late String _openaiBaseUrl; - + late String _openaiModel; // 聊天历史服务 late final ChatHistoryService _chatHistoryService; @@ -60,11 +60,12 @@ class VoiceInteractionService extends GetxService { _azureSpeechRegion = dotenv.env['AZURE_SPEECH_REGION'] ?? ''; _openaiApiKey = dotenv.env['OPENAI_API_KEY'] ?? ''; _openaiBaseUrl = dotenv.env['OPENAI_BASE_URL'] ?? ''; + _openaiModel = dotenv.env['OPENAI_MODEL'] ?? ''; if (_azureSpeechKey.isEmpty || _azureSpeechRegion.isEmpty) { Logger.warning('未找到 Azure 语音服务配置。请在 .env 文件中设置 AZURE_SPEECH_KEY 和 AZURE_SPEECH_REGION'); } - + } /// 处理来自原生层的事件 @@ -196,6 +197,7 @@ class VoiceInteractionService extends GetxService { 'azure_speech_region': _azureSpeechRegion, 'openai_api_key': _openaiApiKey, 'openai_base_url': _openaiBaseUrl, + 'openai_model': _openaiModel, }) ?? false; if (result) { diff --git a/local_plugins/azure_speech/android/build.gradle.kts b/local_plugins/azure_speech/android/build.gradle.kts index 665d495d1..a26b13967 100644 --- a/local_plugins/azure_speech/android/build.gradle.kts +++ b/local_plugins/azure_speech/android/build.gradle.kts @@ -33,8 +33,8 @@ configure { } compileOptions { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 } sourceSets { @@ -53,13 +53,12 @@ configure { // 显式设置Kotlin JVM目标版本 tasks.withType { kotlinOptions { - jvmTarget = "1.8" + jvmTarget = "11" } } dependencies { - // 直接通过本地依赖方式添加Flutter - implementation(fileTree(mapOf("dir" to "libs", "include" to listOf("*.jar")))) + // 添加Microsoft语音SDK implementation("com.microsoft.cognitiveservices.speech:client-sdk:1.30.0") } \ No newline at end of file diff --git a/local_plugins/open_ai_service/README.md b/local_plugins/open_ai_service/README.md new file mode 100644 index 000000000..d7ccb709d --- /dev/null +++ b/local_plugins/open_ai_service/README.md @@ -0,0 +1,348 @@ +# OpenAI服务插件 + +提供与OpenAI API的原生交互功能，支持流式输出和函数调用，同时支持Android和iOS平台。 + +## 功能特点 + +- 原生实现，性能更好 +- 支持流式输出，实时显示AI回复 +- 支持函数调用（工具调用） +- 支持自定义API基础URL +- 支持Android和iOS平台 +- 提供简洁易用的Dart接口 + +## 安装 + +将该插件添加到您的`pubspec.yaml`文件中： + +```yaml +dependencies: + open_ai_service: + path: local_plugins/open_ai_service +``` + +## 使用方法 + +### 初始化 + +首先需要初始化OpenAI服务，提供API密钥和可选的基础URL和模型： + +```dart +import 'package:open_ai_service/open_ai_service.dart'; + +final openAIService = OpenAIService(); + +// 使用OpenAI官方API和默认模型 +await openAIService.initialize(apiKey: 'your_openai_api_key'); + +// 或使用自定义API端点和模型 +await openAIService.initialize( + apiKey: 'your_api_key', + baseUrl: 'https://your-custom-endpoint.com/v1/chat/completions', + model: 'gpt-4-0125-preview', // 自定义模型 +); +``` + +### 普通聊天 + +发送消息并获取非流式回复： + +```dart +// 创建用户消息 +final userMessage = await openAIService.createUserMessage('你好，请介绍一下自己'); + +// 发送消息 +final response = await openAIService.sendMessage( + messages: [userMessage], + systemPrompt: '你是一个有帮助的AI助手，请简洁地回答问题。', +); + +print('AI回复: $response'); +``` + +### 流式输出 + +使用流式输出获取实时回复： + +```dart +// 注册事件监听 +final subscription = openAIService.processEvents( + onToken: (token) { + // 收到部分回复 + print('收到部分回复: $token'); + // 更新UI... + }, + onComplete: () { + // 回复完成 + print('回复完成'); + }, + onError: (error) { + // 处理错误 + print('错误: $error'); + }, + onFunctionCall: (functionCall) { + // 处理函数调用 + print('函数调用: $functionCall'); + handleFunctionCall(functionCall); + }, +); + +// 发送流式消息 +await openAIService.sendMessageStream( + messages: [userMessage], + systemPrompt: '你是一个有帮助的AI助手，请简洁地回答问题。', +); + +// 完成后，记得取消订阅 +// subscription.cancel(); +``` + +### 函数调用 + +注册函数并处理函数调用： + +```dart +// 注册一个天气查询函数 +await openAIService.registerFunction( + name: 'get_weather', + description: '获取指定城市的天气信息', + parameters: { + 'type': 'object', + 'properties': { + 'city': { + 'type': 'string', + 'description': '城市名称，如北京、上海等', + }, + 'date': { + 'type': 'string', + 'description': '查询日期，默认为今天', + }, + }, + 'required': ['city'], + }, +); + +// 处理函数调用 +void handleFunctionCall(Map functionCall) { + final name = functionCall['name']; + final arguments = jsonDecode(functionCall['arguments']); + final id = functionCall['id']; + + if (name == 'get_weather') { + final city = arguments['city']; + final date = arguments['date'] ?? '今天'; + + // 获取天气信息（实际应用中应调用天气API） + final weatherResult = '{"temperature": 25, "condition": "晴天", "humidity": 40}'; + + // 发送函数调用结果 + openAIService.sendFunctionCallResult( + messages: messages, // 当前的消息历史 + systemPrompt: systemPrompt, + functionCall: functionCall, + functionResult: weatherResult, + ); + } +} +``` + +## 示例应用 + +一个简单的聊天机器人示例： + +```dart +import 'package:flutter/material.dart'; +import 'package:open_ai_service/open_ai_service.dart'; + +void main() { + runApp(MyApp()); +} + +class MyApp extends StatelessWidget { + @override + Widget build(BuildContext context) { + return MaterialApp( + title: 'OpenAI 聊天', + theme: ThemeData(primarySwatch: Colors.blue), + home: ChatScreen(), + ); + } +} + +class ChatScreen extends StatefulWidget { + @override + _ChatScreenState createState() => _ChatScreenState(); +} + +class _ChatScreenState extends State { + final OpenAIService _openAIService = OpenAIService(); + final TextEditingController _textController = TextEditingController(); + final List> _messages = []; + StreamSubscription? _subscription; + String _currentResponse = ''; + bool _isLoading = false; + + @override + void initState() { + super.initState(); + _initializeOpenAI(); + } + + Future _initializeOpenAI() async { + try { + await _openAIService.initialize( + apiKey: 'your_api_key_here', + ); + + // 注册事件监听 + _subscription = _openAIService.processEvents( + onToken: (token) { + setState(() { + _currentResponse += token; + }); + }, + onComplete: () { + _addAIMessage(_currentResponse); + setState(() { + _currentResponse = ''; + _isLoading = false; + }); + }, + onError: (error) { + ScaffoldMessenger.of(context).showSnackBar( + SnackBar(content: Text('错误: $error')), + ); + setState(() { + _isLoading = false; + }); + }, + onFunctionCall: (functionCall) { + // 处理函数调用... + }, + ); + } catch (e) { + ScaffoldMessenger.of(context).showSnackBar( + SnackBar(content: Text('初始化失败: $e')), + ); + } + } + + void _sendMessage() async { + if (_textController.text.isEmpty) return; + + final userMessage = _textController.text; + _addUserMessage(userMessage); + _textController.clear(); + + try { + setState(() { + _isLoading = true; + }); + + final message = await _openAIService.createUserMessage(userMessage); + + await _openAIService.sendMessageStream( + messages: _messages, + systemPrompt: '你是一个有帮助的AI助手，请简洁地回答问题。', + ); + } catch (e) { + ScaffoldMessenger.of(context).showSnackBar( + SnackBar(content: Text('发送消息失败: $e')), + ); + setState(() { + _isLoading = false; + }); + } + } + + void _addUserMessage(String content) { + setState(() { + _messages.add({ + 'role': 'user', + 'content': content, + }); + }); + } + + void _addAIMessage(String content) { + setState(() { + _messages.add({ + 'role': 'assistant', + 'content': content, + }); + }); + } + + @override + void dispose() { + _subscription?.cancel(); + _textController.dispose(); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + return Scaffold( + appBar: AppBar(title: Text('OpenAI 聊天')), + body: Column( + children: [ + Expanded( + child: ListView.builder( + itemCount: _messages.length + (_currentResponse.isNotEmpty ? 1 : 0), + itemBuilder: (context, index) { + if (index < _messages.length) { + final message = _messages[index]; + final isUser = message['role'] == 'user'; + + return ListTile( + leading: Icon(isUser ? Icons.person : Icons.computer), + title: Text(message['content']), + ); + } else { + // 显示当前正在生成的回复 + return ListTile( + leading: Icon(Icons.computer), + title: Text(_currentResponse), + ); + } + }, + ), + ), + if (_isLoading) + LinearProgressIndicator(), + Padding( + padding: const EdgeInsets.all(8.0), + child: Row( + children: [ + Expanded( + child: TextField( + controller: _textController, + decoration: InputDecoration( + hintText: '输入消息...', + border: OutlineInputBorder(), + ), + ), + ), + IconButton( + icon: Icon(Icons.send), + onPressed: _sendMessage, + ), + ], + ), + ), + ], + ), + ); + } +} +``` + +## 注意事项 + +- 请确保您的API密钥安全存储，不要直接硬编码在应用程序中 +- 流式输出需要在调用`sendMessageStream`之前注册事件监听 +- 使用完毕后记得取消事件订阅 + +## 许可证 + +此插件遵循MIT许可证。 \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/build.gradle.kts b/local_plugins/open_ai_service/android/build.gradle.kts new file mode 100644 index 000000000..4f7b2f643 --- /dev/null +++ b/local_plugins/open_ai_service/android/build.gradle.kts @@ -0,0 +1,37 @@ +plugins { + // Android Library 插件 + id("com.android.library") + // Kotlin Android 插件 + id("org.jetbrains.kotlin.android") +} + +android { + // 命名空间，对应你插件的包名（需与代码内包名保持一致） + namespace = "com.yunqiinnovation.open_ai_service" + + // 目标 SDK 版本 + compileSdk = 33 + + defaultConfig { + // 最低 SDK 版本 + minSdk = 21 + targetSdk = 33 + } + + // Java 语言级别兼容配置 + compileOptions { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 + } + + // Kotlin 语言级别 + kotlinOptions { + jvmTarget = "11" + } +} + +dependencies { + + implementation("com.squareup.okhttp3:okhttp:4.10.0") + +} \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/settings.gradle.kts b/local_plugins/open_ai_service/android/settings.gradle.kts new file mode 100644 index 000000000..1c33f4971 --- /dev/null +++ b/local_plugins/open_ai_service/android/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "open_ai_service" \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/src/main/AndroidManifest.xml b/local_plugins/open_ai_service/android/src/main/AndroidManifest.xml new file mode 100644 index 000000000..e38cdeb21 --- /dev/null +++ b/local_plugins/open_ai_service/android/src/main/AndroidManifest.xml @@ -0,0 +1,11 @@ + + + + + + + + + + diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt new file mode 100644 index 000000000..7b9213046 --- /dev/null +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt @@ -0,0 +1,456 @@ +package com.yunqiinnovation.open_ai_service + +import android.util.Log +import okhttp3.* +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONArray +import org.json.JSONObject +import java.io.IOException +import java.util.concurrent.TimeUnit + +/** + * OpenAI服务的原生实现 + */ +class OpenAIService() { + private val TAG = "OpenAIService" + private var baseUrl = "" + private val client = OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .writeTimeout(30, TimeUnit.SECONDS) + .build() + + private var apiKey: String = "" + private var isInitialized = false + private var model: String = "" // 默认模型 + + // 用于存储注册的函数 + private val registeredFunctions = mutableListOf() + + /** + * 创建用户消息 + */ + fun createUserMessage(content: String): JSONObject { + return JSONObject().apply { + put("role", "user") + put("content", content) + } + } + + /** + * 创建助手消息 + */ + fun createAssistantMessage(content: String): JSONObject { + return JSONObject().apply { + put("role", "assistant") + put("content", content) + } + } + + /** + * 初始化OpenAI服务 + */ + fun initialize(apiKey: String, baseUrl: String, model: String): Boolean { + this.apiKey = apiKey + if (baseUrl.isNotEmpty()) { + this.baseUrl = baseUrl + } + if (model.isNotEmpty()) { + this.model = model + } + isInitialized = apiKey.isNotEmpty() + return isInitialized + } + + /** + * 注册函数 + */ + fun registerFunction(name: String, description: String, parameters: JSONObject): Boolean { + try { + val function = JSONObject().apply { + put("name", name) + put("description", description) + put("parameters", parameters) + } + + // 检查是否已存在相同名称的函数 + val existingIndex = registeredFunctions.indexOfFirst { + it.getString("name") == name + } + + if (existingIndex >= 0) { + // 如果已存在，则替换 + registeredFunctions[existingIndex] = function + } else { + // 如果不存在，则添加 + registeredFunctions.add(function) + } + + return true + } catch (e: Exception) { + return false + } + } + + /** + * 发送消息（非流式输出） + */ + @Throws(OpenAIException::class) + fun sendMessage(messages: JSONArray, systemPrompt: String): String { + if (!isInitialized || apiKey.isEmpty()) { + throw OpenAIException("OpenAI服务未初始化") + } + + val fullMessages = JSONArray().apply { + put(JSONObject().apply { + put("role", "system") + put("content", systemPrompt) + }) + for (i in 0 until messages.length()) { + put(messages.getJSONObject(i)) + } + } + + val requestBody = JSONObject().apply { + put("model", model) + put("messages", fullMessages) + put("temperature", 0.7) + put("max_tokens", 2000) + put("stream", false) + + // 如果有注册的函数，则添加到请求中 + if (registeredFunctions.isNotEmpty()) { + val tools = JSONArray() + for (function in registeredFunctions) { + val tool = JSONObject().apply { + put("type", "function") + put("function", function) + } + tools.put(tool) + } + put("tools", tools) + } + } + + val mediaType = "application/json".toMediaTypeOrNull() + val request = Request.Builder() + .url(baseUrl) + .addHeader("Content-Type", "application/json") + .addHeader("Authorization", "Bearer $apiKey") + .post(requestBody.toString().toRequestBody(mediaType)) + .build() + + try { + Log.d(TAG, "request: $request") + client.newCall(request).execute().use { response -> + if (!response.isSuccessful) { + throw OpenAIException("API调用失败: ${response.code}") + } + + val responseBody = response.body?.string() ?: throw OpenAIException("Empty response") + val jsonResponse = JSONObject(responseBody) + + // 检查是否有函数调用 + if (jsonResponse.has("choices") && + jsonResponse.getJSONArray("choices").length() > 0) { + + val choice = jsonResponse.getJSONArray("choices").getJSONObject(0) + + // 检查是否是函数调用 + if (choice.has("message")) { + val message = choice.getJSONObject("message") + + // 检查是否有工具调用 + if (message.has("tool_calls")) { + val toolCalls = message.getJSONArray("tool_calls") + if (toolCalls.length() > 0) { + val toolCall = toolCalls.getJSONObject(0) + if (toolCall.has("function")) { + val function = toolCall.getJSONObject("function") + val functionCall = JSONObject().apply { + put("name", function.getString("name")) + put("arguments", function.getString("arguments")) + put("id", toolCall.getString("id")) + } + return functionCall.toString() + } + } + } + + // 如果没有工具调用，返回消息内容 + if (message.has("content")) { + return message.getString("content") + } + } + } + + throw OpenAIException("Invalid response format") + } + } catch (e: Exception) { + if (e is OpenAIException) throw e + throw OpenAIException("Failed to communicate with AI service: ${e.message}") + } + } + + /** + * 发送消息（流式输出） + */ + fun sendMessageStream(messages: JSONArray, systemPrompt: String, callback: StreamCallback) { + if (!isInitialized || apiKey.isEmpty()) { + callback.onError(OpenAIException("OpenAI服务未初始化")) + return + } + + val fullMessages = JSONArray().apply { + put(JSONObject().apply { + put("role", "system") + put("content", systemPrompt) + }) + for (i in 0 until messages.length()) { + put(messages.getJSONObject(i)) + } + } + + val requestBody = JSONObject().apply { + put("model", model) + put("messages", fullMessages) + put("temperature", 0.7) + put("max_tokens", 2000) + put("stream", true) + + // 如果有注册的函数，则添加到请求中 + if (registeredFunctions.isNotEmpty()) { + val tools = JSONArray() + for (function in registeredFunctions) { + val tool = JSONObject().apply { + put("type", "function") + put("function", function) + } + tools.put(tool) + } + put("tools", tools) + } + } + + val mediaType = "application/json".toMediaTypeOrNull() + val request = Request.Builder() + .url(baseUrl) + .addHeader("Content-Type", "application/json") + .addHeader("Authorization", "Bearer $apiKey") + .addHeader("Accept", "text/event-stream") + .post(requestBody.toString().toRequestBody(mediaType)) + .build() + + Log.d(TAG, "request: $request") + client.newCall(request).enqueue(object : Callback { + override fun onFailure(call: Call, e: IOException) { + callback.onError(OpenAIException(e.message ?: "请求失败")) + } + + override fun onResponse(call: Call, response: Response) { + + if (!response.isSuccessful) { + callback.onError(OpenAIException("API调用失败: ${response.code}")) + return + } + + val responseBody = response.body ?: return + val source = responseBody.source() + + try { + // 预取数据到缓冲区 + source.request(Long.MAX_VALUE) + val bufferedSource = source.buffer + + // 用于存储函数调用的各个部分 + val finalToolCalls = mutableMapOf() + + while (!bufferedSource.exhausted()) { + val line = bufferedSource.readUtf8Line() ?: continue + val trimmedLine = line.trim() + if (trimmedLine.isEmpty()) continue + if (trimmedLine.startsWith("data:")) { + val data = trimmedLine.substring(5) + + // 处理[DONE]消息 + if (data == "[DONE]") { + processToolCalls(finalToolCalls, callback) + callback.onComplete() + break + } + + try { + val jsonData = JSONObject(data) + + // 处理消息内容 + if (jsonData.has("choices")) { + val choices = jsonData.getJSONArray("choices") + if (choices.length() > 0) { + val choice = choices.getJSONObject(0) + + if (choice.has("delta")) { + val delta = choice.getJSONObject("delta") + + // 处理普通文本内容 + if (delta.has("content")) { + val content = delta.getString("content") + callback.onToken(content) + } + + // 处理工具调用（函数调用） + if (delta.has("tool_calls")) { + val toolCalls = delta.getJSONArray("tool_calls") + for (i in 0 until toolCalls.length()) { + val toolCall = toolCalls.getJSONObject(i) + val index = toolCall.getInt("index") + + // 创建或获取现有的工具调用信息 + val toolCallInfo = finalToolCalls.getOrPut(index) { ToolCallInfo() } + + // 更新ID + if (toolCall.has("id")) { + toolCallInfo.id = toolCall.getString("id") + } + + // 更新函数信息 + if (toolCall.has("function")) { + val function = toolCall.getJSONObject("function") + + if (function.has("name")) { + toolCallInfo.name = function.getString("name") + } + + if (function.has("arguments")) { + toolCallInfo.arguments += function.getString("arguments") + } + } + } + } + } + } + } + } catch (e: Exception) { + // 忽略解析错误 + Log.e(TAG, "解析JSON出错: ${e.message}") + } + } + } + } catch (e: Exception) { + callback.onError(OpenAIException("处理响应流时出错: ${e.message}")) + } finally { + responseBody.close() + } + } + }) + } + + /** + * 发送函数调用结果 + */ + fun sendFunctionCallResult( + messages: JSONArray, + systemPrompt: String, + functionCall: JSONObject, + functionResult: String, + callback: StreamCallback + ) { + try { + val fullMessages = JSONArray().apply { + // 添加系统提示 + put(JSONObject().apply { + put("role", "system") + put("content", systemPrompt) + }) + + // 添加用户消息 + for (i in 0 until messages.length()) { + put(messages.getJSONObject(i)) + } + + // 添加函数调用消息 + put(JSONObject().apply { + put("role", "assistant") + put("content", null) + + // 添加工具调用 + val toolCalls = JSONArray().apply { + val toolCall = JSONObject().apply { + put("id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) + put("type", "function") + put("function", JSONObject().apply { + put("name", functionCall.getString("name")) + put("arguments", functionCall.getString("arguments")) + }) + } + put(toolCall) + } + put("tool_calls", toolCalls) + }) + + // 添加函数调用结果 + put(JSONObject().apply { + put("role", "tool") + put("content", functionResult) + put("tool_call_id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) + }) + } + + // 发送完整对话 + sendMessageStream(fullMessages, systemPrompt, callback) + + } catch (e: Exception) { + callback.onError(OpenAIException("发送函数调用结果失败: ${e.message}")) + } + } + + /** + * 处理工具调用（处理完整的函数调用并回调） + */ + private fun processToolCalls(toolCalls: Map, callback: StreamCallback) { + if (toolCalls.isEmpty()) return + + // 只处理第一个工具调用 + val firstToolCall = toolCalls.entries.firstOrNull()?.value ?: return + + if (firstToolCall.isValid()) { + // 创建函数调用JSON对象 + val functionCall = JSONObject().apply { + put("name", firstToolCall.name) + put("arguments", firstToolCall.arguments) + put("id", firstToolCall.id) + } + + // 回调 + callback.onFunctionCall(functionCall) + } + } + + /** + * 工具调用信息类 + */ + private class ToolCallInfo { + var id: String = "" + var name: String = "" + var arguments: String = "" + + fun isValid(): Boolean { + return id.isNotEmpty() && name.isNotEmpty() + } + } + + /** + * 流式输出回调接口 + */ + interface StreamCallback { + fun onToken(token: String) + fun onComplete() + fun onError(e: Exception) + fun onFunctionCall(functionCall: JSONObject) + } +} + +/** + * OpenAI服务异常 + */ +class OpenAIException(message: String) : Exception(message) \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt new file mode 100644 index 000000000..5c93e5be6 --- /dev/null +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt @@ -0,0 +1,322 @@ +package com.yunqiinnovation.open_ai_service + +import android.content.Context +import android.util.Log +import androidx.annotation.NonNull +import io.flutter.embedding.engine.plugins.FlutterPlugin +import io.flutter.plugin.common.MethodCall +import io.flutter.plugin.common.MethodChannel +import io.flutter.plugin.common.MethodChannel.MethodCallHandler +import io.flutter.plugin.common.MethodChannel.Result +import io.flutter.plugin.common.EventChannel +import io.flutter.plugin.common.EventChannel.EventSink +import io.flutter.plugin.common.EventChannel.StreamHandler +import org.json.JSONArray +import org.json.JSONObject +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors + +/** OpenAIServicePlugin */ +class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { + /// 方法通道名称 + private val methodChannelName = "com.yunqiinnovation.open_ai_service/methods" + + /// 事件通道名称 + private val eventChannelName = "com.yunqiinnovation.open_ai_service/events" + + /// 方法通道 + private lateinit var methodChannel: MethodChannel + + /// 事件通道 + private lateinit var eventChannel: EventChannel + + /// 应用上下文 + private lateinit var context: Context + + /// OpenAI服务实例 + private val openAIService = OpenAIService() + + /// 事件接收器（用于流式输出） + private var eventSink: EventSink? = null + + /// 执行器（用于后台线程） + private val executor = Executors.newSingleThreadExecutor() + + override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { + // 保存上下文 + context = flutterPluginBinding.applicationContext + + // 初始化方法通道 + methodChannel = MethodChannel(flutterPluginBinding.binaryMessenger, methodChannelName) + methodChannel.setMethodCallHandler(this) + + // 初始化事件通道 + eventChannel = EventChannel(flutterPluginBinding.binaryMessenger, eventChannelName) + eventChannel.setStreamHandler(this) + } + + override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) { + when (call.method) { + "initialize" -> { + val apiKey = call.argument("apiKey") ?: "" + val baseUrl = call.argument("baseUrl") ?: "" + val model = call.argument("model") ?: "" + + val initialized = openAIService.initialize(apiKey, baseUrl, model) + result.success(initialized) + } + + "registerFunction" -> { + val name = call.argument("name") ?: "" + val description = call.argument("description") ?: "" + val parameters = call.argument>("parameters") + + if (name.isEmpty() || parameters == null) { + result.error("INVALID_ARGUMENT", "函数注册参数无效", null) + return + } + + val parametersJson = JSONObject(parameters) + val registered = openAIService.registerFunction(name, description, parametersJson) + result.success(registered) + } + + "sendMessage" -> { + val messagesRaw = call.argument>>("messages") ?: emptyList() + val systemPrompt = call.argument("systemPrompt") ?: "" + + // 转换消息格式 + val messages = JSONArray() + for (message in messagesRaw) { + messages.put(JSONObject(message)) + } + + // 在后台线程执行请求 + executor.execute { + try { + val response = openAIService.sendMessage(messages, systemPrompt) + // 在主线程返回结果 + android.os.Handler(android.os.Looper.getMainLooper()).post { + result.success(response) + } + } catch (e: Exception) { + // 在主线程返回错误 + android.os.Handler(android.os.Looper.getMainLooper()).post { + result.error("OPENAI_ERROR", e.message, null) + } + } + } + } + + "sendMessageStream" -> { + val messagesRaw = call.argument>>("messages") ?: emptyList() + val systemPrompt = call.argument("systemPrompt") ?: "" + + // 检查事件接收器 + if (eventSink == null) { + result.error("NO_EVENT_SINK", "没有可用的事件流接收器", null) + return + } + + // 转换消息格式 + val messages = JSONArray() + for (message in messagesRaw) { + messages.put(JSONObject(message)) + } + + // 在后台线程执行请求 + executor.execute { + try { + openAIService.sendMessageStream( + messages = messages, + systemPrompt = systemPrompt, + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + // 发送token事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + eventSink?.success(mapOf("type" to "token", "content" to token)) + } + } + + override fun onComplete() { + // 发送完成事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + eventSink?.success(mapOf("type" to "complete")) + } + } + + override fun onError(e: Exception) { + // 发送错误事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + eventSink?.success(mapOf("type" to "error", "content" to e.message)) + } + } + + override fun onFunctionCall(functionCall: JSONObject) { + // 发送函数调用事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + val functionCallMap = functionCall.toMap() + eventSink?.success(mapOf("type" to "functionCall", "content" to functionCallMap)) + } + } + } + ) + + // 请求已开始 + android.os.Handler(android.os.Looper.getMainLooper()).post { + result.success(true) + } + } catch (e: Exception) { + // 在主线程返回错误 + android.os.Handler(android.os.Looper.getMainLooper()).post { + result.error("OPENAI_ERROR", e.message, null) + } + } + } + } + + "sendFunctionCallResult" -> { + val messagesRaw = call.argument>>("messages") ?: emptyList() + val systemPrompt = call.argument("systemPrompt") ?: "" + val functionCallRaw = call.argument>("functionCall") ?: emptyMap() + val functionResult = call.argument("functionResult") ?: "" + + // 检查事件接收器 + if (eventSink == null) { + result.error("NO_EVENT_SINK", "没有可用的事件流接收器", null) + return + } + + // 转换消息格式 + val messages = JSONArray() + for (message in messagesRaw) { + messages.put(JSONObject(message)) + } + + // 转换函数调用 + val functionCall = JSONObject(functionCallRaw) + + // 在后台线程执行请求 + executor.execute { + try { + openAIService.sendFunctionCallResult( + messages = messages, + systemPrompt = systemPrompt, + functionCall = functionCall, + functionResult = functionResult, + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + // 发送token事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + eventSink?.success(mapOf("type" to "token", "content" to token)) + } + } + + override fun onComplete() { + // 发送完成事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + eventSink?.success(mapOf("type" to "complete")) + } + } + + override fun onError(e: Exception) { + // 发送错误事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + eventSink?.success(mapOf("type" to "error", "content" to e.message)) + } + } + + override fun onFunctionCall(nestedFunctionCall: JSONObject) { + // 发送函数调用事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + val functionCallMap = nestedFunctionCall.toMap() + eventSink?.success(mapOf("type" to "functionCall", "content" to functionCallMap)) + } + } + } + ) + + // 请求已开始 + android.os.Handler(android.os.Looper.getMainLooper()).post { + result.success(true) + } + } catch (e: Exception) { + // 在主线程返回错误 + android.os.Handler(android.os.Looper.getMainLooper()).post { + result.error("OPENAI_ERROR", e.message, null) + } + } + } + } + + "createUserMessage" -> { + val content = call.argument("content") ?: "" + val message = openAIService.createUserMessage(content) + result.success(message.toMap()) + } + + "createAssistantMessage" -> { + val content = call.argument("content") ?: "" + val message = openAIService.createAssistantMessage(content) + result.success(message.toMap()) + } + + else -> { + result.notImplemented() + } + } + } + + override fun onDetachedFromEngine(@NonNull binding: FlutterPlugin.FlutterPluginBinding) { + methodChannel.setMethodCallHandler(null) + eventChannel.setStreamHandler(null) + executor.shutdown() + } + + // Stream事件处理 + override fun onListen(arguments: Any?, eventSink: EventSink?) { + this.eventSink = eventSink + } + + override fun onCancel(arguments: Any?) { + this.eventSink = null + } + + // 工具方法：JSONObject转Map + private fun JSONObject.toMap(): Map { + val map = mutableMapOf() + val keys = this.keys() + while (keys.hasNext()) { + val key = keys.next() + var value: Any? = this.opt(key) + + value = when (value) { + JSONObject.NULL -> null + is JSONObject -> value.toMap() + is JSONArray -> value.toList() + else -> value + } + + map[key] = value + } + return map + } + + // 工具方法：JSONArray转List + private fun JSONArray.toList(): List { + val list = mutableListOf() + for (i in 0 until this.length()) { + var value: Any? = this.opt(i) + + value = when (value) { + JSONObject.NULL -> null + is JSONObject -> value.toMap() + is JSONArray -> value.toList() + else -> value + } + + list.add(value) + } + return list + } +} \ No newline at end of file diff --git a/local_plugins/open_ai_service/ios/Classes/OpenAIService.swift b/local_plugins/open_ai_service/ios/Classes/OpenAIService.swift new file mode 100644 index 000000000..671affd73 --- /dev/null +++ b/local_plugins/open_ai_service/ios/Classes/OpenAIService.swift @@ -0,0 +1,473 @@ +import Foundation + +/// OpenAI服务异常 +public class OpenAIError: Error { + let message: String + + init(_ message: String) { + self.message = message + } +} + +/// 工具调用信息 +private class ToolCallInfo { + var id: String = "" + var name: String = "" + var arguments: String = "" + + var isValid: Bool { + return !id.isEmpty && !name.isEmpty + } +} + +/// OpenAI服务iOS原生实现 +public class OpenAIService { + private let TAG = "OpenAIService" + private var baseUrl = "https://api.openai.com/v1/chat/completions" + private var apiKey: String = "" + private var isInitialized = false + private var model: String = "doubao-1-5-lite-32k-250115" // 默认模型 + + // 用于存储注册的函数 + private var registeredFunctions: [[String: Any]] = [] + + // URL会话 + private let session: URLSession + + public init() { + // 创建URL会话配置 + let config = URLSessionConfiguration.default + config.timeoutIntervalForRequest = 30.0 + config.timeoutIntervalForResource = 30.0 + session = URLSession(configuration: config) + } + + /// 创建用户消息 + public func createUserMessage(content: String) -> [String: Any] { + return ["role": "user", "content": content] + } + + /// 创建助手消息 + public func createAssistantMessage(content: String) -> [String: Any] { + return ["role": "assistant", "content": content] + } + + /// 初始化OpenAI服务 + public func initialize(apiKey: String, baseUrl: String = "", model: String = "") -> Bool { + self.apiKey = apiKey + if !baseUrl.isEmpty { + self.baseUrl = baseUrl + } + if !model.isEmpty { + self.model = model + } + isInitialized = !apiKey.isEmpty + return isInitialized + } + + /// 注册函数 + public func registerFunction(name: String, description: String, parameters: [String: Any]) -> Bool { + do { + let function: [String: Any] = [ + "name": name, + "description": description, + "parameters": parameters + ] + + // 检查是否已存在相同名称的函数 + if let existingIndex = registeredFunctions.firstIndex(where: { ($0["name"] as? String) == name }) { + // 如果已存在，则替换 + registeredFunctions[existingIndex] = function + } else { + // 如果不存在，则添加 + registeredFunctions.append(function) + } + + return true + } catch { + return false + } + } + + /// 发送消息（非流式输出） + public func sendMessage(messages: [[String: Any]], systemPrompt: String) throws -> String { + guard isInitialized, !apiKey.isEmpty else { + throw OpenAIError("OpenAI服务未初始化") + } + + // 构建完整消息，添加系统提示 + var fullMessages: [[String: Any]] = [ + ["role": "system", "content": systemPrompt] + ] + fullMessages.append(contentsOf: messages) + + // 构建请求体 + var requestDict: [String: Any] = [ + "model": model, + "messages": fullMessages, + "temperature": 0.7, + "max_tokens": 2000, + "stream": false + ] + + // 如果有注册的函数，添加到请求中 + if !registeredFunctions.isEmpty { + var tools: [[String: Any]] = [] + for function in registeredFunctions { + let tool: [String: Any] = [ + "type": "function", + "function": function + ] + tools.append(tool) + } + requestDict["tools"] = tools + } + + // 将请求数据转换为JSON数据 + guard let jsonData = try? JSONSerialization.data(withJSONObject: requestDict) else { + throw OpenAIError("无法序列化请求数据") + } + + // 创建URL请求 + guard let url = URL(string: baseUrl) else { + throw OpenAIError("无效的URL") + } + + var request = URLRequest(url: url) + request.httpMethod = "POST" + request.addValue("application/json", forHTTPHeaderField: "Content-Type") + request.addValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization") + request.httpBody = jsonData + + // 创建信号量用于同步请求 + let semaphore = DispatchSemaphore(value: 0) + var responseResult: Result = .failure(OpenAIError("未收到响应")) + + // 执行请求 + let task = session.dataTask(with: request) { data, response, error in + if let error = error { + responseResult = .failure(OpenAIError("请求失败: \(error.localizedDescription)")) + semaphore.signal() + return + } + + guard let httpResponse = response as? HTTPURLResponse else { + responseResult = .failure(OpenAIError("无效的HTTP响应")) + semaphore.signal() + return + } + + guard httpResponse.statusCode == 200 else { + responseResult = .failure(OpenAIError("API调用失败: \(httpResponse.statusCode)")) + semaphore.signal() + return + } + + guard let data = data else { + responseResult = .failure(OpenAIError("响应数据为空")) + semaphore.signal() + return + } + + do { + // 解析JSON响应 + guard let jsonResponse = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { + responseResult = .failure(OpenAIError("无法解析JSON响应")) + semaphore.signal() + return + } + + // 检查是否有函数调用 + if let choices = jsonResponse["choices"] as? [[String: Any]], !choices.isEmpty, + let choice = choices.first, + let message = choice["message"] as? [String: Any] { + + // 检查是否有工具调用 + if let toolCalls = message["tool_calls"] as? [[String: Any]], !toolCalls.isEmpty, + let toolCall = toolCalls.first, + let function = toolCall["function"] as? [String: Any], + let name = function["name"] as? String, + let arguments = function["arguments"] as? String, + let id = toolCall["id"] as? String { + + let functionCallDict: [String: Any] = [ + "name": name, + "arguments": arguments, + "id": id + ] + + // 将函数调用转为JSON字符串 + if let functionCallData = try? JSONSerialization.data(withJSONObject: functionCallDict), + let functionCallString = String(data: functionCallData, encoding: .utf8) { + responseResult = .success(functionCallString) + semaphore.signal() + return + } + } + + // 如果没有工具调用，返回消息内容 + if let content = message["content"] as? String { + responseResult = .success(content) + semaphore.signal() + return + } + } + + responseResult = .failure(OpenAIError("无效的响应格式")) + semaphore.signal() + + } catch { + responseResult = .failure(OpenAIError("解析响应时出错: \(error.localizedDescription)")) + semaphore.signal() + } + } + + task.resume() + + // 等待响应完成 + _ = semaphore.wait(timeout: .distantFuture) + + // 返回结果或抛出错误 + switch responseResult { + case .success(let result): + return result + case .failure(let error): + throw error + } + } + + /// 发送消息（流式输出） + public func sendMessageStream(messages: [[String: Any]], systemPrompt: String, callback: @escaping StreamCallback) { + guard isInitialized, !apiKey.isEmpty else { + callback.onError(OpenAIError("OpenAI服务未初始化")) + return + } + + // 构建完整消息，添加系统提示 + var fullMessages: [[String: Any]] = [ + ["role": "system", "content": systemPrompt] + ] + fullMessages.append(contentsOf: messages) + + // 构建请求体 + var requestDict: [String: Any] = [ + "model": model, + "messages": fullMessages, + "temperature": 0.7, + "max_tokens": 2000, + "stream": true + ] + + // 如果有注册的函数，添加到请求中 + if !registeredFunctions.isEmpty { + var tools: [[String: Any]] = [] + for function in registeredFunctions { + let tool: [String: Any] = [ + "type": "function", + "function": function + ] + tools.append(tool) + } + requestDict["tools"] = tools + } + + // 将请求数据转换为JSON数据 + guard let jsonData = try? JSONSerialization.data(withJSONObject: requestDict) else { + callback.onError(OpenAIError("无法序列化请求数据")) + return + } + + // 创建URL请求 + guard let url = URL(string: baseUrl) else { + callback.onError(OpenAIError("无效的URL")) + return + } + + var request = URLRequest(url: url) + request.httpMethod = "POST" + request.addValue("application/json", forHTTPHeaderField: "Content-Type") + request.addValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization") + request.addValue("text/event-stream", forHTTPHeaderField: "Accept") + request.httpBody = jsonData + + // 用于存储函数调用的各个部分 + var finalToolCalls: [Int: ToolCallInfo] = [:] + + // 创建数据任务 + let task = session.dataTask(with: request) { data, response, error in + if let error = error { + callback.onError(OpenAIError("请求失败: \(error.localizedDescription)")) + return + } + + guard let httpResponse = response as? HTTPURLResponse else { + callback.onError(OpenAIError("无效的HTTP响应")) + return + } + + guard httpResponse.statusCode == 200 else { + callback.onError(OpenAIError("API调用失败: \(httpResponse.statusCode)")) + return + } + + guard let data = data else { + callback.onError(OpenAIError("响应数据为空")) + return + } + + // 处理SSE数据流 + if let text = String(data: data, encoding: .utf8) { + let lines = text.components(separatedBy: "\n") + + for line in lines { + if line.isEmpty { continue } + + if line.hasPrefix("data: ") { + let dataContent = line.dropFirst(6) + + // 处理[DONE]消息 + if dataContent == "[DONE]" { + self.processToolCalls(finalToolCalls, callback: callback) + callback.onComplete() + break + } + + // 解析JSON数据 + do { + if let data = dataContent.data(using: .utf8), + let jsonData = try JSONSerialization.jsonObject(with: data) as? [String: Any] { + + // 处理消息内容 + if let choices = jsonData["choices"] as? [[String: Any]], !choices.isEmpty, + let choice = choices.first { + + if let delta = choice["delta"] as? [String: Any] { + // 处理普通文本内容 + if let content = delta["content"] as? String { + callback.onToken(content) + } + + // 处理工具调用（函数调用） + if let toolCalls = delta["tool_calls"] as? [[String: Any]] { + for toolCall in toolCalls { + if let index = toolCall["index"] as? Int { + // 创建或获取现有的工具调用信息 + let toolCallInfo = finalToolCalls[index] ?? ToolCallInfo() + + // 更新ID + if let id = toolCall["id"] as? String { + toolCallInfo.id = id + } + + // 更新函数信息 + if let function = toolCall["function"] as? [String: Any] { + if let name = function["name"] as? String { + toolCallInfo.name = name + } + + if let arguments = function["arguments"] as? String { + toolCallInfo.arguments += arguments + } + } + + finalToolCalls[index] = toolCallInfo + } + } + } + } + } + } + } catch { + NSLog("解析JSON出错: \(error.localizedDescription)") + // 忽略解析错误，继续处理其他行 + } + } + } + } + } + + task.resume() + } + + /// 发送函数调用结果 + public func sendFunctionCallResult( + messages: [[String: Any]], + systemPrompt: String, + functionCall: [String: Any], + functionResult: String, + callback: @escaping StreamCallback + ) { + do { + // 构建完整消息数组 + var fullMessages: [[String: Any]] = [ + // 添加系统提示 + ["role": "system", "content": systemPrompt] + ] + + // 添加用户消息 + fullMessages.append(contentsOf: messages) + + // 获取函数相关信息 + guard let name = functionCall["name"] as? String, + let arguments = functionCall["arguments"] as? String else { + callback.onError(OpenAIError("函数调用信息不完整")) + return + } + + let id = functionCall["id"] as? String ?? "call_\(Int(Date().timeIntervalSince1970 * 1000))" + + // 添加函数调用消息 + fullMessages.append([ + "role": "assistant", + "content": NSNull(), + "tool_calls": [ + [ + "id": id, + "type": "function", + "function": [ + "name": name, + "arguments": arguments + ] + ] + ] + ]) + + // 添加函数调用结果 + fullMessages.append([ + "role": "tool", + "content": functionResult, + "tool_call_id": id + ]) + + // 发送完整对话 + sendMessageStream(messages: fullMessages, systemPrompt: systemPrompt, callback: callback) + + } catch { + callback.onError(OpenAIError("发送函数调用结果失败: \(error.localizedDescription)")) + } + } + + /// 处理工具调用（函数调用）并回调 + private func processToolCalls(_ toolCalls: [Int: ToolCallInfo], callback: StreamCallback) { + if toolCalls.isEmpty { return } + + // 只处理第一个工具调用 + guard let firstToolCall = toolCalls.values.first, firstToolCall.isValid else { return } + + // 创建函数调用字典 + let functionCall: [String: Any] = [ + "name": firstToolCall.name, + "arguments": firstToolCall.arguments, + "id": firstToolCall.id + ] + + // 回调 + callback.onFunctionCall(functionCall) + } + + /// 流式输出回调协议 + public typealias StreamCallback = (onToken: (String) -> Void, + onComplete: () -> Void, + onError: (Error) -> Void, + onFunctionCall: ([String: Any]) -> Void) +} \ No newline at end of file diff --git a/local_plugins/open_ai_service/ios/Classes/OpenAIServicePlugin.swift b/local_plugins/open_ai_service/ios/Classes/OpenAIServicePlugin.swift new file mode 100644 index 000000000..d919e2584 --- /dev/null +++ b/local_plugins/open_ai_service/ios/Classes/OpenAIServicePlugin.swift @@ -0,0 +1,215 @@ +import Flutter +import UIKit + +public class OpenAIServicePlugin: NSObject, FlutterPlugin, FlutterStreamHandler { + // OpenAI服务实例 + private let openAIService = OpenAIService() + + // 事件接收器 + private var eventSink: FlutterEventSink? + + // 注册插件 + public static func register(with registrar: FlutterPluginRegistrar) { + let methodChannel = FlutterMethodChannel(name: "com.yunqiinnovation.open_ai_service/methods", binaryMessenger: registrar.messenger()) + let eventChannel = FlutterEventChannel(name: "com.yunqiinnovation.open_ai_service/events", binaryMessenger: registrar.messenger()) + + let instance = OpenAIServicePlugin() + registrar.addMethodCallDelegate(instance, channel: methodChannel) + eventChannel.setStreamHandler(instance) + } + + // 处理方法调用 + public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { + switch call.method { + case "initialize": + if let args = call.arguments as? [String: Any], + let apiKey = args["apiKey"] as? String { + let baseUrl = args["baseUrl"] as? String ?? "" + let model = args["model"] as? String ?? "" + let initialized = openAIService.initialize(apiKey: apiKey, baseUrl: baseUrl, model: model) + result(initialized) + } else { + result(FlutterError(code: "INVALID_ARGUMENT", message: "初始化参数无效", details: nil)) + } + + case "registerFunction": + if let args = call.arguments as? [String: Any], + let name = args["name"] as? String, + let description = args["description"] as? String, + let parameters = args["parameters"] as? [String: Any] { + + let registered = openAIService.registerFunction(name: name, description: description, parameters: parameters) + result(registered) + } else { + result(FlutterError(code: "INVALID_ARGUMENT", message: "函数注册参数无效", details: nil)) + } + + case "sendMessage": + guard let args = call.arguments as? [String: Any], + let messagesRaw = args["messages"] as? [[String: Any]], + let systemPrompt = args["systemPrompt"] as? String else { + result(FlutterError(code: "INVALID_ARGUMENT", message: "发送消息参数无效", details: nil)) + return + } + + // 在后台线程执行 + DispatchQueue.global(qos: .userInitiated).async { + do { + let response = try self.openAIService.sendMessage(messages: messagesRaw, systemPrompt: systemPrompt) + // 在主线程返回结果 + DispatchQueue.main.async { + result(response) + } + } catch { + // 在主线程返回错误 + DispatchQueue.main.async { + result(FlutterError(code: "OPENAI_ERROR", message: error.localizedDescription, details: nil)) + } + } + } + + case "sendMessageStream": + guard let args = call.arguments as? [String: Any], + let messagesRaw = args["messages"] as? [[String: Any]], + let systemPrompt = args["systemPrompt"] as? String else { + result(FlutterError(code: "INVALID_ARGUMENT", message: "发送消息参数无效", details: nil)) + return + } + + // 检查事件接收器 + guard let eventSink = self.eventSink else { + result(FlutterError(code: "NO_EVENT_SINK", message: "没有可用的事件流接收器", details: nil)) + return + } + + // 在后台线程执行 + DispatchQueue.global(qos: .userInitiated).async { + let callback: OpenAIService.StreamCallback = ( + onToken: { token in + // 发送token事件 + DispatchQueue.main.async { + eventSink(["type": "token", "content": token]) + } + }, + onComplete: { + // 发送完成事件 + DispatchQueue.main.async { + eventSink(["type": "complete"]) + } + }, + onError: { error in + // 发送错误事件 + DispatchQueue.main.async { + eventSink(["type": "error", "content": error.localizedDescription]) + } + }, + onFunctionCall: { functionCall in + // 发送函数调用事件 + DispatchQueue.main.async { + eventSink(["type": "functionCall", "content": functionCall]) + } + } + ) + + self.openAIService.sendMessageStream(messages: messagesRaw, systemPrompt: systemPrompt, callback: callback) + + // 请求已开始 + DispatchQueue.main.async { + result(true) + } + } + + case "sendFunctionCallResult": + guard let args = call.arguments as? [String: Any], + let messagesRaw = args["messages"] as? [[String: Any]], + let systemPrompt = args["systemPrompt"] as? String, + let functionCallRaw = args["functionCall"] as? [String: Any], + let functionResult = args["functionResult"] as? String else { + result(FlutterError(code: "INVALID_ARGUMENT", message: "发送函数调用结果参数无效", details: nil)) + return + } + + // 检查事件接收器 + guard let eventSink = self.eventSink else { + result(FlutterError(code: "NO_EVENT_SINK", message: "没有可用的事件流接收器", details: nil)) + return + } + + // 在后台线程执行 + DispatchQueue.global(qos: .userInitiated).async { + let callback: OpenAIService.StreamCallback = ( + onToken: { token in + // 发送token事件 + DispatchQueue.main.async { + eventSink(["type": "token", "content": token]) + } + }, + onComplete: { + // 发送完成事件 + DispatchQueue.main.async { + eventSink(["type": "complete"]) + } + }, + onError: { error in + // 发送错误事件 + DispatchQueue.main.async { + eventSink(["type": "error", "content": error.localizedDescription]) + } + }, + onFunctionCall: { functionCall in + // 发送函数调用事件 + DispatchQueue.main.async { + eventSink(["type": "functionCall", "content": functionCall]) + } + } + ) + + self.openAIService.sendFunctionCallResult( + messages: messagesRaw, + systemPrompt: systemPrompt, + functionCall: functionCallRaw, + functionResult: functionResult, + callback: callback + ) + + // 请求已开始 + DispatchQueue.main.async { + result(true) + } + } + + case "createUserMessage": + if let args = call.arguments as? [String: Any], + let content = args["content"] as? String { + let message = openAIService.createUserMessage(content: content) + result(message) + } else { + result(FlutterError(code: "INVALID_ARGUMENT", message: "创建用户消息参数无效", details: nil)) + } + + case "createAssistantMessage": + if let args = call.arguments as? [String: Any], + let content = args["content"] as? String { + let message = openAIService.createAssistantMessage(content: content) + result(message) + } else { + result(FlutterError(code: "INVALID_ARGUMENT", message: "创建助手消息参数无效", details: nil)) + } + + default: + result(FlutterMethodNotImplemented) + } + } + + // MARK: - FlutterStreamHandler + + public func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { + self.eventSink = events + return nil + } + + public func onCancel(withArguments arguments: Any?) -> FlutterError? { + self.eventSink = nil + return nil + } +} \ No newline at end of file diff --git a/local_plugins/open_ai_service/lib/open_ai_service.dart b/local_plugins/open_ai_service/lib/open_ai_service.dart new file mode 100644 index 000000000..b946d1bff --- /dev/null +++ b/local_plugins/open_ai_service/lib/open_ai_service.dart @@ -0,0 +1,305 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:flutter/services.dart'; + +/// OpenAI服务异常 +class OpenAIException implements Exception { + final String message; + + OpenAIException(this.message); + + @override + String toString() => 'OpenAIException: $message'; +} + +/// OpenAI服务事件类型 +enum OpenAIEventType { + token, + complete, + error, + functionCall, +} + +/// OpenAI服务事件 +class OpenAIEvent { + final OpenAIEventType type; + final dynamic content; + + OpenAIEvent({required this.type, this.content}); + + factory OpenAIEvent.fromMap(Map map) { + final typeStr = map['type'] as String; + final content = map['content']; + + return OpenAIEvent( + type: _typeFromString(typeStr), + content: content, + ); + } + + static OpenAIEventType _typeFromString(String typeStr) { + switch (typeStr) { + case 'token': + return OpenAIEventType.token; + case 'complete': + return OpenAIEventType.complete; + case 'error': + return OpenAIEventType.error; + case 'functionCall': + return OpenAIEventType.functionCall; + default: + throw ArgumentError('未知的事件类型: $typeStr'); + } + } +} + +/// OpenAI服务插件 +class OpenAIService { + static const MethodChannel _channel = MethodChannel('com.yunqiinnovation.open_ai_service/methods'); + static const EventChannel _eventChannel = EventChannel('com.yunqiinnovation.open_ai_service/events'); + + /// 事件流控制器 + StreamController? _eventStreamController; + + /// 事件流 + Stream? _eventStream; + + /// 获取事件流 + Stream get eventStream { + if (_eventStream == null) { + _eventStreamController = StreamController.broadcast(); + _eventStream = _eventStreamController!.stream; + + // 监听原生事件 + _eventChannel.receiveBroadcastStream().listen( + (dynamic event) { + if (event is Map) { + final eventMap = Map.from(event); + final openAIEvent = OpenAIEvent.fromMap(eventMap); + _eventStreamController!.add(openAIEvent); + } + }, + onError: (error) { + _eventStreamController!.addError(OpenAIException('事件流错误: $error')); + }, + ); + } + + return _eventStream!; + } + + /// 初始化OpenAI服务 + /// + /// [apiKey] OpenAI API密钥 + /// [baseUrl] 可选，自定义API基础URL + /// [model] 可选，自定义使用的模型 + Future initialize({ + required String apiKey, + String baseUrl = '', + String model = '', + }) async { + try { + final result = await _channel.invokeMethod( + 'initialize', + { + 'apiKey': apiKey, + 'baseUrl': baseUrl, + 'model': model, + }, + ); + + return result ?? false; + } catch (e) { + throw OpenAIException('初始化失败: $e'); + } + } + + /// 注册函数 + /// + /// [name] 函数名称 + /// [description] 函数描述 + /// [parameters] 函数参数 + Future registerFunction({ + required String name, + required String description, + required Map parameters, + }) async { + try { + final result = await _channel.invokeMethod( + 'registerFunction', + { + 'name': name, + 'description': description, + 'parameters': parameters, + }, + ); + + return result ?? false; + } catch (e) { + throw OpenAIException('注册函数失败: $e'); + } + } + + /// 创建用户消息 + /// + /// [content] 消息内容 + Future> createUserMessage(String content) async { + try { + final result = await _channel.invokeMethod>( + 'createUserMessage', + {'content': content}, + ); + + if (result == null) { + throw OpenAIException('创建用户消息失败: 结果为空'); + } + + return Map.from(result); + } catch (e) { + throw OpenAIException('创建用户消息失败: $e'); + } + } + + /// 创建助手消息 + /// + /// [content] 消息内容 + Future> createAssistantMessage(String content) async { + try { + final result = await _channel.invokeMethod>( + 'createAssistantMessage', + {'content': content}, + ); + + if (result == null) { + throw OpenAIException('创建助手消息失败: 结果为空'); + } + + return Map.from(result); + } catch (e) { + throw OpenAIException('创建助手消息失败: $e'); + } + } + + /// 发送消息（非流式输出） + /// + /// [messages] 消息列表 + /// [systemPrompt] 系统提示词 + Future sendMessage({ + required List> messages, + required String systemPrompt, + }) async { + try { + final result = await _channel.invokeMethod( + 'sendMessage', + { + 'messages': messages, + 'systemPrompt': systemPrompt, + }, + ); + + if (result == null) { + throw OpenAIException('发送消息失败: 结果为空'); + } + + return result; + } catch (e) { + throw OpenAIException('发送消息失败: $e'); + } + } + + /// 发送消息（流式输出） + /// + /// [messages] 消息列表 + /// [systemPrompt] 系统提示词 + /// + /// 返回一个布尔值，表示请求是否已开始 + Future sendMessageStream({ + required List> messages, + required String systemPrompt, + }) async { + try { + final result = await _channel.invokeMethod( + 'sendMessageStream', + { + 'messages': messages, + 'systemPrompt': systemPrompt, + }, + ); + + return result ?? false; + } catch (e) { + throw OpenAIException('发送流式消息失败: $e'); + } + } + + /// 发送函数调用结果 + /// + /// [messages] 消息列表 + /// [systemPrompt] 系统提示词 + /// [functionCall] 函数调用信息 + /// [functionResult] 函数调用结果 + /// + /// 返回一个布尔值，表示请求是否已开始 + Future sendFunctionCallResult({ + required List> messages, + required String systemPrompt, + required Map functionCall, + required String functionResult, + }) async { + try { + final result = await _channel.invokeMethod( + 'sendFunctionCallResult', + { + 'messages': messages, + 'systemPrompt': systemPrompt, + 'functionCall': functionCall, + 'functionResult': functionResult, + }, + ); + + return result ?? false; + } catch (e) { + throw OpenAIException('发送函数调用结果失败: $e'); + } + } + + /// 处理流式事件 + /// + /// [onToken] 收到token时调用 + /// [onComplete] 完成时调用 + /// [onError] 错误时调用 + /// [onFunctionCall] 函数调用时调用 + StreamSubscription processEvents({ + void Function(String token)? onToken, + void Function()? onComplete, + void Function(String error)? onError, + void Function(Map functionCall)? onFunctionCall, + }) { + return eventStream.listen((event) { + switch (event.type) { + case OpenAIEventType.token: + onToken?.call(event.content as String); + break; + case OpenAIEventType.complete: + onComplete?.call(); + break; + case OpenAIEventType.error: + onError?.call(event.content as String); + break; + case OpenAIEventType.functionCall: + onFunctionCall?.call(Map.from(event.content as Map)); + break; + } + }); + } + + /// 从JSON字符串解析函数调用 + Map parseFunctionCall(String functionCallJson) { + try { + return json.decode(functionCallJson) as Map; + } catch (e) { + throw OpenAIException('解析函数调用失败: $e'); + } + } +} \ No newline at end of file diff --git a/local_plugins/open_ai_service/pubspec.yaml b/local_plugins/open_ai_service/pubspec.yaml new file mode 100644 index 000000000..8e00ed3ea --- /dev/null +++ b/local_plugins/open_ai_service/pubspec.yaml @@ -0,0 +1,26 @@ +name: open_ai_service +description: 原生OpenAI服务插件，提供与OpenAI API的交互功能，支持流式输出和函数调用。 +version: 0.0.1 +homepage: https://github.com/yunqiinnovation/deep_voice + +environment: + sdk: ">=2.17.0 <4.0.0" + flutter: ">=2.5.0" + +dependencies: + flutter: + sdk: flutter + +dev_dependencies: + flutter_test: + sdk: flutter + flutter_lints: ^2.0.0 + +flutter: + plugin: + platforms: + android: + package: com.yunqiinnovation.open_ai_service + pluginClass: OpenAIServicePlugin + ios: + pluginClass: OpenAIServicePlugin \ No newline at end of file diff --git a/pubspec.yaml b/pubspec.yaml index 3e1dd7c90..19366500b 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -62,6 +62,8 @@ dependencies: path: ^1.9.1 azure_speech: path: local_plugins/azure_speech + open_ai_service: + path: local_plugins/open_ai_service dev_dependencies: flutter_test: