From fd5478dbf608b5d304743cc556fe4d70f44b922a Mon Sep 17 00:00:00 2001 From: wolfplus Date: Sat, 5 Apr 2025 13:29:57 +0100 Subject: [PATCH 01/14] init --- android/app/build.gradle.kts | 10 + .../com/example/deep_voice/AzureAsrHelper.kt | 654 --------------- .../example/deep_voice/VolcanoAIService.kt | 344 -------- .../deepsound}/ClassicBluetoothHelper.kt | 0 .../deepsound}/MainActivity.kt | 255 +----- .../deepsound/OpenAIService.kt | 606 ++++++++++++++ .../deepsound/VoiceFunctionHandler.kt | 180 +++++ .../deepsound}/VoiceInteractionService.kt | 210 +++-- .../deepsound}/core/utils/FileLogger.kt | 0 android/settings.gradle.kts | 4 + azure/LICENSE | 21 - azure/README.md | 66 -- azure/ios/Classes/AzureAsrHelper.swift | 757 ------------------ .../AzureSpeechRecognitionPlugin.swift | 18 - azure/ios/Classes/AzureTtsHelper.swift | 427 ---------- .../SwiftAzureSpeechRecognitionPlugin.swift | 259 ------ azure/ios/azure_speech_recognition.podspec | 24 - azure/lib/azure_speech_recognition.dart | 6 - azure/pubspec.yaml | 23 - .../speech_impl/azure_asr_service.dart | 4 +- .../speech_impl/azure_tts_service.dart | 2 +- .../services/voice_interaction_service.dart | 12 +- local_plugins/azure_speech/README.md | 136 ++++ .../azure_speech/android/build.gradle.kts | 65 ++ .../azure_speech/android/settings.gradle.kts | 1 + .../android/src/main/AndroidManifest.xml | 6 + .../azure_speech/AzureAsrHelper.kt | 593 ++++++++++++++ .../azure_speech/AzureSpeechPlugin.kt | 297 +++++++ .../azure_speech}/AzureTtsHelper.kt | 12 +- .../azure_speech/utils/FileLogger.kt | 45 ++ .../ios/Classes/AzureAsrHelper.swift | 461 +++++++++++ .../ios/Classes/AzureSpeechPlugin.swift | 182 +++++ .../ios/Classes/AzureTtsHelper.swift | 330 ++++++++ local_plugins/azure_speech/pubspec.yaml | 32 + pubspec.yaml | 2 + 35 files changed, 3148 insertions(+), 2896 deletions(-) delete mode 100644 android/app/src/main/kotlin/com/example/deep_voice/AzureAsrHelper.kt delete mode 100644 android/app/src/main/kotlin/com/example/deep_voice/VolcanoAIService.kt rename android/app/src/main/kotlin/com/{example/deep_voice => yunqiinnovation/deepsound}/ClassicBluetoothHelper.kt (100%) rename android/app/src/main/kotlin/com/{example/deep_voice => yunqiinnovation/deepsound}/MainActivity.kt (67%) create mode 100644 android/app/src/main/kotlin/com/yunqiinnovation/deepsound/OpenAIService.kt create mode 100644 android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt rename android/app/src/main/kotlin/com/{example/deep_voice => yunqiinnovation/deepsound}/VoiceInteractionService.kt (78%) rename android/app/src/main/kotlin/com/{example/deep_voice => yunqiinnovation/deepsound}/core/utils/FileLogger.kt (100%) delete mode 100644 azure/LICENSE delete mode 100644 azure/README.md delete mode 100644 azure/ios/Classes/AzureAsrHelper.swift delete mode 100644 azure/ios/Classes/AzureSpeechRecognitionPlugin.swift delete mode 100644 azure/ios/Classes/AzureTtsHelper.swift delete mode 100644 azure/ios/Classes/SwiftAzureSpeechRecognitionPlugin.swift delete mode 100644 azure/ios/azure_speech_recognition.podspec delete mode 100644 azure/lib/azure_speech_recognition.dart delete mode 100644 azure/pubspec.yaml create mode 100644 local_plugins/azure_speech/README.md create mode 100644 local_plugins/azure_speech/android/build.gradle.kts create mode 100644 local_plugins/azure_speech/android/settings.gradle.kts create mode 100644 local_plugins/azure_speech/android/src/main/AndroidManifest.xml create mode 100644 local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt create mode 100644 local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt rename {android/app/src/main/kotlin/com/example/deep_voice => local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech}/AzureTtsHelper.kt (98%) create mode 100644 local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/utils/FileLogger.kt create mode 100644 local_plugins/azure_speech/ios/Classes/AzureAsrHelper.swift create mode 100644 local_plugins/azure_speech/ios/Classes/AzureSpeechPlugin.swift create mode 100644 local_plugins/azure_speech/ios/Classes/AzureTtsHelper.swift create mode 100644 local_plugins/azure_speech/pubspec.yaml diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 542cf24d8..78b816b75 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -34,6 +34,11 @@ android { jvmTarget = JavaVersion.VERSION_11.toString() } + // 添加lint选项 + lintOptions { + isCheckReleaseBuilds = false + } + defaultConfig { // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html). applicationId = "com.yunqiinnovation.deepsound" @@ -82,6 +87,11 @@ android { dependencies { + implementation("io.modelcontextprotocol:kotlin-sdk:0.4.0") + + + // 添加本地插件模块依赖 + implementation(project(":azure_speech")) // 添加OkHttp依赖 implementation("com.squareup.okhttp3:okhttp:4.9.3") diff --git a/android/app/src/main/kotlin/com/example/deep_voice/AzureAsrHelper.kt b/android/app/src/main/kotlin/com/example/deep_voice/AzureAsrHelper.kt deleted file mode 100644 index 171adf80e..000000000 --- a/android/app/src/main/kotlin/com/example/deep_voice/AzureAsrHelper.kt +++ /dev/null @@ -1,654 +0,0 @@ -package com.yunqiinnovation.deepsound - -import android.content.Context -import android.media.AudioAttributes -import android.media.AudioFormat -import android.media.AudioRecord -import android.media.MediaRecorder -import android.media.audiofx.AcousticEchoCanceler -import android.media.audiofx.NoiseSuppressor -import android.media.audiofx.AutomaticGainControl -import android.os.Process -import com.yunqiinnovation.deepsound.core.utils.FileLogger -import com.microsoft.cognitiveservices.speech.* -import com.microsoft.cognitiveservices.speech.audio.* -import com.microsoft.cognitiveservices.speech.util.EventHandler -import java.util.concurrent.ExecutionException -import java.util.concurrent.atomic.AtomicBoolean -// 移除WebRTC相关导入 - -class AzureAsrHelper(private val context: Context) { - private var recognizer: SpeechRecognizer? = null - private var speechConfig: SpeechConfig? = null - private val TAG = "AzureAsrHelper" - private var isContinuousRecognitionActive = false - private var currentLanguage = "zh-CN" - private var subscriptionKey = "" - private var serviceRegion = "" - private var isAutoDetectLanguage = false - private var supportedLanguages = arrayOf("zh-CN", "en-US") - - // 是否使用回音消除 - 内部控制常量 - private val useEchoCancellation = true - - // 自定义音频处理相关 - private var customAudioProcessor: CustomAudioProcessor? = null - private var pushStream: PushAudioInputStream? = null - private var audioConfig: AudioConfig? = null - - // 初始化SDK并创建recognizer - fun initialize(subscriptionKey: String, serviceRegion: String, - supportedLanguages: Array = arrayOf("zh-CN", "en-US")): Boolean { - try { - FileLogger.d(TAG, "初始化 Azure 语音服务") - - // 检查配置是否为空 - if (subscriptionKey.isEmpty() || serviceRegion.isEmpty()) { - FileLogger.e(TAG, "Azure 配置信息不完整") - return false - } - - // 释放之前的资源 - dispose() - - this.subscriptionKey = subscriptionKey - this.serviceRegion = serviceRegion - - // 设置语言 - if (supportedLanguages.isNotEmpty()) { - this.supportedLanguages = supportedLanguages - } - - // 根据支持的语言数量决定是否启用自动语言检测 - this.isAutoDetectLanguage = supportedLanguages.size >= 2 - - // 如果只有一种语言，设置为当前语言 - if (!isAutoDetectLanguage && supportedLanguages.isNotEmpty()) { - this.currentLanguage = supportedLanguages[0] - } - - // 创建语音配置 - speechConfig = SpeechConfig.fromSubscription(subscriptionKey, serviceRegion) - - // 设置语言配置 - if (isAutoDetectLanguage) { - // 设置自动语言检测 - speechConfig?.setProperty(PropertyId.SpeechServiceConnection_LanguageIdMode, "Continuous") - } else { - // 设置指定的识别语言 - speechConfig?.speechRecognitionLanguage = currentLanguage - } - - // 创建识别器 - try { - if (useEchoCancellation) { - // 如果使用回音消除，创建自定义音频输入流 - setupCustomAudioProcessing() - - if (isAutoDetectLanguage) { - val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) - recognizer = SpeechRecognizer(speechConfig, autoDetectConfig, audioConfig) - } else { - recognizer = SpeechRecognizer(speechConfig, audioConfig) - } - } else { - // 使用默认麦克风输入 - if (isAutoDetectLanguage) { - val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) - recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) - } else { - recognizer = SpeechRecognizer(speechConfig) - } - } - - FileLogger.d(TAG, "Azure 语音服务初始化成功") - return true - } catch (e: Exception) { - FileLogger.e(TAG, "创建识别器失败: ${e.message}") - stopCustomAudioProcessing() - return false - } - } catch (e: Exception) { - FileLogger.e(TAG, "初始化失败: ${e.message}") - return false - } - } - - // 重置 recognizer - private fun resetRecognizer(): Boolean { - try { - // 释放之前的 recognizer - recognizer?.close() - recognizer = null - - // 停止当前的音频处理 - stopCustomAudioProcessing() - - // 使用现有配置重新创建 recognizer - if (speechConfig != null) { - if (useEchoCancellation) { - // 如果使用回音消除，创建自定义音频输入流 - setupCustomAudioProcessing() - - if (isAutoDetectLanguage) { - val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) - recognizer = SpeechRecognizer(speechConfig, autoDetectConfig, audioConfig) - } else { - recognizer = SpeechRecognizer(speechConfig, audioConfig) - } - } else { - // 使用默认麦克风输入 - if (isAutoDetectLanguage) { - val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) - recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) - } else { - recognizer = SpeechRecognizer(speechConfig) - } - } - return true - } else { - FileLogger.e(TAG, "语音配置未初始化") - return false - } - } catch (e: Exception) { - FileLogger.e(TAG, "重置识别器失败: ${e.message}") - return false - } - } - - // 开始一次性语音识别 - fun recognizeOnce(callback: RecognizeCallback) { - if (speechConfig == null) { - callback.onError("语音服务未初始化") - return - } - - // 重置 recognizer - if (!resetRecognizer()) { - callback.onError("重置识别器失败") - return - } - - try { - // 启动音频处理 - startCustomAudioProcessing() - - // 执行识别 - val result = recognizer?.recognizeOnceAsync()?.get() - - // 停止音频处理 - stopCustomAudioProcessing() - - if (result != null && result.reason == ResultReason.RecognizedSpeech) { - val detectedLanguage = AutoDetectSourceLanguageResult.fromResult(result)?.language - callback.onResult(result.text, detectedLanguage ?: "") - } else { - callback.onError("未能识别语音") - } - } catch (e: Exception) { - // 停止音频处理 - stopCustomAudioProcessing() - callback.onError("识别异常: ${e.message}") - } - } - - // 开始连续语音识别 - fun startContinuousRecognition(callback: ContinuousRecognizeCallback): Boolean { - if (speechConfig == null) { - callback.onError("语音服务未初始化") - return false - } - - // 如果已经在进行连续识别，先停止 - if (isContinuousRecognitionActive) { - stopContinuousRecognition(callback) - } - - // 重置 recognizer - if (!resetRecognizer()) { - callback.onError("重置识别器失败") - return false - } - - try { - // 启动音频处理 - startCustomAudioProcessing() - - // 设置识别事件处理 - // 最终识别结果 - recognizer?.recognized?.addEventListener( - EventHandler { _, event -> - if (event.result.reason == ResultReason.RecognizedSpeech) { - val detectedLanguage = if (isAutoDetectLanguage) { - AutoDetectSourceLanguageResult.fromResult(event.result)?.language ?: "" - } else { - currentLanguage - } - // FileLogger.d(TAG, "最终识别结果: ${event.result.text}") - callback.onResult(event.result.text, detectedLanguage) - } - } - ) - - // 识别中事件 - recognizer?.recognizing?.addEventListener( - EventHandler { _, event -> - if (event.result.reason == ResultReason.RecognizingSpeech) { - val detectedLanguage = if (isAutoDetectLanguage) { - AutoDetectSourceLanguageResult.fromResult(event.result)?.language ?: "" - } else { - currentLanguage - } - // FileLogger.d(TAG, "识别中结果: ${event.result.text}") - callback.onRecognizing(event.result.text, detectedLanguage) - } - } - ) - - // 会话事件 - recognizer?.sessionStarted?.addEventListener( - EventHandler { _, _ -> - isContinuousRecognitionActive = true - callback.onSessionStarted() - } - ) - - recognizer?.sessionStopped?.addEventListener( - EventHandler { _, _ -> - isContinuousRecognitionActive = false - callback.onSessionStopped() - } - ) - - // 取消事件 - recognizer?.canceled?.addEventListener( - EventHandler { _, event -> - val errorDetails = if (event.reason == CancellationReason.Error) event.errorDetails else "" - callback.onCanceled(event.reason.toString(), errorDetails) - isContinuousRecognitionActive = false - } - ) - - // 开始连续识别 - recognizer?.startContinuousRecognitionAsync()?.get() - isContinuousRecognitionActive = true - - return true - } catch (e: Exception) { - callback.onError("开始连续识别失败: ${e.message}") - isContinuousRecognitionActive = false - stopCustomAudioProcessing() - return false - } - } - - // 停止连续语音识别 - fun stopContinuousRecognition(callback: ContinuousRecognizeCallback): Boolean { - if (!isContinuousRecognitionActive || recognizer == null) { - return true - } - - try { - recognizer?.stopContinuousRecognitionAsync() - isContinuousRecognitionActive = false - callback.onSessionStopped() - - // 停止音频处理 - stopCustomAudioProcessing() - - return true - } catch (e: Exception) { - callback.onError("停止连续识别失败: ${e.message}") - isContinuousRecognitionActive = false - stopCustomAudioProcessing() - return false - } - } - - // 检查连续识别是否活跃 - fun isContinuousRecognitionActive(): Boolean { - return isContinuousRecognitionActive - } - - // 设置自定义音频处理 - private fun setupCustomAudioProcessing() { - if (!useEchoCancellation) { - return - } - - try { - // 1. 创建PushAudioInputStream - pushStream = PushAudioInputStream.create() - - // 2. 创建AudioConfig - audioConfig = AudioConfig.fromStreamInput(pushStream) - - // 3. 创建自定义音频处理器 - customAudioProcessor = CustomAudioProcessor(pushStream) - - FileLogger.d(TAG, "自定义音频处理设置完成") - } catch (e: Exception) { - FileLogger.e(TAG, "设置自定义音频处理失败: ${e.message}") - releaseCustomAudioProcessing() - - // 降级处理：如果自定义处理设置失败，尝试使用默认麦克风 - try { - FileLogger.d(TAG, "尝试降级到默认麦克风输入") - audioConfig = AudioConfig.fromDefaultMicrophoneInput() - } catch (e2: Exception) { - FileLogger.e(TAG, "默认麦克风输入设置也失败: ${e2.message}") - audioConfig = null - } - } - } - - // 启动自定义音频处理 - private fun startCustomAudioProcessing() { - if (!useEchoCancellation || customAudioProcessor == null) { - return - } - - try { - customAudioProcessor?.startRecording() - FileLogger.d(TAG, "自定义音频处理已启动") - } catch (e: Exception) { - FileLogger.e(TAG, "启动自定义音频处理失败: ${e.message}") - } - } - - // 停止自定义音频处理 - private fun stopCustomAudioProcessing() { - if (!useEchoCancellation || customAudioProcessor == null) { - return - } - - try { - customAudioProcessor?.stopRecording() - FileLogger.d(TAG, "自定义音频处理已停止") - } catch (e: Exception) { - FileLogger.e(TAG, "停止自定义音频处理失败: ${e.message}") - } - } - - // 释放自定义音频处理资源 - private fun releaseCustomAudioProcessing() { - stopCustomAudioProcessing() - - try { - customAudioProcessor = null - pushStream?.close() - pushStream = null - audioConfig?.close() - audioConfig = null - - FileLogger.d(TAG, "自定义音频处理资源已释放") - } catch (e: Exception) { - FileLogger.e(TAG, "释放自定义音频处理资源时出错: ${e.message}") - } - } - - // 释放所有资源 - fun dispose() { - try { - // 停止和释放音频处理 - releaseCustomAudioProcessing() - - recognizer?.close() - recognizer = null - - speechConfig?.close() - speechConfig = null - - isContinuousRecognitionActive = false - - FileLogger.d(TAG, "语音识别资源已释放") - } catch (e: Exception) { - FileLogger.e(TAG, "释放资源出错: ${e.message}") - } - } - - // 自定义音频处理器 - 使用Android原生回音消除 - private inner class CustomAudioProcessor(private val pushStream: PushAudioInputStream?) { - private val SAMPLE_RATE = 16000 - private val CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO - private val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT - private val BUFFER_SIZE = SAMPLE_RATE * 2 // 简化缓冲区大小计算，更稳定 - - private var audioRecord: AudioRecord? = null - private var echoCanceler: AcousticEchoCanceler? = null - private val isRecording = AtomicBoolean(false) - private var recordingThread: Thread? = null - - // 启动录音并处理音频数据 - fun startRecording() { - if (isRecording.get() || pushStream == null) { - return - } - - try { - // 使用Builder模式构建AudioFormat - val audioFormat = AudioFormat.Builder() - .setSampleRate(SAMPLE_RATE) - .setEncoding(AUDIO_FORMAT) - .setChannelMask(CHANNEL_CONFIG) - .build() - - // 使用Builder模式创建AudioRecord实例 - audioRecord = AudioRecord.Builder() - .setAudioSource(MediaRecorder.AudioSource.VOICE_COMMUNICATION) - .setAudioFormat(audioFormat) - .setBufferSizeInBytes(BUFFER_SIZE) - .build() - - // 检查AudioRecord初始化状态 - if (audioRecord?.state != AudioRecord.STATE_INITIALIZED) { - FileLogger.e(TAG, "AudioRecord初始化失败，状态: ${audioRecord?.state}") - // 尝试使用DEFAULT音频源重试一次 - audioRecord?.release() - audioRecord = AudioRecord.Builder() - .setAudioSource(MediaRecorder.AudioSource.DEFAULT) - .setAudioFormat(audioFormat) - .setBufferSizeInBytes(BUFFER_SIZE) - .build() - - if (audioRecord?.state != AudioRecord.STATE_INITIALIZED) { - FileLogger.e(TAG, "AudioRecord初始化第二次尝试也失败，放弃") - releaseAudioResources() - return - } else { - FileLogger.d(TAG, "使用默认音频源成功初始化AudioRecord") - } - } - - // 启用音频效果（回音消除、噪声抑制等） - enableAudioEffects() - - // 启动录音 - audioRecord?.startRecording() - isRecording.set(true) - - // 创建录音线程 - recordingThread = Thread({ - val buffer = ByteArray(BUFFER_SIZE) - - while (isRecording.get()) { - try { - val readSize = audioRecord?.read(buffer, 0, BUFFER_SIZE) ?: 0 - - if (readSize > 0) { - try { - // 将处理后的音频数据推送到流 - if (readSize == buffer.size) { - // 如果读取的大小等于buffer的大小，直接写入整个buffer - pushStream.write(buffer) - } else { - // 如果只读取了部分数据，创建新的数组只包含有效数据 - val validData = buffer.copyOfRange(0, readSize) - pushStream.write(validData) - } - } catch (e: Exception) { - FileLogger.e(TAG, "写入音频数据失败: ${e.message}") - break - } - } else if (readSize == 0) { - // 读取为0，可能是临时的，等待一下继续尝试 - Thread.sleep(10) - } else { - // 负值表示错误 - FileLogger.e(TAG, "读取音频数据失败，错误码: $readSize") - break - } - } catch (e: Exception) { - FileLogger.e(TAG, "录音线程异常: ${e.message}") - break - } - } - }, "AudioRecordingThread") - - // 设置线程优先级并启动 - recordingThread?.priority = Thread.MAX_PRIORITY - recordingThread?.start() - - FileLogger.d(TAG, "音频录制已启动" + (if(echoCanceler?.enabled == true) "，回音消除已启用" else "")) - } catch (e: Exception) { - FileLogger.e(TAG, "启动音频录制失败: ${e.message}") - releaseAudioResources() - } - } - - // 启用音频效果（回音消除、噪声抑制等） - private fun enableAudioEffects() { - try { - val audioSessionId = audioRecord?.audioSessionId ?: -1 - - if (audioSessionId != -1) { - // 启用回音消除 - if (AcousticEchoCanceler.isAvailable()) { - try { - echoCanceler = AcousticEchoCanceler.create(audioSessionId) - if (echoCanceler != null) { - echoCanceler?.enabled = true - FileLogger.d(TAG, "回音消除已启用，会话ID: $audioSessionId") - } else { - FileLogger.w(TAG, "回音消除器创建返回null") - } - } catch (e: Exception) { - FileLogger.e(TAG, "创建回音消除器时出错: ${e.message}") - } - } else { - FileLogger.d(TAG, "设备不支持回音消除") - } - - // 以下功能暂时不启用，可根据需要取消注释 - /* - // 启用噪声抑制 - if (NoiseSuppressor.isAvailable()) { - try { - val ns = NoiseSuppressor.create(audioSessionId) - ns?.enabled = true - FileLogger.d(TAG, "噪声抑制已启用") - } catch (e: Exception) { - FileLogger.e(TAG, "创建噪声抑制器时出错: ${e.message}") - } - } - - // 启用自动增益控制 - if (AutomaticGainControl.isAvailable()) { - try { - val agc = AutomaticGainControl.create(audioSessionId) - agc?.enabled = true - FileLogger.d(TAG, "自动增益控制已启用") - } catch (e: Exception) { - FileLogger.e(TAG, "创建自动增益控制时出错: ${e.message}") - } - } - */ - } else { - FileLogger.w(TAG, "无效的音频会话ID，无法启用音频效果") - } - } catch (e: Exception) { - FileLogger.e(TAG, "启用音频效果时出错: ${e.message}") - } - } - - // 停止录音 - fun stopRecording() { - if (!isRecording.get()) { - return - } - - isRecording.set(false) - - try { - // 等待录音线程结束 - recordingThread?.join(1000) - - // 释放资源 - releaseAudioResources() - - FileLogger.d(TAG, "音频录制已停止") - } catch (e: Exception) { - FileLogger.e(TAG, "停止音频录制失败: ${e.message}") - } - } - - // 释放音频资源 - private fun releaseAudioResources() { - try { - // 停止录音 - try { - if (audioRecord?.state == AudioRecord.STATE_INITIALIZED) { - audioRecord?.stop() - } - } catch (e: Exception) { - // 忽略可能的IllegalStateException - FileLogger.w(TAG, "停止AudioRecord时出错: ${e.message}") - } - - // 释放回音消除器 - try { - if (echoCanceler != null) { - echoCanceler?.enabled = false - echoCanceler?.release() - echoCanceler = null - } - } catch (e: Exception) { - FileLogger.w(TAG, "释放回音消除器时出错: ${e.message}") - } finally { - echoCanceler = null - } - - // 释放音频记录器 - try { - audioRecord?.release() - } catch (e: Exception) { - FileLogger.w(TAG, "释放AudioRecord时出错: ${e.message}") - } finally { - audioRecord = null - } - - // 重置线程 - recordingThread = null - - } catch (e: Exception) { - FileLogger.e(TAG, "释放音频资源失败: ${e.message}") - } - } - } - - // 一次性识别回调接口 - interface RecognizeCallback { - fun onResult(result: String, detectedLanguage: String = "") - fun onError(error: String) - } - - // 连续识别回调接口 - interface ContinuousRecognizeCallback { - fun onResult(result: String, detectedLanguage: String = "") - fun onRecognizing(recognizing: String, detectedLanguage: String = "") - fun onSessionStarted() - fun onSessionStopped() - fun onCanceled(reason: String, errorDetails: String) - fun onError(error: String) - } -} \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/example/deep_voice/VolcanoAIService.kt b/android/app/src/main/kotlin/com/example/deep_voice/VolcanoAIService.kt deleted file mode 100644 index 2fbcd8da8..000000000 --- a/android/app/src/main/kotlin/com/example/deep_voice/VolcanoAIService.kt +++ /dev/null @@ -1,344 +0,0 @@ -package com.yunqiinnovation.deepsound - -import android.util.Log -import okhttp3.* -import okhttp3.MediaType.Companion.toMediaTypeOrNull -import okhttp3.RequestBody.Companion.toRequestBody -import org.json.JSONArray -import org.json.JSONObject -import java.io.IOException -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit - -/** - * 火山AI服务的原生实现 - * - * 参考Flutter端的VolcanoAIService实现，提供同步和异步的API调用方式 - */ -class VolcanoAIService() { - private val TAG = "VolcanoAIService" - private val baseUrl = "https://ark.cn-beijing.volces.com/api/v3" - private val chatEndpoint = "/chat/completions" - private val client = OkHttpClient.Builder() - .connectTimeout(30, TimeUnit.SECONDS) - .readTimeout(30, TimeUnit.SECONDS) - .writeTimeout(30, TimeUnit.SECONDS) - .build() - - private var apiKey: String = "" - private var isInitialized = false - - /** - * 初始化火山AI服务 - * - * @param apiKey 火山AI API密钥 - * @return 初始化是否成功 - */ - fun initialize(apiKey: String): Boolean { - this.apiKey = apiKey - isInitialized = apiKey.isNotEmpty() - - if (!isInitialized) { - Log.e(TAG, "初始化失败：API key 不能为空") - } else { - Log.d(TAG, "火山AI服务初始化成功") - } - - return isInitialized - } - - /** - * 生成个性化问候语 - * - * @param agentName 代理名称 - * @param systemPrompt 系统提示词 - * @param callback 回调函数，返回生成的问候语 - */ - fun generateGreeting(agentName: String, systemPrompt: String, callback: (String?, Exception?) -> Unit) { - val messages = JSONArray().apply { - put(JSONObject().apply { - put("role", "system") - put("content", systemPrompt) - }) - put(JSONObject().apply { - put("role", "user") - put("content", "请用一句简短的话向我打个招呼，要符合你的身份和性格特点，不要超过18个字。") - }) - } - - sendMessageStream(messages, systemPrompt, object : StreamCallback { - val stringBuilder = StringBuilder() - - override fun onToken(token: String) { - stringBuilder.append(token) - } - - override fun onComplete() { - callback(stringBuilder.toString(), null) - } - - override fun onError(e: Exception) { - callback(null, e) - } - }) - } - - /** - * 发送消息（非流式输出） - * - * @param messages 消息列表 - * @param systemPrompt 系统提示词 - * @return 返回AI的回复 - * @throws VolcanoAIException 如果API调用失败 - */ - @Throws(VolcanoAIException::class) - fun sendMessage(messages: JSONArray, systemPrompt: String): String { - // 检查是否已初始化 - if (!isInitialized || apiKey.isEmpty()) { - throw VolcanoAIException("火山AI服务未初始化或API key为空，请先调用initialize方法") - } - - val fullMessages = JSONArray().apply { - put(JSONObject().apply { - put("role", "system") - put("content", systemPrompt) - }) - for (i in 0 until messages.length()) { - put(messages.getJSONObject(i)) - } - } - - val requestBody = JSONObject().apply { - put("model", "doubao-1-5-lite-32k-250115") - put("messages", fullMessages) - put("temperature", 0.7) - put("max_tokens", 2000) - put("stream", false) - } - - val mediaType = "application/json".toMediaTypeOrNull() - val request = Request.Builder() - .url("$baseUrl$chatEndpoint") - .addHeader("Content-Type", "application/json") - .addHeader("Authorization", "Bearer $apiKey") - .post(requestBody.toString().toRequestBody(mediaType)) - .build() - - try { - client.newCall(request).execute().use { response -> - if (!response.isSuccessful) { - val errorBody = response.body?.string() ?: "" - val errorMessage = try { - JSONObject(errorBody).getJSONObject("error").getString("message") - } catch (e: Exception) { - "Unknown error occurred" - } - throw VolcanoAIException(errorMessage) - } - - val responseBody = response.body?.string() ?: throw VolcanoAIException("Empty response") - val jsonResponse = JSONObject(responseBody) - - if (jsonResponse.has("choices") && - jsonResponse.getJSONArray("choices").length() > 0 && - jsonResponse.getJSONArray("choices").getJSONObject(0).has("message")) { - return jsonResponse.getJSONArray("choices") - .getJSONObject(0) - .getJSONObject("message") - .getString("content") - } - - throw VolcanoAIException("Invalid response format") - } - } catch (e: Exception) { - if (e is VolcanoAIException) throw e - throw VolcanoAIException("Failed to communicate with AI service: ${e.message}") - } - } - - /** - * 发送消息（流式输出） - * - * @param messages 消息列表 - * @param systemPrompt 系统提示词 - * @param callback 回调函数，用于接收流式输出的结果 - */ - fun sendMessageStream(messages: JSONArray, systemPrompt: String, callback: StreamCallback) { - // 检查是否已初始化 - if (!isInitialized || apiKey.isEmpty()) { - callback.onError(VolcanoAIException("火山AI服务未初始化或API key为空，请先调用initialize方法")) - return - } - - val fullMessages = JSONArray().apply { - put(JSONObject().apply { - put("role", "system") - put("content", systemPrompt) - }) - for (i in 0 until messages.length()) { - put(messages.getJSONObject(i)) - } - } - - val requestBody = JSONObject().apply { - put("model", "doubao-1-5-lite-32k-250115") - put("messages", fullMessages) - put("temperature", 0.7) - put("max_tokens", 2000) - put("stream", true) - } - - val mediaType = "application/json".toMediaTypeOrNull() - val request = Request.Builder() - .url("$baseUrl$chatEndpoint") - .addHeader("Content-Type", "application/json") - .addHeader("Authorization", "Bearer $apiKey") - .addHeader("Accept", "text/event-stream") - .post(requestBody.toString().toRequestBody(mediaType)) - .build() - - client.newCall(request).enqueue(object : Callback { - override fun onFailure(call: Call, e: IOException) { - callback.onError(VolcanoAIException("Failed to communicate with AI service: ${e.message}")) - } - - override fun onResponse(call: Call, response: Response) { - if (!response.isSuccessful) { - val errorBody = response.body?.string() ?: "" - val errorMessage = try { - JSONObject(errorBody).getJSONObject("error").getString("message") - } catch (e: Exception) { - "Unknown error occurred" - } - callback.onError(VolcanoAIException(errorMessage)) - return - } - - val responseBody = response.body ?: return - val source = responseBody.source() - val bufferedSource = source.buffer - - try { - while (!bufferedSource.exhausted()) { - val line = bufferedSource.readUtf8Line() ?: continue - - if (line.isEmpty()) continue - if (line.startsWith("data: ")) { - val data = line.substring(6) - if (data == "[DONE]") { - callback.onComplete() - break - } - - try { - val jsonData = JSONObject(data) - if (jsonData.has("choices") && - jsonData.getJSONArray("choices").length() > 0 && - jsonData.getJSONArray("choices").getJSONObject(0).has("delta") && - jsonData.getJSONArray("choices").getJSONObject(0).getJSONObject("delta").has("content")) { - val content = jsonData.getJSONArray("choices") - .getJSONObject(0) - .getJSONObject("delta") - .getString("content") - callback.onToken(content) - } - } catch (e: Exception) { - // 忽略无效的JSON数据 - continue - } - } - } - } catch (e: Exception) { - callback.onError(VolcanoAIException("Error processing stream: ${e.message}")) - } finally { - response.close() - } - } - }) - } - - /** - * 同步方式发送消息（流式输出） - * - * 注意：此方法会阻塞当前线程，请在后台线程中调用 - * - * @param messages 消息列表 - * @param systemPrompt 系统提示词 - * @return 返回完整的AI回复 - * @throws VolcanoAIException 如果API调用失败 - */ - @Throws(VolcanoAIException::class) - fun sendMessageStreamSync(messages: JSONArray, systemPrompt: String): String { - val result = StringBuilder() - val latch = CountDownLatch(1) - var exception: Exception? = null - - sendMessageStream(messages, systemPrompt, object : StreamCallback { - override fun onToken(token: String) { - result.append(token) - } - - override fun onComplete() { - latch.countDown() - } - - override fun onError(e: Exception) { - exception = e - latch.countDown() - } - }) - - // 等待流式输出完成或出错 - latch.await(60, TimeUnit.SECONDS) - - if (exception != null) { - throw exception as VolcanoAIException - } - - return result.toString() - } - - /** - * 创建用户消息 - */ - fun createUserMessage(content: String): JSONObject { - return JSONObject().apply { - put("role", "user") - put("content", content) - } - } - - /** - * 创建系统消息 - */ - fun createSystemMessage(content: String): JSONObject { - return JSONObject().apply { - put("role", "system") - put("content", content) - } - } - - /** - * 创建助手消息 - */ - fun createAssistantMessage(content: String): JSONObject { - return JSONObject().apply { - put("role", "assistant") - put("content", content) - } - } - - /** - * 流式输出回调接口 - */ - interface StreamCallback { - fun onToken(token: String) - fun onComplete() - fun onError(e: Exception) - } -} - -/** - * 火山AI异常 - */ -class VolcanoAIException(message: String) : Exception(message) \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/example/deep_voice/ClassicBluetoothHelper.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/ClassicBluetoothHelper.kt similarity index 100% rename from android/app/src/main/kotlin/com/example/deep_voice/ClassicBluetoothHelper.kt rename to android/app/src/main/kotlin/com/yunqiinnovation/deepsound/ClassicBluetoothHelper.kt diff --git a/android/app/src/main/kotlin/com/example/deep_voice/MainActivity.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt similarity index 67% rename from android/app/src/main/kotlin/com/example/deep_voice/MainActivity.kt rename to android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt index 49fd8fdf6..df36efb44 100644 --- a/android/app/src/main/kotlin/com/example/deep_voice/MainActivity.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt @@ -23,18 +23,12 @@ import com.yunqiinnovation.deepsound.core.utils.FileLogger class MainActivity: FlutterActivity() { - private val AZURE_ASR_CHANNEL = "com.deep_voice.azure_asr" - private val AZURE_ASR_EVENT_CHANNEL = "com.deep_voice.azure_asr_events" - private val AZURE_TTS_CHANNEL = "com.deep_voice.azure_tts" private val VOICE_INTERACTION_CHANNEL = "com.deep_voice.voice_interaction" private val VOICE_INTERACTION_EVENT_CHANNEL = "com.deep_voice.voice_interaction_events" private val CLASSIC_BLUETOOTH_CHANNEL = "com.deep_voice.classic_bluetooth" private val CLASSIC_BLUETOOTH_EVENT_CHANNEL = "com.deep_voice.classic_bluetooth_events" private val TAG = "MainActivity" - private lateinit var azureAsrHelper: AzureAsrHelper - private lateinit var azureTtsHelper: AzureTtsHelper private lateinit var classicBluetoothHelper: ClassicBluetoothHelper - private var azureAsrEventSink: EventChannel.EventSink? = null private var voiceInteractionEventSink: EventChannel.EventSink? = null private var bluetoothEventSink: EventChannel.EventSink? = null @@ -58,10 +52,6 @@ class MainActivity: FlutterActivity() { val assistantMessage = intent.getStringExtra("assistantMessage") ?: "" val timestamp = intent.getLongExtra("timestamp", System.currentTimeMillis()) - Log.d(TAG, "收到聊天记录更新广播: agentId=$agentId, timestamp=$timestamp") - Log.d(TAG, "用户消息: ${userMessage.take(50)}...") - Log.d(TAG, "助手回复: ${assistantMessage.take(50)}...") - sendChatHistoryEvent(agentId, userMessage, assistantMessage, timestamp) } } @@ -143,13 +133,17 @@ class MainActivity: FlutterActivity() { var azureSpeechKey: String = "" var azureSpeechRegion: String = "" var volcanoAiApiKey: String = "" + var openaiApiKey: String = "" + var openaiBaseUrl: String? = null // 安全存储相关常量 private const val SECURE_PREFS_FILENAME = "deep_voice_secure_prefs" private const val KEY_AZURE_SPEECH_KEY = "azure_speech_key" private const val KEY_AZURE_SPEECH_REGION = "azure_speech_region" private const val KEY_VOLCANO_AI_API_KEY = "volcano_ai_api_key" - + private const val KEY_OPENAI_API_KEY = "openai_api_key" + private const val KEY_OPENAI_BASE_URL = "openai_base_url" + private const val KEY_MCP_SERVER_ENDPOINT = "mcp_server_endpoint" // 会话管理 private const val KEY_SESSION_ID = "session_id" private var currentSessionId = "" @@ -183,6 +177,8 @@ class MainActivity: FlutterActivity() { .putString(KEY_AZURE_SPEECH_KEY, azureSpeechKey) .putString(KEY_AZURE_SPEECH_REGION, azureSpeechRegion) .putString(KEY_VOLCANO_AI_API_KEY, volcanoAiApiKey) + .putString(KEY_OPENAI_API_KEY, openaiApiKey) + .putString(KEY_OPENAI_BASE_URL, openaiBaseUrl) .putString(KEY_SESSION_ID, currentSessionId) .apply() @@ -227,11 +223,13 @@ class MainActivity: FlutterActivity() { azureSpeechKey = sharedPreferences.getString(KEY_AZURE_SPEECH_KEY, "") ?: "" azureSpeechRegion = sharedPreferences.getString(KEY_AZURE_SPEECH_REGION, "") ?: "" volcanoAiApiKey = sharedPreferences.getString(KEY_VOLCANO_AI_API_KEY, "") ?: "" - + openaiApiKey = sharedPreferences.getString(KEY_OPENAI_API_KEY, "") ?: "" + openaiBaseUrl = sharedPreferences.getString(KEY_OPENAI_BASE_URL, null) FileLogger.d("MainActivity", "已从加密存储加载密钥") - // 检查是否成功获取所有密钥 - return azureSpeechKey.isNotEmpty() && azureSpeechRegion.isNotEmpty() && volcanoAiApiKey.isNotEmpty() + // 检查是否成功获取所有必要密钥 + return azureSpeechKey.isNotEmpty() && azureSpeechRegion.isNotEmpty() && + (openaiApiKey.isNotEmpty() || volcanoAiApiKey.isNotEmpty()) } catch (e: Exception) { FileLogger.e("MainActivity", "从加密存储加载密钥失败: ${e.message}") e.printStackTrace() @@ -247,8 +245,6 @@ class MainActivity: FlutterActivity() { FileLogger.init(applicationContext) // 初始化 Azure 语音服务 - azureAsrHelper = AzureAsrHelper(applicationContext) - azureTtsHelper = AzureTtsHelper(applicationContext) classicBluetoothHelper = ClassicBluetoothHelper(applicationContext) // 注册广播接收器 @@ -300,8 +296,7 @@ class MainActivity: FlutterActivity() { setupMethodChannels(flutterEngine) // 初始化 Azure 语音服务 - azureAsrHelper = AzureAsrHelper(this) - azureTtsHelper = AzureTtsHelper(this) + classicBluetoothHelper = ClassicBluetoothHelper(this) Log.d(TAG, "Flutter 引擎配置完成") } @@ -327,20 +322,6 @@ class MainActivity: FlutterActivity() { } ) - // Azure ASR 事件通道 - EventChannel(flutterEngine.dartExecutor.binaryMessenger, AZURE_ASR_EVENT_CHANNEL).setStreamHandler( - object : EventChannel.StreamHandler { - override fun onListen(arguments: Any?, events: EventChannel.EventSink?) { - Log.d(TAG, "ASR事件通道开始监听") - azureAsrEventSink = events - } - - override fun onCancel(arguments: Any?) { - azureAsrEventSink = null - } - } - ) - // 设置蓝牙事件通道 EventChannel(flutterEngine.dartExecutor.binaryMessenger, CLASSIC_BLUETOOTH_EVENT_CHANNEL).setStreamHandler( object : EventChannel.StreamHandler { @@ -367,195 +348,6 @@ class MainActivity: FlutterActivity() { private fun setupMethodChannels(flutterEngine: FlutterEngine) { Log.d(TAG, "开始设置方法通道") - // 设置 Azure ASR 方法通道 - MethodChannel(flutterEngine.dartExecutor.binaryMessenger, AZURE_ASR_CHANNEL).setMethodCallHandler { call, result -> - when (call.method) { - "initialize" -> { - val subscriptionKey = call.argument("subscriptionKey") - val region = call.argument("region") - val supportedLanguages = call.argument>("supportedLanguages")?.toTypedArray() ?: arrayOf("zh-CN", "en-US") - - if (subscriptionKey == null || region == null) { - result.error("INVALID_ARGUMENTS", "订阅密钥和区域不能为空", null) - return@setMethodCallHandler - } - - try { - val success = azureAsrHelper.initialize(subscriptionKey, region, supportedLanguages) - result.success(success) - } catch (e: Exception) { - result.error("INITIALIZATION_ERROR", e.message, null) - } - } - "recognizeOnce" -> { - - azureAsrHelper.recognizeOnce(object : AzureAsrHelper.RecognizeCallback { - override fun onResult(text: String, detectedLanguage: String) { - result.success(mapOf( - "text" to text, - "detectedLanguage" to detectedLanguage - )) - } - - override fun onError(error: String) { - result.error("RECOGNITION_ERROR", error, null) - } - }) - } - "startContinuousRecognition" -> { - // 确保事件通道已准备好 - if (azureAsrEventSink == null) { - result.error("EVENT_CHANNEL_NOT_READY", "事件通道未准备好，无法开始连续识别", null) - return@setMethodCallHandler - } - - val success = azureAsrHelper.startContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { - override fun onResult(text: String, detectedLanguage: String) { - sendAsrEvent(mapOf( - "type" to "result", - "text" to text, - "detectedLanguage" to detectedLanguage - )) - } - - override fun onRecognizing(recognizing: String, detectedLanguage: String) { - sendAsrEvent(mapOf( - "type" to "recognizing", - "text" to recognizing, - "detectedLanguage" to detectedLanguage - )) - } - - override fun onSessionStarted() { - sendAsrEvent(mapOf("type" to "sessionStarted")) - } - - override fun onSessionStopped() { - sendAsrEvent(mapOf("type" to "sessionStopped")) - } - - override fun onCanceled(reason: String, errorDetails: String) { - sendAsrEvent(mapOf( - "type" to "canceled", - "reason" to reason, - "errorDetails" to errorDetails - )) - } - - override fun onError(error: String) { - sendAsrEvent(mapOf("type" to "error", "message" to error)) - } - }) - result.success(success) - } - "stopContinuousRecognition" -> { - try { - if (!azureAsrHelper.isContinuousRecognitionActive()) { - result.success(true) - return@setMethodCallHandler - } - - val success = azureAsrHelper.stopContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { - override fun onResult(text: String, detectedLanguage: String) {} - override fun onRecognizing(recognizing: String, detectedLanguage: String) {} - override fun onSessionStarted() {} - override fun onSessionStopped() {} - override fun onCanceled(reason: String, errorDetails: String) {} - override fun onError(error: String) { - result.error("STOP_ERROR", error, null) - } - }) - result.success(success) - } catch (e: Exception) { - result.error("STOP_ERROR", e.message, null) - } - } - "isContinuousRecognitionActive" -> { - result.success(azureAsrHelper.isContinuousRecognitionActive()) - } - "dispose" -> { - azureAsrHelper.dispose() - result.success(true) - } - else -> { - result.notImplemented() - } - } - } - - // 设置 Azure TTS 方法通道 - MethodChannel(flutterEngine.dartExecutor.binaryMessenger, AZURE_TTS_CHANNEL).setMethodCallHandler { call, result -> - when (call.method) { - "initialize" -> { - val subscriptionKey = call.argument("subscriptionKey") ?: "" - val region = call.argument("region") ?: "" - val language = call.argument("language") ?: "zh-CN" - - val success = azureTtsHelper.initialize(subscriptionKey, region, language) - result.success(success) - } - "setVoice" -> { - val voiceName = call.argument("voiceName") ?: return@setMethodCallHandler result.error("INVALID_ARGUMENTS", "语音名称不能为空", null) - result.success(azureTtsHelper.setVoice(voiceName)) - } - "setSpeechParams" -> { - val rate = call.argument("rate") ?: 0 - val pitch = call.argument("pitch") ?: 0 - val volume = call.argument("volume") ?: 100 - result.success(azureTtsHelper.setSpeechParams(rate, pitch, volume)) - } - "setAudioOutputType" -> { - val outputTypeStr = call.argument("outputType") ?: "speaker" - val outputType = when (outputTypeStr.lowercase()) { - "speaker" -> AzureTtsHelper.AudioOutputType.SPEAKER - "earpiece" -> AzureTtsHelper.AudioOutputType.EARPIECE - "auto" -> AzureTtsHelper.AudioOutputType.AUTO - else -> AzureTtsHelper.AudioOutputType.SPEAKER - } - result.success(azureTtsHelper.setAudioOutputType(outputType)) - } - "speakText" -> { - val text = call.argument("text") ?: return@setMethodCallHandler result.error("INVALID_ARGUMENTS", "文本不能为空", null) - - azureTtsHelper.speakText(text, object : AzureTtsHelper.TTSCallback { - override fun onSuccess(message: String) { - runOnUiThread { result.success(message) } - } - - override fun onError(error: String) { - runOnUiThread { result.error("SPEAK_ERROR", error, null) } - } - }) - } - "speakSsml" -> { - val ssml = call.argument("ssml") ?: return@setMethodCallHandler result.error("INVALID_ARGUMENTS", "SSML不能为空", null) - - azureTtsHelper.speakSsml(ssml, object : AzureTtsHelper.TTSCallback { - override fun onSuccess(message: String) { - runOnUiThread { result.success(message) } - } - - override fun onError(error: String) { - runOnUiThread { result.error("SPEAK_ERROR", error, null) } - } - }) - } - "stopSpeaking" -> { - result.success(azureTtsHelper.stopSpeaking()) - } - "isSpeaking" -> { - result.success(azureTtsHelper.isSpeaking()) - } - "dispose" -> { - azureTtsHelper.dispose() - result.success(true) - } - else -> { - result.notImplemented() - } - } - } - // 设置语音交互方法通道 MethodChannel(flutterEngine.dartExecutor.binaryMessenger, VOICE_INTERACTION_CHANNEL).setMethodCallHandler { call, result -> when (call.method) { @@ -657,15 +449,6 @@ class MainActivity: FlutterActivity() { } } - // ASR 事件发送方法 - private fun sendAsrEvent(event: Map) { - if (azureAsrEventSink == null) return - - runOnUiThread { - azureAsrEventSink?.success(event) - } - } - /** * 启动语音交互服务（带配置参数） */ @@ -675,12 +458,16 @@ class MainActivity: FlutterActivity() { // 获取配置参数 val key = call.argument("azure_speech_key") ?: "" val region = call.argument("azure_speech_region") ?: "" - val aiKey = call.argument("volcano_ai_api_key") ?: "" + val openaiKey = call.argument("openai_api_key") ?: "" + val baseUrl = call.argument("openai_base_url") - // 设置 Azure Speech 配置 + // 设置 Azure Speech 和 AI 配置 azureSpeechKey = key azureSpeechRegion = region - volcanoAiApiKey = aiKey + openaiApiKey = openaiKey + if (baseUrl != null) { + openaiBaseUrl = baseUrl + } // 保存密钥到安全存储 saveKeysToSecureStorage(applicationContext) @@ -799,8 +586,6 @@ class MainActivity: FlutterActivity() { } // 释放资源 - azureAsrHelper.dispose() - azureTtsHelper.dispose() classicBluetoothHelper.dispose() super.onDestroy() diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/OpenAIService.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/OpenAIService.kt new file mode 100644 index 000000000..39907e6d5 --- /dev/null +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/OpenAIService.kt @@ -0,0 +1,606 @@ +package com.yunqiinnovation.deepsound + +import android.util.Log +import okhttp3.* +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONArray +import org.json.JSONObject +import java.io.IOException +import java.util.concurrent.TimeUnit +import com.yunqiinnovation.deepsound.core.utils.FileLogger + +/** + * OpenAI服务的原生实现 + */ +class OpenAIService() { + private val TAG = "OpenAIService" + private var baseUrl = "https://api.openai.com/v1/chat/completions" + private val client = OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .writeTimeout(30, TimeUnit.SECONDS) + .build() + + private var apiKey: String = "" + private var isInitialized = false + private var model: String = "doubao-1-5-lite-32k-250115" // 默认模型 + + // 用于存储注册的函数 + private val registeredFunctions = mutableListOf() + + + /** + * 初始化OpenAI服务 + */ + fun initialize(apiKey: String, baseUrl: String = ""): Boolean { + this.apiKey = apiKey + if (baseUrl.isNotEmpty()) { + this.baseUrl = baseUrl + } + isInitialized = apiKey.isNotEmpty() + return isInitialized + } + + /** + * 注册函数 + */ + fun registerFunction(name: String, description: String, parameters: JSONObject): Boolean { + try { + val function = JSONObject().apply { + put("name", name) + put("description", description) + put("parameters", parameters) + } + + // 检查是否已存在相同名称的函数 + val existingIndex = registeredFunctions.indexOfFirst { + it.getString("name") == name + } + + if (existingIndex >= 0) { + // 如果已存在，则替换 + registeredFunctions[existingIndex] = function + } else { + // 如果不存在，则添加 + registeredFunctions.add(function) + } + + return true + } catch (e: Exception) { + return false + } + } + + /** + * 发送消息（非流式输出） + */ + @Throws(OpenAIException::class) + fun sendMessage(messages: JSONArray, systemPrompt: String): String { + if (!isInitialized || apiKey.isEmpty()) { + throw OpenAIException("OpenAI服务未初始化") + } + + val fullMessages = JSONArray().apply { + put(JSONObject().apply { + put("role", "system") + put("content", systemPrompt) + }) + for (i in 0 until messages.length()) { + put(messages.getJSONObject(i)) + } + } + + val requestBody = JSONObject().apply { + put("model", model) + put("messages", fullMessages) + put("temperature", 0.7) + put("max_tokens", 2000) + put("stream", false) + + // 如果有注册的函数，则添加到请求中 + if (registeredFunctions.isNotEmpty()) { + val tools = JSONArray() + for (function in registeredFunctions) { + val tool = JSONObject().apply { + put("type", "function") + put("function", function) + } + tools.put(tool) + } + put("tools", tools) + } + } + + val mediaType = "application/json".toMediaTypeOrNull() + val request = Request.Builder() + .url(baseUrl) + .addHeader("Content-Type", "application/json") + .addHeader("Authorization", "Bearer $apiKey") + .post(requestBody.toString().toRequestBody(mediaType)) + .build() + + try { + client.newCall(request).execute().use { response -> + if (!response.isSuccessful) { + throw OpenAIException("API调用失败: ${response.code}") + } + + val responseBody = response.body?.string() ?: throw OpenAIException("Empty response") + val jsonResponse = JSONObject(responseBody) + + // 检查是否有函数调用 + if (jsonResponse.has("choices") && + jsonResponse.getJSONArray("choices").length() > 0) { + + val choice = jsonResponse.getJSONArray("choices").getJSONObject(0) + + // 检查是否是函数调用 + if (choice.has("message")) { + val message = choice.getJSONObject("message") + + // 检查是否有工具调用 + if (message.has("tool_calls")) { + val toolCalls = message.getJSONArray("tool_calls") + if (toolCalls.length() > 0) { + val toolCall = toolCalls.getJSONObject(0) + if (toolCall.has("function")) { + val function = toolCall.getJSONObject("function") + val functionCall = JSONObject().apply { + put("name", function.getString("name")) + put("arguments", function.getString("arguments")) + put("id", toolCall.getString("id")) + } + return functionCall.toString() + } + } + } + + // 如果没有工具调用，返回消息内容 + if (message.has("content")) { + return message.getString("content") + } + } + } + + throw OpenAIException("Invalid response format") + } + } catch (e: Exception) { + if (e is OpenAIException) throw e + throw OpenAIException("Failed to communicate with AI service: ${e.message}") + } + } + + /** + * 发送消息（流式输出） + */ + fun sendMessageStream(messages: JSONArray, systemPrompt: String, callback: StreamCallback) { + if (!isInitialized || apiKey.isEmpty()) { + callback.onError(OpenAIException("OpenAI服务未初始化")) + return + } + + val fullMessages = JSONArray().apply { + put(JSONObject().apply { + put("role", "system") + put("content", systemPrompt) + }) + for (i in 0 until messages.length()) { + put(messages.getJSONObject(i)) + } + } + + val requestBody = JSONObject().apply { + put("model", model) + put("messages", fullMessages) + put("temperature", 0.7) + put("max_tokens", 2000) + put("stream", true) + + // 如果有注册的函数，则添加到请求中 + if (registeredFunctions.isNotEmpty()) { + val tools = JSONArray() + for (function in registeredFunctions) { + val tool = JSONObject().apply { + put("type", "function") + put("function", function) + } + tools.put(tool) + } + put("tools", tools) + } + } + + val mediaType = "application/json".toMediaTypeOrNull() + val request = Request.Builder() + .url(baseUrl) + .addHeader("Content-Type", "application/json") + .addHeader("Authorization", "Bearer $apiKey") + .addHeader("Accept", "text/event-stream") + .post(requestBody.toString().toRequestBody(mediaType)) + .build() + + client.newCall(request).enqueue(object : Callback { + override fun onFailure(call: Call, e: IOException) { + callback.onError(OpenAIException(e.message ?: "请求失败")) + } + + override fun onResponse(call: Call, response: Response) { + if (!response.isSuccessful) { + callback.onError(OpenAIException("API调用失败: ${response.code}")) + return + } + + val responseBody = response.body ?: return + val source = responseBody.source() + + try { + // 预取数据到缓冲区 + source.request(Long.MAX_VALUE) + val bufferedSource = source.buffer + + // 用于存储函数调用的各个部分 + val finalToolCalls = mutableMapOf() + + while (!bufferedSource.exhausted()) { + val line = bufferedSource.readUtf8Line() ?: continue + + if (line.isEmpty()) continue + if (line.startsWith("data: ")) { + val data = line.substring(6) + if (data == "[DONE]") { + callback.onComplete() + break + } + + try { + val jsonData = JSONObject(data) + if (jsonData.has("choices") && + jsonData.getJSONArray("choices").length() > 0) { + + val choice = jsonData.getJSONArray("choices").getJSONObject(0) + + // 检查是否有delta + if (choice.has("delta")) { + val delta = choice.getJSONObject("delta") + + // 检查是否有工具调用 + if (delta.has("tool_calls")) { + val toolCalls = delta.getJSONArray("tool_calls") + for (i in 0 until toolCalls.length()) { + val toolCall = toolCalls.getJSONObject(i) + val index = toolCall.optInt("index", i) + + // 如果是新的工具调用，初始化 + if (!finalToolCalls.containsKey(index)) { + finalToolCalls[index] = ToolCallInfo() + } + + // 获取ID + if (toolCall.has("id")) { + finalToolCalls[index]?.id = toolCall.getString("id") + } + + // 处理函数信息 + if (toolCall.has("function")) { + val function = toolCall.getJSONObject("function") + + if (function.has("name")) { + finalToolCalls[index]?.name = function.getString("name") + } + + if (function.has("arguments")) { + finalToolCalls[index]?.arguments += function.getString("arguments") + } + } + } + continue + } + + // 如果有内容，发送给回调 + if (delta.has("content") && !delta.isNull("content")) { + val content = delta.getString("content") + callback.onToken(content) + } + } + } + } catch (e: Exception) { + // 忽略无效的JSON + continue + } + } + } + + // 处理完整的函数调用 + for ((_, toolCallInfo) in finalToolCalls) { + if (toolCallInfo.name.isNotEmpty()) { + try { + // 创建函数调用对象 + val functionCall = JSONObject().apply { + put("id", toolCallInfo.id) + put("name", toolCallInfo.name) + put("arguments", toolCallInfo.arguments.trim()) + } + + callback.onFunctionCall(functionCall) + } catch (e: Exception) { + // 出错时使用空参数 + val functionCall = JSONObject().apply { + put("id", toolCallInfo.id) + put("name", toolCallInfo.name) + put("arguments", "{}") + } + callback.onFunctionCall(functionCall) + } + } + } + } catch (e: Exception) { + callback.onError(OpenAIException("处理流式响应出错: ${e.message}")) + } finally { + response.close() + } + } + }) + } + + /** + * 发送函数调用结果 + */ + fun sendFunctionCallResult( + messages: JSONArray, + systemPrompt: String, + functionCall: JSONObject, + functionResult: String, + callback: StreamCallback + ) { + if (!isInitialized || apiKey.isEmpty()) { + callback.onError(OpenAIException("OpenAI服务未初始化")) + return + } + + // 构建完整的消息历史 + val fullMessages = JSONArray().apply { + // 添加系统提示 + put(JSONObject().apply { + put("role", "system") + put("content", systemPrompt) + }) + + // 添加历史消息 + for (i in 0 until messages.length()) { + put(messages.getJSONObject(i)) + } + + // 添加函数调用信息 + put(JSONObject().apply { + put("role", "assistant") + put("content", null) + put("tool_calls", JSONArray().apply { + put(JSONObject().apply { + put("id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) + put("type", "function") + put("function", JSONObject().apply { + put("name", functionCall.getString("name")) + put("arguments", functionCall.getString("arguments")) + }) + }) + }) + }) + + // 添加函数返回结果 + put(JSONObject().apply { + put("role", "tool") + put("tool_call_id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) + put("content", functionResult) + }) + } + + // 构建请求 + val requestBody = JSONObject().apply { + put("model", model) + put("messages", fullMessages) + put("temperature", 0.7) + put("max_tokens", 2000) + put("stream", true) + + // 如果有注册的函数，则添加到请求中 + if (registeredFunctions.isNotEmpty()) { + val tools = JSONArray() + for (function in registeredFunctions) { + val tool = JSONObject().apply { + put("type", "function") + put("function", function) + } + tools.put(tool) + } + put("tools", tools) + } + } + + val mediaType = "application/json".toMediaTypeOrNull() + val request = Request.Builder() + .url(baseUrl) + .addHeader("Content-Type", "application/json") + .addHeader("Authorization", "Bearer $apiKey") + .addHeader("Accept", "text/event-stream") + .post(requestBody.toString().toRequestBody(mediaType)) + .build() + + // 发送请求 + client.newCall(request).enqueue(object : Callback { + override fun onFailure(call: Call, e: IOException) { + callback.onError(OpenAIException(e.message ?: "请求失败")) + } + + override fun onResponse(call: Call, response: Response) { + if (!response.isSuccessful) { + callback.onError(OpenAIException("API调用失败: ${response.code}")) + return + } + + val responseBody = response.body ?: return + val source = responseBody.source() + + try { + // 预取数据到缓冲区 + source.request(Long.MAX_VALUE) + val bufferedSource = source.buffer + + // 用于存储函数调用的各个部分 + val finalToolCalls = mutableMapOf() + + while (!bufferedSource.exhausted()) { + val line = bufferedSource.readUtf8Line() ?: continue + + if (line.isEmpty()) continue + if (line.startsWith("data: ")) { + val data = line.substring(6) + if (data == "[DONE]") { + callback.onComplete() + break + } + + try { + val jsonData = JSONObject(data) + if (jsonData.has("choices") && + jsonData.getJSONArray("choices").length() > 0) { + + val choice = jsonData.getJSONArray("choices").getJSONObject(0) + + // 检查是否有delta + if (choice.has("delta")) { + val delta = choice.getJSONObject("delta") + + // 检查是否有工具调用 + if (delta.has("tool_calls")) { + val toolCalls = delta.getJSONArray("tool_calls") + for (i in 0 until toolCalls.length()) { + val toolCall = toolCalls.getJSONObject(i) + val index = toolCall.optInt("index", i) + + // 如果是新的工具调用，初始化 + if (!finalToolCalls.containsKey(index)) { + finalToolCalls[index] = ToolCallInfo() + } + + // 获取ID + if (toolCall.has("id")) { + finalToolCalls[index]?.id = toolCall.getString("id") + } + + // 处理函数信息 + if (toolCall.has("function")) { + val function = toolCall.getJSONObject("function") + + if (function.has("name")) { + finalToolCalls[index]?.name = function.getString("name") + } + + if (function.has("arguments")) { + finalToolCalls[index]?.arguments += function.getString("arguments") + } + } + } + continue + } + + // 如果有内容，发送给回调 + if (delta.has("content") && !delta.isNull("content")) { + val content = delta.getString("content") + callback.onToken(content) + } + } + } + } catch (e: Exception) { + // 忽略无效的JSON + continue + } + } + } + + // 处理完整的函数调用 + for ((_, toolCallInfo) in finalToolCalls) { + if (toolCallInfo.name.isNotEmpty()) { + try { + // 创建函数调用对象 + val functionCall = JSONObject().apply { + put("id", toolCallInfo.id) + put("name", toolCallInfo.name) + put("arguments", toolCallInfo.arguments.trim()) + } + + callback.onFunctionCall(functionCall) + } catch (e: Exception) { + // 出错时使用空参数 + val functionCall = JSONObject().apply { + put("id", toolCallInfo.id) + put("name", toolCallInfo.name) + put("arguments", "{}") + } + callback.onFunctionCall(functionCall) + } + } + } + } catch (e: Exception) { + callback.onError(OpenAIException("处理流式响应出错: ${e.message}")) + } finally { + response.close() + } + } + }) + } + + /** + * 创建用户消息 + */ + fun createUserMessage(content: String): JSONObject { + return JSONObject().apply { + put("role", "user") + put("content", content) + } + } + + /** + * 创建系统消息 + */ + fun createSystemMessage(content: String): JSONObject { + return JSONObject().apply { + put("role", "system") + put("content", content) + } + } + + /** + * 创建助手消息 + */ + fun createAssistantMessage(content: String): JSONObject { + return JSONObject().apply { + put("role", "assistant") + put("content", content) + } + } + + /** + * 流式输出回调接口 + */ + interface StreamCallback { + fun onToken(token: String) + fun onComplete() + fun onError(e: Exception) + fun onFunctionCall(functionCall: JSONObject) {} + } + + /** + * 用于存储工具调用信息的辅助类 + */ + private class ToolCallInfo { + var id: String = "" + var name: String = "" + var arguments: String = "" + } +} + +/** + * OpenAI异常 + */ +class OpenAIException(message: String) : Exception(message) \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt new file mode 100644 index 000000000..a89441506 --- /dev/null +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt @@ -0,0 +1,180 @@ +package com.yunqiinnovation.deepsound + +import org.json.JSONArray +import org.json.JSONObject +import com.yunqiinnovation.deepsound.OpenAIService +import com.yunqiinnovation.deepsound.core.utils.FileLogger + +/** + * 语音功能处理器 - 处理AI函数调用 + */ +class VoiceFunctionHandler( + private val openAIService: OpenAIService, + private val systemPrompt: String +) { + companion object { + private const val TAG = "VoiceFunctionHandler" + } + + /** + * 初始化并注册所有可用的函数 + */ + fun initialize() { + try { + // 注册退出交互函数 + registerExitInteractionFunction() + + // 在这里可以注册更多函数 + + } catch (e: Exception) { + FileLogger.e(TAG, "初始化函数处理器失败: ${e.message}", e) + } + } + + /** + * 注册退出交互函数 + */ + private fun registerExitInteractionFunction() { + try { + openAIService.registerFunction( + "exit_interaction", + "退出当前语音交互", + JSONObject(""" + { + "type": "object", + "properties": {}, + "required": [] + } + """) + ) + FileLogger.d(TAG, "退出交互功能已注册") + } catch (e: Exception) { + FileLogger.e(TAG, "注册退出交互函数失败: ${e.message}", e) + } + } + + /** + * 处理函数调用 + * + * @param functionCall 函数调用信息 + * @param messages 消息历史 + * @param callback 回调，处理退出等操作 + * @return 是否已处理函数调用 + */ + fun handleFunctionCall( + functionCall: JSONObject, + messages: JSONArray, + callback: FunctionCallCallback + ): Boolean { + val functionName = functionCall.getString("name") + FileLogger.d(TAG, "处理函数调用: $functionName") + + return when (functionName) { + "exit_interaction" -> { + handleExitInteraction(functionCall, messages, callback) + true + } + else -> { + // 未知函数，返回默认结果 + handleUnknownFunction(functionCall, messages, callback) + false + } + } + } + + /** + * 处理退出交互函数 + */ + private fun handleExitInteraction( + functionCall: JSONObject, + messages: JSONArray, + callback: FunctionCallCallback + ) { + FileLogger.d(TAG, "处理退出交互函数") + + val responseBuilder = StringBuilder() + + openAIService.sendFunctionCallResult( + messages = messages, + systemPrompt = systemPrompt, + functionCall = functionCall, + functionResult = "{\"result\": \"已退出语音交互\"}", + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + responseBuilder.append(token) + } + + override fun onComplete() { + FileLogger.d(TAG, "handleExitInteraction onComplete: ${responseBuilder.toString()}") + val response = responseBuilder.toString() + if (response.isNotEmpty()) { + callback.onExitWithMessage(response) + } else { + callback.onExitWithMessage("已退出语音交互") + } + } + + override fun onError(e: Exception) { + FileLogger.e(TAG, "处理退出交互函数调用出错: ${e.message}") + callback.onError("退出交互时出错") + } + + override fun onFunctionCall(nestedCall: JSONObject) { + FileLogger.e(TAG, "意外收到嵌套函数调用: ${nestedCall.getString("name")}") + } + } + ) + } + + /** + * 处理未知函数调用 + */ + private fun handleUnknownFunction( + functionCall: JSONObject, + messages: JSONArray, + callback: FunctionCallCallback + ) { + FileLogger.d(TAG, "处理未知函数: ${functionCall.getString("name")}") + + try { + openAIService.sendFunctionCallResult( + messages = messages, + systemPrompt = systemPrompt, + functionCall = functionCall, + functionResult = "{\"result\": \"处理函数调用中\"}", + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + callback.onTokenReceived(token) + } + + override fun onComplete() { + callback.onComplete() + } + + override fun onError(e: Exception) { + FileLogger.e(TAG, "处理函数调用失败: ${e.message}") + callback.onError("处理函数调用失败: ${e.message}") + } + + override fun onFunctionCall(nestedCall: JSONObject) { + callback.onFunctionCall(nestedCall) + } + } + ) + } catch (e: Exception) { + FileLogger.e(TAG, "处理函数调用失败: ${e.message}") + callback.onError("处理函数调用失败: ${e.message}") + } + } + + /** + * 函数调用回调接口 + */ + interface FunctionCallCallback { + fun onTokenReceived(token: String) + fun onComplete() + fun onError(message: String) + fun onFunctionCall(functionCall: JSONObject) + fun onExitWithMessage(farewell: String) + } +} \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/example/deep_voice/VoiceInteractionService.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt similarity index 78% rename from android/app/src/main/kotlin/com/example/deep_voice/VoiceInteractionService.kt rename to android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt index f469c136e..ad20ab6b7 100644 --- a/android/app/src/main/kotlin/com/example/deep_voice/VoiceInteractionService.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt @@ -22,10 +22,15 @@ import android.os.Handler import android.os.Looper import java.util.concurrent.atomic.AtomicBoolean import org.json.JSONArray +import org.json.JSONObject import android.media.MediaPlayer import android.media.AudioAttributes import android.net.Uri import com.yunqiinnovation.deepsound.core.utils.FileLogger +import com.yunqiinnovation.azure_speech.AzureAsrHelper +import com.yunqiinnovation.azure_speech.AzureTtsHelper +import com.yunqiinnovation.deepsound.OpenAIService + /** * 后台语音交互 Service： @@ -78,7 +83,8 @@ class VoiceInteractionService : Service() { private lateinit var audioManager: AudioManager private lateinit var azureAsrHelper: AzureAsrHelper private lateinit var azureTtsHelper: AzureTtsHelper - private lateinit var volcanoAIService: VolcanoAIService + private lateinit var openAIService: OpenAIService + private lateinit var functionHandler: VoiceFunctionHandler // 定时器 private val handler = Handler(Looper.getMainLooper()) @@ -95,7 +101,10 @@ class VoiceInteractionService : Service() { 请保持回答简短、准确，避免过长的解释。 如果用户的问题不清楚，请礼貌地请求澄清。 不要使用复杂的术语，除非用户明确要求。 - 用户用语音和你交互. + 用户用语音和你交互。 + + 当用户说"退出"、"再见"、"结束对话"等类似意图时，你应该使用exit_interaction函数来结束对话， + 并在结束前说一句友好的告别语，例如"再见，有需要随时找我"。 """.trimIndent() // 添加媒体播放器 @@ -157,10 +166,11 @@ class VoiceInteractionService : Service() { // 尝试从静态变量获取配置 var subscriptionKey = MainActivity.azureSpeechKey var serviceRegion = MainActivity.azureSpeechRegion - var volcanoKey = MainActivity.volcanoAiApiKey + var openaiKey = MainActivity.openaiApiKey + var openaiBaseUrl = MainActivity.openaiBaseUrl ?: "" // OpenAI API基本URL // 如果静态变量中没有配置，尝试从加密存储中加载 - if (subscriptionKey.isEmpty() || serviceRegion.isEmpty() || volcanoKey.isEmpty()) { + if (subscriptionKey.isEmpty() || serviceRegion.isEmpty() || openaiKey.isEmpty()) { FileLogger.d(TAG, "静态变量中的配置信息不完整，尝试从加密存储加载") // 从加密存储加载密钥 @@ -170,7 +180,8 @@ class VoiceInteractionService : Service() { // 更新本地变量 subscriptionKey = MainActivity.azureSpeechKey serviceRegion = MainActivity.azureSpeechRegion - volcanoKey = MainActivity.volcanoAiApiKey + openaiKey = MainActivity.openaiApiKey + openaiBaseUrl = MainActivity.openaiBaseUrl ?: "" FileLogger.d(TAG, "已从加密存储加载配置信息") } else { @@ -191,15 +202,29 @@ class VoiceInteractionService : Service() { FileLogger.e(TAG, "Azure配置信息不完整，无法初始化Azure服务") } - // 初始化火山AI服务 - volcanoAIService = VolcanoAIService() + // 初始化OpenAI服务 + openAIService = OpenAIService() - // 初始化火山AI服务 - if (volcanoKey.isNotEmpty()) { - volcanoAIService.initialize(volcanoKey) - FileLogger.d(TAG, "火山AI服务已初始化") + // 初始化OpenAI服务 + if (openaiKey.isNotEmpty()) { + val initialized = if (openaiBaseUrl.isNotEmpty()) { + openAIService.initialize(openaiKey, openaiBaseUrl) + } else { + openAIService.initialize(openaiKey) + } + + if (initialized) { + FileLogger.d(TAG, "OpenAI服务已初始化") + + // 初始化函数处理器 + functionHandler = VoiceFunctionHandler(openAIService, systemPrompt) + functionHandler.initialize() + + } else { + FileLogger.e(TAG, "OpenAI服务初始化失败") + } } else { - FileLogger.e(TAG, "火山AI配置信息不完整，无法初始化火山AI服务") + FileLogger.e(TAG, "OpenAI配置信息不完整，无法初始化OpenAI服务") } } @@ -404,7 +429,7 @@ class VoiceInteractionService : Service() { override fun onResult(result: String, detectedLanguage: String) { if (result.isNotEmpty()) { - processWithVolcanoAI(result) + processWithOpenAI(result) } // 重置状态，继续识别 @@ -483,27 +508,114 @@ class VoiceInteractionService : Service() { } /** - * 使用VolcanoAI处理语音识别结果 + * 使用OpenAI处理语音识别结果 */ - private fun processWithVolcanoAI(text: String) { + private fun processWithOpenAI(text: String) { // 保存当前用户输入，用于后续同步聊天记录 currentUserInput = text Thread { try { val messages = JSONArray().apply { - put(volcanoAIService.createUserMessage(text)) + put(openAIService.createUserMessage(text)) } + + // 创建响应构建器 + val responseBuilder = StringBuilder() - val response = volcanoAIService.sendMessage(messages, systemPrompt) - - // 播放AI回复 - speakAIResponse(response) + openAIService.sendMessageStream( + messages = messages, + systemPrompt = systemPrompt, + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + // 累加响应内容 + responseBuilder.append(token) + } + + override fun onComplete() { + // 处理完整响应 + val response = responseBuilder.toString() + if (response.isNotEmpty()) { + // 播放AI回复 + Log.d(TAG, "AI 回复: $response") + + speakAIResponse(response) + + // 同步聊天记录到Flutter端 + notifyChatHistoryUpdated("personal_assistant", text, response) + } + } + + override fun onError(e: Exception) { + FileLogger.e(TAG, "AI流式处理出错: ${e.message}", e) + playNotification("AI处理出错") + } + + override fun onFunctionCall(call: JSONObject) { + FileLogger.d(TAG, "收到函数调用请求: ${call.getString("name")}") + + // 使用函数处理器处理函数调用 + val handled = functionHandler.handleFunctionCall( + functionCall = call, + messages = messages, + callback = object : VoiceFunctionHandler.FunctionCallCallback { + override fun onTokenReceived(token: String) { + responseBuilder.append(token) + } + + override fun onComplete() { + val response = responseBuilder.toString() + if (response.isNotEmpty()) { + // 播放AI回复 + Log.d(TAG, "AI Function Call 回复: $response") + speakAIResponse(response) + + // 同步聊天记录到Flutter端 + notifyChatHistoryUpdated("personal_assistant", text, response) + } + updateLastActivityTime() + } + + override fun onError(message: String) { + FileLogger.e(TAG, "函数处理出错: $message") + playNotification(message) + } + + override fun onFunctionCall(nestedCall: JSONObject) { + FileLogger.d(TAG, "收到嵌套函数调用: ${nestedCall.getString("name")}") + // 递归处理嵌套函数调用 + functionHandler.handleFunctionCall( + functionCall = nestedCall, + messages = messages, + callback = this + ) + } + + override fun onExitWithMessage(farewell: String) { + // 播放退出消息 + speakAIResponse(farewell) + + // 同步聊天记录 + notifyChatHistoryUpdated("personal_assistant", text, farewell) + + // 停止语音识别 + stopVoiceRecognition() + } + } + ) + + if (!handled) { + // 如果函数没有被处理，作为普通文本处理 + FileLogger.d(TAG, "函数未处理，作为普通文本处理") + speakAIResponse("我无法处理这个请求") + notifyChatHistoryUpdated("personal_assistant", text, "我无法处理这个请求") + } + } + } + ) - // 同步聊天记录到Flutter端 - notifyChatHistoryUpdated("personal_assistant", text, response) } catch (e: Exception) { - FileLogger.e(TAG, "AI处理出错: ${e.message}") + FileLogger.e(TAG, "AI处理出错: ${e.message}", e) playNotification("AI处理出错") } }.start() @@ -746,38 +858,45 @@ class VoiceInteractionService : Service() { override fun onBind(intent: Intent?): IBinder? = null override fun onDestroy() { - FileLogger.d(TAG, "onDestroy - 语音交互服务正在销毁") - - // 释放音频播放器 - audioPlayer?.release() - audioPlayer = null + super.onDestroy() + FileLogger.d(TAG, "onDestroy - 语音交互服务即将销毁") - // 停止监控 + // 停止服务监控 stopMonitoring() // 停止语音识别 - if (isRecognitionActive) { - stopVoiceRecognition() - } + stopVoiceRecognition() + + // 停止媒体会话 + mediaSession.release() + FileLogger.d(TAG, "媒体会话已释放") // 停止TTS stopCurrentTTS() - // 释放 MediaSession - mediaSession.release() + // 关闭Azure语音服务 + if (::azureAsrHelper.isInitialized) { + FileLogger.d(TAG, "关闭Azure语音服务") + azureAsrHelper.dispose() + } - // 释放Azure服务实例 - FileLogger.d(TAG, "释放Azure服务实例") - azureAsrHelper.dispose() - azureTtsHelper.dispose() + if (::azureTtsHelper.isInitialized) { + FileLogger.d(TAG, "关闭Azure TTS服务") + azureTtsHelper.dispose() + } - // 更新服务状态 - isRunning.set(false) + // 关闭OpenAI服务 + if (::openAIService.isInitialized) { + FileLogger.d(TAG, "关闭OpenAI服务") + } - // 关闭日志系统 - FileLogger.shutdown() + // 关闭音频播放器 + audioPlayer?.release() - super.onDestroy() + // 设置服务状态 + isRunning.set(false) + + FileLogger.d(TAG, "语音交互服务已销毁") } /** @@ -804,9 +923,7 @@ class VoiceInteractionService : Service() { * 通知 Flutter 端聊天记录已更新 */ private fun notifyChatHistoryUpdated(agentId: String, userMessage: String, assistantMessage: String) { - FileLogger.d(TAG, "通知Flutter聊天记录已更新: agentId=$agentId") - FileLogger.d(TAG, "用户消息: ${userMessage.take(50)}...") - FileLogger.d(TAG, "助手回复: ${assistantMessage.take(50)}...") + // 创建广播 Intent val intent = Intent(ACTION_CHAT_HISTORY_UPDATED).apply { @@ -818,7 +935,6 @@ class VoiceInteractionService : Service() { // 发送广播 sendBroadcast(intent) - FileLogger.d(TAG, "已发送聊天记录广播") } /** diff --git a/android/app/src/main/kotlin/com/example/deep_voice/core/utils/FileLogger.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/core/utils/FileLogger.kt similarity index 100% rename from android/app/src/main/kotlin/com/example/deep_voice/core/utils/FileLogger.kt rename to android/app/src/main/kotlin/com/yunqiinnovation/deepsound/core/utils/FileLogger.kt diff --git a/android/settings.gradle.kts b/android/settings.gradle.kts index 6296e34ed..6a3483c9c 100644 --- a/android/settings.gradle.kts +++ b/android/settings.gradle.kts @@ -29,4 +29,8 @@ plugins { } include(":app") +include(":azure_speech") + +// 设置azure_speech项目的路径 +project(":azure_speech").projectDir = file("../local_plugins/azure_speech/android") diff --git a/azure/LICENSE b/azure/LICENSE deleted file mode 100644 index d3df29b8a..000000000 --- a/azure/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2024 Your Company - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/azure/README.md b/azure/README.md deleted file mode 100644 index a19025680..000000000 --- a/azure/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Azure Speech Recognition - -A Flutter plugin for Microsoft Azure Speech services, providing both speech recognition (ASR) and text-to-speech (TTS) capabilities. - -## Features - -- Speech-to-text (Azure Speech Recognition) -- Text-to-speech (Azure Speech Synthesis) -- Support for multiple languages -- Language detection -- Continuous recognition -- Streaming synthesis - -## Getting Started - -### Prerequisites - -- Azure Speech service subscription key -- Azure Speech service region - -### Installation - -Add this to your package's `pubspec.yaml` file: - -```yaml -dependencies: - azure_speech_recognition: - path: ./azure -``` - -### Usage - -```dart -import 'package:azure_speech_recognition/azure_speech_recognition.dart'; - -// Initialize the service -await AzureSpeechRecognition.initialize( - subscriptionKey: 'your_subscription_key', - region: 'your_region', - supportedLanguages: ['zh-CN', 'en-US'], -); - -// Start continuous recognition -await AzureSpeechRecognition.startContinuousRecognition(); - -// Listen for recognition events -AzureSpeechRecognition.onRecognitionEvent.listen((event) { - if (event['type'] == 'result') { - print('Recognized: ${event['text']}'); - print('Detected language: ${event['detectedLanguage']}'); - } -}); - -// Stop recognition when done -await AzureSpeechRecognition.stopContinuousRecognition(); - -// Speak text -await AzureSpeechRecognition.speakText('Hello, world!'); - -// Clean up -await AzureSpeechRecognition.dispose(); -``` - -## License - -This project is licensed under the MIT License - see the LICENSE file for details. \ No newline at end of file diff --git a/azure/ios/Classes/AzureAsrHelper.swift b/azure/ios/Classes/AzureAsrHelper.swift deleted file mode 100644 index 58b142f60..000000000 --- a/azure/ios/Classes/AzureAsrHelper.swift +++ /dev/null @@ -1,757 +0,0 @@ -import Foundation -import MicrosoftCognitiveServicesSpeech -import AVFoundation -import AudioToolbox - -/// Azure ASR工具类，负责实现语音识别服务接口 -@available(iOS 13.0, *) -class AzureAsrHelper: NSObject { - // MARK: - 属性 - - /// 事件处理回调 - private var eventHandler: (String, [String: Any]) -> Void - - /// 语音配置信息 - private var speechSubscriptionKey: String = "" - private var serviceRegion: String = "" - - /// 语音识别相关 - private var speechConfig: SPXSpeechConfiguration? - private var recognizer: SPXSpeechRecognizer? - private var audioConfig: SPXAudioConfiguration? - private var pushStream: SPXPushAudioInputStream? - - /// 音频处理相关 - private var audioProcessor: CustomAudioProcessor? - private var isProcessingAudio = false - private var audioProcessingTimer: Timer? - - /// 状态标志 - private var isInitialized = false - private var _isContinuousRecognitionActive = false - - /// 当前语言和支持的语言 - private var currentLanguage = "zh-CN" - private var supportedLanguages: [String] = ["zh-CN", "en-US"] - private var isAutoDetectLanguage = false - - // MARK: - 初始化 - - init(eventHandler: @escaping (String, [String: Any]) -> Void) { - self.eventHandler = eventHandler - super.init() - } - - deinit { - dispose() - } - - // MARK: - ASR Service 接口实现 - - /// 初始化语音识别服务 - /// - Parameters: - /// - speechSubscriptionKey: Azure 语音服务订阅密钥 - /// - serviceRegion: Azure 服务区域 (如 eastasia) - /// - supportedLanguages: 支持的语言代码数组 (可选) - /// - Returns: 初始化是否成功 - func initialize(speechSubscriptionKey: String, serviceRegion: String, supportedLanguages: [String]? = nil) -> Bool { - print("[AzureAsrHelper] 初始化 Azure 语音服务") - - // 检查配置是否为空 - if speechSubscriptionKey.isEmpty || serviceRegion.isEmpty { - print("[AzureAsrHelper] 错误: Azure 配置信息不完整") - eventHandler("error", ["message": "Azure 配置信息不完整"]) - return false - } - - // 释放之前的资源 - dispose() - - // 记录配置信息 - self.speechSubscriptionKey = speechSubscriptionKey - self.serviceRegion = serviceRegion - - // 设置语言 - if let languages = supportedLanguages, !languages.isEmpty { - self.supportedLanguages = languages - } - - // 根据支持的语言数量决定是否启用自动语言检测 - isAutoDetectLanguage = self.supportedLanguages.count >= 2 - - // 如果只有一种语言，设置为当前语言 - if !isAutoDetectLanguage && !self.supportedLanguages.isEmpty { - currentLanguage = self.supportedLanguages[0] - } - - // 创建识别器和设置回调 - if !createRecognizerAndSetupCallbacks() { - return false - } - - print("[AzureAsrHelper] Azure 语音服务初始化成功") - isInitialized = true - return true - } - - /// 创建识别器并设置回调 - private func createRecognizerAndSetupCallbacks() -> Bool { - // 释放之前的 recognizer - recognizer = nil - audioConfig = nil - - do { - // 创建语音配置 - speechConfig = try SPXSpeechConfiguration(subscription: speechSubscriptionKey, region: serviceRegion) - - // 设置音频输入参数 - try setupAudioSession() - - // 创建自定义推送流，替代默认的麦克风输入 - pushStream = try SPXPushAudioInputStream() - audioConfig = try SPXAudioConfiguration(streamInput: pushStream!) - - // 初始化自定义音频处理器 - audioProcessor = CustomAudioProcessor() - - // 设置语言配置 - if isAutoDetectLanguage { - // 设置自动语言检测 - speechConfig?.setPropertyTo("Continuous", by: SPXPropertyId.speechServiceConnectionLanguageIdMode) - - // 创建自动语言检测配置 - let autoDetectSourceLanguageConfig = try SPXAutoDetectSourceLanguageConfiguration(supportedLanguages) - - // 创建识别器 - recognizer = try SPXSpeechRecognizer( - speechConfiguration: speechConfig!, - autoDetectSourceLanguageConfiguration: autoDetectSourceLanguageConfig, - audioConfiguration: audioConfig! - ) - } else { - // 设置指定的识别语言 - speechConfig?.speechRecognitionLanguage = currentLanguage - - // 创建识别器 - recognizer = try SPXSpeechRecognizer(speechConfiguration: speechConfig!, audioConfiguration: audioConfig!) - } - - // 设置所有回调 - setupAllCallbacks() - - return true - } catch { - print("[AzureAsrHelper] 错误: 创建识别器失败: \(error.localizedDescription)") - eventHandler("error", ["message": "创建识别器失败: \(error.localizedDescription)"]) - return false - } - } - - /// 设置音频会话 - private func setupAudioSession() throws { - let audioSession = AVAudioSession.sharedInstance() - - // 使用playAndRecord类别允许同时录音和播放 - try audioSession.setCategory(.playAndRecord, - mode: .voiceChat, // 使用voiceChat模式能够更好地支持回音消除 - options: [.allowBluetooth, .defaultToSpeaker, .allowAirPlay, .mixWithOthers]) - - // 设置首选的输入和输出 - let currentRoute = audioSession.currentRoute - - // 获取当前是否连接了耳机或外部麦克风 - let hasHeadphones = currentRoute.outputs.contains { - $0.portType == .headphones || $0.portType == .bluetoothA2DP || $0.portType == .bluetoothHFP - } - - // 如果没有耳机，明确启用内置麦克风和扬声器的回音消除 - if !hasHeadphones { - try audioSession.setMode(.voiceChat) // 语音聊天模式有更强的回音消除 - - // 启用回音消除和噪声抑制 - try audioSession.setInputGain(0.8) // 适当降低输入增益以减少扬声器音频被麦克风捕获的可能性 - } else { - // 耳机模式，可以使用不同的设置 - try audioSession.setMode(.voiceChat) - try audioSession.setInputGain(1.0) - } - - // 设置合适的采样率 - try audioSession.setPreferredSampleRate(16000.0) // Azure语音识别推荐的采样率 - try audioSession.setPreferredIOBufferDuration(0.01) // 较小的缓冲区大小以减少延迟 - - // 激活音频会话 - try audioSession.setActive(true, options: .notifyOthersOnDeactivation) - - print("[AzureAsrHelper] 音频会话配置成功，已启用回音消除") - } - - /// 设置所有回调 - private func setupAllCallbacks() { - guard let recognizer = recognizer else { return } - - // 最终识别结果 - recognizer.addRecognizedEventHandler { [weak self] _, event in - guard let self = self else { return } - - if event.result.reason == SPXResultReason.recognizedSpeech { - let detectedLanguage = self.getDetectedLanguage(from: event.result) - print("[AzureAsrHelper] 识别结果: \(event.result.text ?? ""), 语言: \(detectedLanguage)") - self.eventHandler("result", [ - "text": event.result.text ?? "", - "detectedLanguage": detectedLanguage - ]) - } - } - - // 识别中事件 - recognizer.addRecognizingEventHandler { [weak self] _, event in - guard let self = self else { return } - - if event.result.reason == SPXResultReason.recognizingSpeech { - let detectedLanguage = self.getDetectedLanguage(from: event.result) - // print("[AzureAsrHelper] 识别中: \(event.result.text ?? ""), 语言: \(detectedLanguage)") - self.eventHandler("recognizing", [ - "text": event.result.text ?? "", - "detectedLanguage": detectedLanguage - ]) - } - } - - // 会话事件 - recognizer.addSessionStartedEventHandler { [weak self] _, _ in - guard let self = self else { return } - - print("[AzureAsrHelper] 识别会话已开始") - self._isContinuousRecognitionActive = true - self.eventHandler("sessionStarted", [:]) - } - - recognizer.addSessionStoppedEventHandler { [weak self] _, _ in - guard let self = self else { return } - - print("[AzureAsrHelper] 识别会话已结束") - self._isContinuousRecognitionActive = false - self.eventHandler("sessionStopped", [:]) - } - - // 取消事件 - recognizer.addCanceledEventHandler { [weak self] _, event in - guard let self = self else { return } - - let reason = event.reason.rawValue - let errorDetails = event.errorDetails ?? "未知错误" - - print("[AzureAsrHelper] 识别取消: \(errorDetails)") - - self.eventHandler("canceled", [ - "reason": reason, - "errorDetails": errorDetails - ]) - - self._isContinuousRecognitionActive = false - } - } - - /// 执行一次性语音识别 - /// - Returns: 是否成功启动识别 - func recognizeOnce() -> Bool { - if !isInitialized { - print("[AzureAsrHelper] 错误: 语音服务未初始化") - eventHandler("error", ["message": "语音服务未初始化"]) - return false - } - - // 如果正在连续识别，先停止 - if _isContinuousRecognitionActive { - stopContinuousRecognition() - } - - // 确保识别器已创建 - if recognizer == nil && !createRecognizerAndSetupCallbacks() { - return false - } - - do { - // 启动音频处理 - startAudioProcessing() - - // 通知会话开始 - eventHandler("sessionStarted", [:]) - - // 执行识别 - try recognizer?.recognizeOnceAsync { [weak self] result in - guard let self = self else { return } - - // 停止音频处理 - self.stopAudioProcessing() - - if result.reason == SPXResultReason.recognizedSpeech { - let detectedLanguage = self.getDetectedLanguage(from: result) - self.eventHandler("result", [ - "text": result.text ?? "", - "detectedLanguage": detectedLanguage - ]) - } else if result.reason == SPXResultReason.noMatch { - print("[AzureAsrHelper] 无匹配结果") - self.eventHandler("noMatch", [:]) - } else if result.reason == SPXResultReason.canceled { - do { - let details = try SPXCancellationDetails(fromCanceledRecognitionResult: result) - let errorDetails = details.errorDetails ?? "未知错误" - self.eventHandler("error", ["message": "识别取消: \(errorDetails)"]) - } catch { - print("[AzureAsrHelper] 错误: 获取取消详情失败: \(error.localizedDescription)") - self.eventHandler("error", ["message": "识别取消，无法获取详细原因"]) - } - } - } - - return true - } catch { - print("[AzureAsrHelper] 错误: 识别异常: \(error.localizedDescription)") - eventHandler("error", ["message": "识别异常: \(error.localizedDescription)"]) - stopAudioProcessing() - return false - } - } - - /// 开始连续语音识别 - /// - Returns: 是否成功启动识别 - func startContinuousRecognition() -> Bool { - if !isInitialized { - print("[AzureAsrHelper] 错误: 语音服务未初始化") - eventHandler("error", ["message": "语音服务未初始化"]) - return false - } - - // 如果已经在进行连续识别，先停止 - if _isContinuousRecognitionActive { - stopContinuousRecognition() - } - - // 确保识别器已创建 - if recognizer == nil && !createRecognizerAndSetupCallbacks() { - return false - } - - // 重新确保音频设置正确 - do { - try setupAudioSession() - } catch { - print("[AzureAsrHelper] 警告: 设置音频会话失败: \(error.localizedDescription)") - } - - do { - // 启动音频处理 - startAudioProcessing() - - // 启动连续识别 - try recognizer?.startContinuousRecognition() - _isContinuousRecognitionActive = true - - print("[AzureAsrHelper] 连续识别开始") - return true - } catch { - print("[AzureAsrHelper] 错误: 开始连续识别失败: \(error.localizedDescription)") - eventHandler("error", ["message": "开始连续识别失败: \(error.localizedDescription)"]) - _isContinuousRecognitionActive = false - stopAudioProcessing() - return false - } - } - - /// 停止连续语音识别 - /// - Returns: 是否成功停止识别 - func stopContinuousRecognition() -> Bool { - // 停止音频处理 - stopAudioProcessing() - - if !_isContinuousRecognitionActive || recognizer == nil { - return true - } - - do { - try recognizer?.stopContinuousRecognition() - _isContinuousRecognitionActive = false - print("[AzureAsrHelper] 连续识别已停止") - return true - } catch { - print("[AzureAsrHelper] 错误: 停止连续识别失败: \(error.localizedDescription)") - eventHandler("error", ["message": "停止连续识别失败: \(error.localizedDescription)"]) - _isContinuousRecognitionActive = false - return false - } - } - - /// 检查连续识别是否活跃 - /// - Returns: 连续识别是否处于活跃状态 - func isContinuousRecognitionActive() -> Bool { - return _isContinuousRecognitionActive - } - - /// 释放资源 - func dispose() { - print("[AzureAsrHelper] 释放资源") - - // 停止音频处理 - stopAudioProcessing() - - // 停止连续识别 - if _isContinuousRecognitionActive { - stopContinuousRecognition() - } - - // 释放音频会话 - do { - try AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) - } catch { - print("[AzureAsrHelper] 警告: 释放音频会话失败: \(error.localizedDescription)") - } - - // 释放资源 - recognizer = nil - speechConfig = nil - audioConfig = nil - pushStream = nil - audioProcessor = nil - - // 重置状态 - _isContinuousRecognitionActive = false - isInitialized = false - } - - /// 从结果中获取检测到的语言 - private func getDetectedLanguage(from result: SPXSpeechRecognitionResult) -> String { - if isAutoDetectLanguage { - do { - let langResult = try SPXAutoDetectSourceLanguageResult(result) - return langResult.language ?? currentLanguage - } catch { - print("[AzureAsrHelper] 错误: 获取检测到的语言失败: \(error.localizedDescription)") - return currentLanguage - } - } else { - return currentLanguage - } - } - - // MARK: - 音频处理 - - /// 开始音频处理 - private func startAudioProcessing() { - guard !isProcessingAudio, let audioProcessor = audioProcessor else { return } - - isProcessingAudio = true - - // 启动音频处理器 - if !audioProcessor.startRecord() { - print("[AzureAsrHelper] 错误: 启动音频处理器失败") - eventHandler("error", ["message": "启动音频处理器失败"]) - return - } - - // 启动音频处理定时器 - audioProcessingTimer = Timer.scheduledTimer(withTimeInterval: 0.08, repeats: true) { [weak self] _ in - guard let self = self, self.isProcessingAudio, let processor = self.audioProcessor, let stream = self.pushStream else { - return - } - - // 读取处理后的音频数据 - var bytes = [UInt8](repeating: 0, count: 2560) - let bytesRead = processor.read(bytes: &bytes) - - if bytesRead > 0 { - // 推送数据到Azure语音服务 - let data = Data(bytes: bytes, count: bytesRead) - stream.write(data) - - // 通知音频数据可用 - self.eventHandler("audioData", ["data": bytes]) - } - } - - print("[AzureAsrHelper] 音频处理已启动") - } - - /// 停止音频处理 - private func stopAudioProcessing() { - // 停止定时器 - audioProcessingTimer?.invalidate() - audioProcessingTimer = nil - - // 停止音频处理器 - audioProcessor?.stopRecord() - - isProcessingAudio = false - print("[AzureAsrHelper] 音频处理已停止") - } -} - -// MARK: - 自定义音频处理器 - -@available(iOS 13.0, *) -class CustomAudioProcessor: NSObject { - // 音频单元 - private var ioUnit: AudioUnit? - - // 音频格式 - private var audioFormat: AudioStreamBasicDescription - - // 音频缓冲 - private var audioBufferList: AudioBufferList - private var audioList: [Float] = [] - private let audioListQueue = DispatchQueue(label: "audioListQueue") - - // 回音消除状态 - private var isEchoCancellationEnabled = true - - override init() { - // 设置音频格式 - 16kHz, 16位, 单声道 - audioFormat = AudioStreamBasicDescription( - mSampleRate: 16000.0, - mFormatID: kAudioFormatLinearPCM, - mFormatFlags: kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked, - mBytesPerPacket: 2, - mFramesPerPacket: 1, - mBytesPerFrame: 2, - mChannelsPerFrame: 1, - mBitsPerChannel: 16, - mReserved: 0 - ) - - // 初始化音频缓冲 - audioBufferList = AudioBufferList( - mNumberBuffers: 1, - mBuffers: AudioBuffer( - mNumberChannels: 1, - mDataByteSize: 4096, - mData: malloc(4096) - ) - ) - - super.init() - } - - deinit { - stopRecord() - free(audioBufferList.mBuffers.mData) - } - - /// 启动音频处理 - /// - Returns: 是否成功启动 - func startRecord() -> Bool { - print("[CustomAudioProcessor] 配置音频单元") - - // 创建音频组件描述 - 使用VoiceProcessingIO类型获取回音消除 - var ioUnitDescription = AudioComponentDescription( - componentType: kAudioUnitType_Output, - componentSubType: kAudioUnitSubType_VoiceProcessingIO, - componentManufacturer: kAudioUnitManufacturer_Apple, - componentFlags: 0, - componentFlagsMask: 0 - ) - - // 查找音频组件 - guard let ioUnitRef = AudioComponentFindNext(nil, &ioUnitDescription) else { - print("[CustomAudioProcessor] 错误: 未找到音频组件") - return false - } - - // 创建音频单元实例 - if checkError(AudioComponentInstanceNew(ioUnitRef, &ioUnit), "创建音频单元") { - ioUnit = nil - return false - } - - // 启用输入端口 - var enableInput: UInt32 = 1 - let kInputBus: AudioUnitElement = 1 - let kOutputBus: AudioUnitElement = 0 - if checkError(AudioUnitSetProperty(ioUnit!, kAudioOutputUnitProperty_EnableIO, - kAudioUnitScope_Input, kInputBus, &enableInput, - UInt32(MemoryLayout.size)), "启用输入端口") { - return false - } - - // 禁用输出端口 (我们只需要输入) - var enableOutput: UInt32 = 0 - if checkError(AudioUnitSetProperty(ioUnit!, kAudioOutputUnitProperty_EnableIO, - kAudioUnitScope_Output, kOutputBus, - &enableOutput, UInt32(MemoryLayout.size)), "禁用输出端口") { - return false - } - - // 设置缓冲区分配标志 - var flag: UInt32 = 0 - if checkError(AudioUnitSetProperty(ioUnit!, kAudioUnitProperty_ShouldAllocateBuffer, - kAudioUnitScope_Output, kInputBus, &flag, UInt32(MemoryLayout.size)), "设置缓冲区分配标志") { - return false - } - - // 设置音频格式 - let size = UInt32(MemoryLayout.size) - if checkError(AudioUnitSetProperty(ioUnit!, kAudioUnitProperty_StreamFormat, - kAudioUnitScope_Output, kInputBus, &audioFormat, size), "设置输入总线输出范围的流格式") { - return false - } - - if checkError(AudioUnitSetProperty(ioUnit!, kAudioUnitProperty_StreamFormat, - kAudioUnitScope_Input, kOutputBus, &audioFormat, size), "设置输出总线输入范围的流格式") { - return false - } - - // 启用回音消除 - if isEchoCancellationEnabled { - var echoCancellation: UInt32 = 1 - AudioUnitSetProperty(ioUnit!, kAUVoiceIOProperty_BypassVoiceProcessing, - kAudioUnitScope_Global, 0, &echoCancellation, UInt32(MemoryLayout.size)) - } - - // 设置输入回调 - 当有新音频数据时调用 - var inputCallback = AURenderCallbackStruct( - inputProc: CustomAudioProcessor.onAudioDataAvailable, - inputProcRefCon: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()) - ) - - if checkError(AudioUnitSetProperty(ioUnit!, - kAudioOutputUnitProperty_SetInputCallback, - kAudioUnitScope_Global, kInputBus, - &inputCallback, UInt32(MemoryLayout.size)), "设置输入回调") { - return false - } - - // 初始化音频单元 - var hasError = checkError(AudioUnitInitialize(ioUnit!), "初始化音频单元") - while hasError { - Thread.sleep(forTimeInterval: 0.1) - hasError = checkError(AudioUnitInitialize(ioUnit!), "初始化音频单元") - } - - // 启动音频单元 - hasError = checkError(AudioOutputUnitStart(ioUnit!), "启动音频单元") - - print("[CustomAudioProcessor] 音频处理器已启动，回音消除\(isEchoCancellationEnabled ? "已启用" : "已禁用")") - return !hasError - } - - /// 停止音频处理 - func stopRecord() { - print("[CustomAudioProcessor] 停止音频处理器") - - if let ioUnit = ioUnit { - // 停止音频单元 - _ = checkError(AudioOutputUnitStop(ioUnit), "停止音频单元") - - // 关闭音频单元 - _ = checkError(AudioUnitUninitialize(ioUnit), "反初始化音频单元") - _ = checkError(AudioComponentInstanceDispose(ioUnit), "释放音频单元") - - self.ioUnit = nil - } - - // 清空音频数据缓冲 - audioListQueue.sync { - audioList.removeAll() - } - } - - /// 音频数据回调 - 当有新的音频数据可用时调用 - private static let onAudioDataAvailable: AURenderCallback = { inRefCon, ioActionFlags, inTimeStamp, inBusNumber, inNumberFrames, ioData in - // 获取实例 - let processor = Unmanaged.fromOpaque(inRefCon).takeUnretainedValue() - - // 计算预期数据大小 - let expectedDataByteSize = inNumberFrames * processor.audioFormat.mBytesPerFrame - - // 确保缓冲区足够大 - if processor.audioBufferList.mBuffers.mDataByteSize < expectedDataByteSize { - processor.audioBufferList.mBuffers.mData = realloc(processor.audioBufferList.mBuffers.mData, Int(expectedDataByteSize)) - processor.audioBufferList.mBuffers.mDataByteSize = expectedDataByteSize - } - - // 渲染音频数据 - let status = processor.checkOSStatus(AudioUnitRender(processor.ioUnit!, ioActionFlags, inTimeStamp, - inBusNumber, inNumberFrames, &processor.audioBufferList), - "渲染音频数据") - - // 将Int16数据转换为浮点数据进行处理 - var audioDataFloat = [Float](repeating: 0.0, count: Int(inNumberFrames)) - let buffer = processor.audioBufferList.mBuffers - let bufferData = buffer.mData!.assumingMemoryBound(to: Int16.self) - - for j in 0...size)) { - // 归一化到[-1.0, 1.0]范围 - audioDataFloat[j] = Float(bufferData[j]) / 32768.0 - } - - // 应用附加处理 (如有需要) - // processor.applyAdditionalProcessing(&audioDataFloat) - - // 保存处理后的数据 - if status == noErr { - processor.audioListQueue.async { - processor.audioList.append(contentsOf: audioDataFloat) - } - } - - return status - } - - /// 读取处理后的音频数据 - /// - Parameter bytes: 输出字节数组 - /// - Returns: 读取的字节数 - func read(bytes: inout [UInt8]) -> Int { - return audioListQueue.sync { - // 如果没有数据，返回0 - if audioList.isEmpty { - return 0 - } - - // 确保有足够的数据 (至少1280个样本) - if audioList.count < 1280 { - return 0 - } - - // 读取一帧数据 (1280个样本) - let frameLength = 1280 - let buffer = Array(audioList.prefix(frameLength)) - audioList.removeFirst(frameLength) - - // 将浮点数据转回Int16格式 - var int16Data = buffer.map { Int16($0 * 32767) } - - // 转换为字节数组 - let data = Data(buffer: UnsafeBufferPointer(start: &int16Data, count: int16Data.count)) - bytes = [UInt8](data) - - // 每个样本2字节 (16位PCM) - return frameLength * 2 - } - } - - /// 检查错误并打印日志 - /// - Parameters: - /// - status: 操作状态 - /// - operation: 操作描述 - /// - Returns: 是否发生错误 - private func checkError(_ status: OSStatus, _ operation: String) -> Bool { - if status != noErr { - print("[CustomAudioProcessor] 错误: \(operation)失败: \(status)") - return true - } - return false - } - - /// 检查OSStatus并返回状态 - /// - Parameters: - /// - status: 操作状态 - /// - operation: 操作描述 - /// - Returns: 原始状态 - private func checkOSStatus(_ status: OSStatus, _ operation: String) -> OSStatus { - if status != noErr { - print("[CustomAudioProcessor] 错误: \(operation)失败: \(status)") - } - return status - } -} \ No newline at end of file diff --git a/azure/ios/Classes/AzureSpeechRecognitionPlugin.swift b/azure/ios/Classes/AzureSpeechRecognitionPlugin.swift deleted file mode 100644 index 141b77ef8..000000000 --- a/azure/ios/Classes/AzureSpeechRecognitionPlugin.swift +++ /dev/null @@ -1,18 +0,0 @@ -import Flutter -import UIKit - -public class AzureSpeechRecognitionPlugin: NSObject, FlutterPlugin { - public static func register(with registrar: FlutterPluginRegistrar) { - if #available(iOS 13.0, *) { - SwiftAzureSpeechRecognitionPlugin.register(with: registrar) - } else { - // 如果低于iOS 13.0，返回不支持的错误 - let channel = FlutterMethodChannel(name: "com.deep_voice.azure_asr", binaryMessenger: registrar.messenger()) - channel.setMethodCallHandler { (call, result) in - result(FlutterError(code: "UNSUPPORTED", - message: "需要iOS 13.0及以上系统", - details: nil)) - } - } - } -} \ No newline at end of file diff --git a/azure/ios/Classes/AzureTtsHelper.swift b/azure/ios/Classes/AzureTtsHelper.swift deleted file mode 100644 index 589df617c..000000000 --- a/azure/ios/Classes/AzureTtsHelper.swift +++ /dev/null @@ -1,427 +0,0 @@ -import Foundation -import MicrosoftCognitiveServicesSpeech -import AVFoundation - -/// Azure TTS工具类，负责实现TTS服务接口 -@available(iOS 13.0, *) -class AzureTtsHelper: NSObject { - // MARK: - 属性 - - /// 事件处理回调 - private var eventHandler: (String, [String: Any]) -> Void - - /// 语音配置信息 - private var speechSubscriptionKey: String = "" - private var serviceRegion: String = "" - - /// 语音合成配置 - private var speechConfig: SPXSpeechConfiguration? - - /// 语音合成器 - private var synthesizer: SPXSpeechSynthesizer? - - /// 是否初始化成功 - private var isInitialized = false - - /// 当前是否正在播放 - private var _isSpeaking = false - - /// 音频会话配置 - private var isAudioSessionConfigured = false - - // MARK: - 语音设置 - - /// 当前语音 - private var currentVoice = "zh-CN-XiaoxiaoNeural" - - /// 支持的语音映射 - private var voiceMap: [String: String] = [ - "zh-CN": "zh-CN-XiaoxiaoNeural", - "en-US": "en-US-JennyNeural", - "ja-JP": "ja-JP-NanamiNeural", - "ko-KR": "ko-KR-SunHiNeural", - "zh-TW": "zh-TW-HsiaoChenNeural", - "zh-HK": "zh-HK-HiuMaanNeural" - ] - - /// 当前语音合成参数 - private var currentSpeechRate = "0%" - private var currentPitch = "0%" - private var currentVolume = "100%" - - // MARK: - 初始化 - - init(eventHandler: @escaping (String, [String: Any]) -> Void) { - self.eventHandler = eventHandler - super.init() - } - - deinit { - dispose() - } - - // MARK: - TTS 接口实现 - - /// 初始化语音合成服务 - /// - Parameters: - /// - speechSubscriptionKey: Azure 语音服务订阅密钥 - /// - serviceRegion: Azure 服务区域 (如 eastasia) - /// - language: 语言代码 (默认 zh-CN) - /// - Returns: 初始化是否成功 - func initialize(speechSubscriptionKey: String, serviceRegion: String, language: String = "zh-CN") -> Bool { - print("[AzureTtsHelper] 初始化语音合成服务") - - // 检查配置是否为空 - if speechSubscriptionKey.isEmpty || serviceRegion.isEmpty { - print("[AzureTtsHelper] 错误: Azure 配置信息不完整") - eventHandler("error", ["error": "Azure 配置信息不完整"]) - return false - } - - // 释放之前的资源 - dispose() - - // 记录配置信息 - self.speechSubscriptionKey = speechSubscriptionKey - self.serviceRegion = serviceRegion - - // 配置音频会话 - if !configureAudioSession() { - print("[AzureTtsHelper] 警告: 音频会话配置失败，将尝试继续初始化") - } - - do { - // 创建语音配置 - speechConfig = try SPXSpeechConfiguration(subscription: speechSubscriptionKey, region: serviceRegion) - - // 设置默认语音 - let defaultVoice = getDefaultVoiceForLanguage(language) - currentVoice = defaultVoice - speechConfig?.speechSynthesisVoiceName = defaultVoice - - // 创建语音合成器 - synthesizer = try SPXSpeechSynthesizer(speechConfig!) - - // 设置事件处理器 - setupSynthesizerEvents() - - isInitialized = true - print("[AzureTtsHelper] TTS 引擎初始化成功") - - return true - } catch { - print("[AzureTtsHelper] 错误: 初始化语音合成服务失败: \(error.localizedDescription)") - eventHandler("error", ["error": "初始化语音合成服务失败: \(error.localizedDescription)"]) - return false - } - } - - /// 配置音频会话 - private func configureAudioSession() -> Bool { - let audioSession = AVAudioSession.sharedInstance() - do { - // 使用playback类别，但支持混合和空中播放 - try audioSession.setCategory(.playback, - mode: .spokenAudio, - options: [.mixWithOthers, .allowAirPlay, .duckOthers]) - - // 根据设备类型选择最佳配置 - let currentRoute = audioSession.currentRoute - let hasHeadphones = currentRoute.outputs.contains { - $0.portType == .headphones || $0.portType == .bluetoothA2DP || $0.portType == .bluetoothHFP - } - - // 优化音频路由 - if hasHeadphones { - // 耳机模式，使用默认设置 - try audioSession.setPreferredIOBufferDuration(0.005) // 较小的缓冲区大小以减少延迟 - } else { - // 扬声器模式 - try audioSession.setPreferredIOBufferDuration(0.005) - } - - // 避免完全激活音频会话，因为ASR可能已经激活 - // 这里使用setActive(false)是为了不与ASR冲突 - if !audioSession.isOtherAudioPlaying { - try audioSession.setActive(true, options: .notifyOthersOnDeactivation) - } - - isAudioSessionConfigured = true - print("[AzureTtsHelper] 音频会话配置成功") - return true - } catch { - print("[AzureTtsHelper] 警告: 音频会话配置失败: \(error.localizedDescription)") - isAudioSessionConfigured = false - return false - } - } - - /// 设置语音 - /// - Parameter voiceName: 语音名称 (如 "zh-CN-XiaoxiaoNeural") - /// - Returns: 设置是否成功 - func setVoice(voiceName: String) -> Bool { - if !isInitialized { - print("[AzureTtsHelper] 错误: TTS 引擎尚未初始化") - eventHandler("error", ["error": "TTS 引擎尚未初始化"]) - return false - } - - if voiceName.isEmpty { - print("[AzureTtsHelper] 错误: 声音名称为空") - eventHandler("error", ["error": "声音名称不能为空"]) - return false - } - - if voiceName == currentVoice { - print("[AzureTtsHelper] 已设置语音: \(voiceName)") - return true - } - - print("[AzureTtsHelper] 设置声音: \(voiceName)") - currentVoice = voiceName - - // 更新语音配置 - if let speechConfig = speechConfig { - speechConfig.speechSynthesisVoiceName = voiceName - return true - } - - return false - } - - /// 设置语音合成参数 - /// - Parameters: - /// - rate: 语速，范围 -100 到 100，默认为 0 - /// - pitch: 音调，范围 -100 到 100，默认为 0 - /// - volume: 音量，范围 0 到 100，默认为 100 - /// - Returns: 是否设置成功 - func setSpeechParams(rate: Int = 0, pitch: Int = 0, volume: Int = 100) -> Bool { - if !isInitialized { - print("[AzureTtsHelper] 错误: TTS 引擎尚未初始化") - eventHandler("error", ["error": "TTS 引擎尚未初始化"]) - return false - } - - // 转换参数格式 - currentSpeechRate = formatRateParam(rate) - currentPitch = formatPitchParam(pitch) - currentVolume = formatVolumeParam(volume) - - print("[AzureTtsHelper] 已设置语音参数: 语速=\(currentSpeechRate), 音调=\(currentPitch), 音量=\(currentVolume)") - return true - } - - /// 合成文本为语音并播放 - /// - Parameter text: 要合成的文本 - /// - Returns: 操作是否成功启动 - func speakText(text: String) -> Bool { - if !isInitialized { - print("[AzureTtsHelper] 错误: TTS 引擎尚未初始化") - eventHandler("error", ["error": "TTS 引擎尚未初始化"]) - return false - } - - if text.isEmpty { - print("[AzureTtsHelper] 警告: 要播放的文本为空") - return true - } - - // 确保音频会话已配置 - if !isAudioSessionConfigured { - _ = configureAudioSession() - } - - print("[AzureTtsHelper] 开始语音合成: \(text.prefix(50))...") - - // 生成SSML - let ssml = generateSsml(text: text) - - // 直接进行SSML合成 - return speakSsmlInternal(text: ssml) - } - - /// 内部SSML合成和播放 - private func speakSsmlInternal(text: String) -> Bool { - guard let synthesizer = synthesizer else { - print("[AzureTtsHelper] 错误: 合成器未初始化") - eventHandler("error", ["error": "合成器未初始化"]) - return false - } - - _isSpeaking = true - eventHandler("started", [:]) - - Task { - do { - // 使用异步方法进行合成并直接播放 - _ = try await synthesizer.startSpeakingSsml(text) - - } catch { - print("[AzureTtsHelper] 错误: 语音合成失败: \(error.localizedDescription)") - DispatchQueue.main.async { - self._isSpeaking = false - self.eventHandler("error", ["error": "语音合成失败: \(error.localizedDescription)"]) - } - } - } - - return true - } - - /// 停止当前语音合成 - /// - Returns: 操作是否成功 - func stopSpeaking() -> Bool { - if !isInitialized || !_isSpeaking { - return true - } - - // 停止合成 - do { - try synthesizer?.stopSpeaking() - _isSpeaking = false - eventHandler("canceled", [:]) - print("[AzureTtsHelper] 已停止语音合成") - return true - } catch { - print("[AzureTtsHelper] 错误: 停止语音合成失败: \(error.localizedDescription)") - eventHandler("error", ["error": "停止语音合成失败: \(error.localizedDescription)"]) - return false - } - } - - /// 检查是否正在播放 - /// - Returns: 当前是否正在播放语音 - func isSpeaking() -> Bool { - return _isSpeaking - } - - /// 释放资源 - func dispose() { - try? stopSpeaking() - - // 释放合成器和配置 - synthesizer = nil - speechConfig = nil - - isInitialized = false - _isSpeaking = false - isAudioSessionConfigured = false - print("[AzureTtsHelper] TTS 引擎已释放") - } - - // MARK: - 私有辅助方法 - - /// 设置合成器事件处理 - private func setupSynthesizerEvents() { - guard let synthesizer = synthesizer else { return } - - // 添加书签到达事件处理 - synthesizer.addBookmarkReachedEventHandler { _, e in - print("[AzureTtsHelper] 书签事件: 音频偏移: \((e.audioOffset + 5000) / 10000)ms, 文本: \"\(e.text)\"") - } - - // 合成完成事件 - synthesizer.addSynthesisCompletedEventHandler { [weak self] _, e in - guard let self = self else { return } - print("[AzureTtsHelper] 语音合成完成: 音频持续时间: \(e.result.audioDuration)") - DispatchQueue.main.async { - self._isSpeaking = false - self.eventHandler("completed", [:]) - } - } - - // 合成取消事件 - synthesizer.addSynthesisCanceledEventHandler { [weak self] _, e in - guard let self = self else { return } - - let result = e.result - do { - let cancellationDetails = try SPXSpeechSynthesisCancellationDetails(fromCanceledSynthesisResult: result) - print("[AzureTtsHelper] 语音合成取消: 原因: \(cancellationDetails.reason)") - - if cancellationDetails.reason == SPXCancellationReason.error { - print("[AzureTtsHelper] 错误代码: \(cancellationDetails.errorCode)") - print("[AzureTtsHelper] 错误详情: \(cancellationDetails.errorDetails ?? "未知")") - } - - DispatchQueue.main.async { - self._isSpeaking = false - self.eventHandler("error", ["error": "语音合成取消: \(cancellationDetails.errorDetails ?? "未知错误")"]) - } - } catch { - print("[AzureTtsHelper] 获取取消详情时出错: \(error)") - - DispatchQueue.main.async { - self._isSpeaking = false - self.eventHandler("error", ["error": "语音合成被取消"]) - } - } - } - - // 合成开始事件 - synthesizer.addSynthesisStartedEventHandler { _, _ in - // print("[AzureTtsHelper] 语音合成开始") - } - - // 合成中事件 - synthesizer.addSynthesizingEventHandler { _, _ in - // print("[AzureTtsHelper] 语音合成中") - } - } - - /// 生成 SSML 文本 - private func generateSsml(text: String) -> String { - return """ - - - - \(text) - - - - """ - } - - /// 格式化语速参数 - private func formatRateParam(_ rate: Int) -> String { - let clampedRate = rate.clamp(min: -100, max: 100) - if clampedRate == 0 { - return "0%" - } else if clampedRate < 0 { - return "\(Int(Double(clampedRate) * 0.9))%" - } else { - return "+\(clampedRate)%" - } - } - - /// 格式化音调参数 - private func formatPitchParam(_ pitch: Int) -> String { - let clampedPitch = pitch.clamp(min: -100, max: 100) - if clampedPitch == 0 { - return "0%" - } else { - return "\(Int(Double(clampedPitch) * 0.5))%" - } - } - - /// 格式化音量参数 - private func formatVolumeParam(_ volume: Int) -> String { - let clampedVolume = volume.clamp(min: 0, max: 100) - return "\(clampedVolume)%" - } - - /// 获取指定语言的默认语音 - private func getDefaultVoiceForLanguage(_ language: String) -> String { - return voiceMap[language] ?? "zh-CN-XiaoxiaoNeural" - } -} - -// MARK: - 扩展 - -extension Int { - func clamp(min: Int, max: Int) -> Int { - if self < min { return min } - if self > max { return max } - return self - } -} \ No newline at end of file diff --git a/azure/ios/Classes/SwiftAzureSpeechRecognitionPlugin.swift b/azure/ios/Classes/SwiftAzureSpeechRecognitionPlugin.swift deleted file mode 100644 index ea393e215..000000000 --- a/azure/ios/Classes/SwiftAzureSpeechRecognitionPlugin.swift +++ /dev/null @@ -1,259 +0,0 @@ -import Flutter -import UIKit -import MicrosoftCognitiveServicesSpeech -import AVFoundation - -@available(iOS 13.0, *) -public class SwiftAzureSpeechRecognitionPlugin: NSObject, FlutterPlugin { - private var azureChannel: FlutterMethodChannel - private var ttsChannel: FlutterMethodChannel - private var asrHelper: AzureAsrHelper - private var ttsHelper: AzureTtsHelper - private static var eventStreamHandler: AzureEventStreamHandler? - - // 创建方法到通道的映射 - private static var ttsMethodHandlers = [String: FlutterMethodCallHandler]() - private static var asrMethodHandlers = [String: FlutterMethodCallHandler]() - - public static func register(with registrar: FlutterPluginRegistrar) { - // ASR通道 - let channel = FlutterMethodChannel(name: "com.deep_voice.azure_asr", binaryMessenger: registrar.messenger()) - - // TTS通道 - let ttsChannel = FlutterMethodChannel(name: "com.deep_voice.azure_tts", binaryMessenger: registrar.messenger()) - - // 设置ASR事件通道 - let eventChannel = FlutterEventChannel(name: "com.deep_voice.azure_asr_events", binaryMessenger: registrar.messenger()) - eventStreamHandler = AzureEventStreamHandler() - eventChannel.setStreamHandler(eventStreamHandler) - - let instance = SwiftAzureSpeechRecognitionPlugin( - azureChannel: channel, - ttsChannel: ttsChannel, - eventStreamHandler: eventStreamHandler! - ) - - // 直接设置各自通道的处理器 - channel.setMethodCallHandler(instance.handleAsrMethodCalls) - ttsChannel.setMethodCallHandler(instance.handleTtsMethodCalls) - } - - - - // 新增直接处理方法调用的函数 - private func handleTtsMethodCalls(_ call: FlutterMethodCall, result: @escaping FlutterResult) { - handleTtsMethod(call, result) - } - - private func handleAsrMethodCalls(_ call: FlutterMethodCall, result: @escaping FlutterResult) { - handleAsrMethod(call, result) - } - - - init(azureChannel: FlutterMethodChannel, ttsChannel: FlutterMethodChannel, eventStreamHandler: AzureEventStreamHandler) { - self.azureChannel = azureChannel - self.ttsChannel = ttsChannel - - // 创建辅助类实例，使用自定义事件回调处理器 - let eventHandler: (String, [String: Any]) -> Void = { eventName, arguments in - DispatchQueue.main.async { - if let eventSink = SwiftAzureSpeechRecognitionPlugin.eventStreamHandler?.eventSink { - var eventData = arguments - eventData["type"] = eventName - eventSink(eventData) - } - } - } - - asrHelper = AzureAsrHelper(eventHandler: eventHandler) - ttsHelper = AzureTtsHelper(eventHandler: eventHandler) - - super.init() - } - - private func handleAsrMethod(_ call: FlutterMethodCall, _ result: @escaping FlutterResult) { - - let args = call.arguments as? Dictionary - - switch call.method { - case "initialize": - // 仅在初始化时读取必要参数 - guard let speechSubscriptionKey = args?["subscriptionKey"] as? String, !speechSubscriptionKey.isEmpty else { - let errorMsg = "语音订阅密钥不能为空" - print("[AzurePlugin] 错误: \(errorMsg)") - result(FlutterError(code: "INVALID_SUBSCRIPTION_KEY", message: errorMsg, details: nil)) - return - } - - guard let serviceRegion = args?["region"] as? String, !serviceRegion.isEmpty else { - let errorMsg = "服务区域不能为空" - print("[AzurePlugin] 错误: \(errorMsg)") - result(FlutterError(code: "INVALID_REGION", message: errorMsg, details: nil)) - return - } - - let supportedLanguages = args?["supportedLanguages"] as? [String] ?? [] - - let success = asrHelper.initialize( - speechSubscriptionKey: speechSubscriptionKey, - serviceRegion: serviceRegion, - supportedLanguages: supportedLanguages.isEmpty ? nil : supportedLanguages - ) - result(success) - - case "startContinuousRecognition": - // 只有使用参数时才验证 - let success = asrHelper.startContinuousRecognition() - result(success) - - case "stopContinuousRecognition": - // 不需要额外参数 - let success = asrHelper.stopContinuousRecognition() - result(success) - - case "recognizeOnce": - // 只有使用参数时才验证 - let success = asrHelper.recognizeOnce() - result(success) - - case "isContinuousRecognitionActive": - // 不需要额外参数 - result(asrHelper.isContinuousRecognitionActive()) - - case "dispose": - // 不需要额外参数 - print("[AzurePlugin] 释放ASR资源") - asrHelper.dispose() - result(true) - - default: - print("[AzurePlugin] 错误: 未知ASR方法: \(call.method)") - result(FlutterMethodNotImplemented) - } - } - - private func handleTtsMethod(_ call: FlutterMethodCall, _ result: @escaping FlutterResult) { - - let args = call.arguments as? Dictionary - - switch call.method { - case "initialize": - // 仅在初始化时验证参数 - guard let speechSubscriptionKey = args?["subscriptionKey"] as? String, !speechSubscriptionKey.isEmpty else { - let errorMsg = "语音订阅密钥不能为空" - print("[AzurePlugin] 错误: \(errorMsg)") - result(FlutterError(code: "INVALID_SUBSCRIPTION_KEY", message: errorMsg, details: nil)) - return - } - - guard let serviceRegion = args?["region"] as? String, !serviceRegion.isEmpty else { - let errorMsg = "服务区域不能为空" - print("[AzurePlugin] 错误: \(errorMsg)") - result(FlutterError(code: "INVALID_REGION", message: errorMsg, details: nil)) - return - } - - let language = args?["language"] as? String ?? "zh-CN" - - print("[AzurePlugin] 初始化TTS，语言: \(language)") - - let success = ttsHelper.initialize(speechSubscriptionKey: speechSubscriptionKey, serviceRegion: serviceRegion, language: language) - result(success) - - case "setVoice": - // 仅获取voice参数 - guard let voiceName = args?["voiceName"] as? String, !voiceName.isEmpty else { - let errorMsg = "声音名称不能为空" - print("[AzurePlugin] 错误: \(errorMsg)") - result(FlutterError(code: "INVALID_VOICE", message: errorMsg, details: nil)) - return - } - - print("[AzurePlugin] 设置声音: \(voiceName)") - - let success = ttsHelper.setVoice(voiceName: voiceName) - result(success) - - case "speakText": - // 仅获取text参数 - let text = args?["text"] as? String ?? "" - - if text.isEmpty { - print("[AzurePlugin] 警告: 要播放的文本为空") - result("OK") - return - } - - print("[AzurePlugin] 播放文本: \(text.prefix(50))...") - - let success = ttsHelper.speakText(text: text) - result(success ? "OK" : "ERROR") - - case "speakSsml": - // 仅获取ssml参数 - guard let ssml = args?["ssml"] as? String, !ssml.isEmpty else { - let errorMsg = "SSML内容不能为空" - print("[AzurePlugin] 错误: \(errorMsg)") - result(FlutterError(code: "INVALID_SSML", message: errorMsg, details: nil)) - return - } - - print("[AzurePlugin] 播放SSML: \(ssml.prefix(100))...") - - // 由于我们移除了speakSsml方法，这里改用speakText方法 - // Azure SDK内部会自动检测是普通文本还是SSML - let success = ttsHelper.speakText(text: ssml) - result(success) - - case "stopSpeaking": - // 不需要参数 - print("[AzurePlugin] 停止播放") - let success = ttsHelper.stopSpeaking() - result(success) - - case "isSpeaking": - // 不需要参数 - result(ttsHelper.isSpeaking()) - - case "setSpeechParams": - // 仅获取语音参数 - let rate = args?["rate"] as? Int ?? 0 - let pitch = args?["pitch"] as? Int ?? 0 - let volume = args?["volume"] as? Int ?? 100 - - print("[AzurePlugin] 设置语音参数: rate=\(rate), pitch=\(pitch), volume=\(volume)") - let success = ttsHelper.setSpeechParams(rate: rate, pitch: pitch, volume: volume) - result(success) - - case "dispose": - // 释放TTS资源 - print("[AzurePlugin] 释放TTS资源") - ttsHelper.dispose() - result(true) - - default: - print("[AzurePlugin] 错误: 未知TTS方法: \(call.method)") - result(FlutterMethodNotImplemented) - } - } -} - -// 用于处理事件流的辅助类 -@available(iOS 13.0, *) -class AzureEventStreamHandler: NSObject, FlutterStreamHandler { - var eventSink: FlutterEventSink? - - func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { - self.eventSink = events - // 通知Flutter端事件通道已准备好 - DispatchQueue.main.async { - events(["type": "channelReady"]) - } - return nil - } - - func onCancel(withArguments arguments: Any?) -> FlutterError? { - self.eventSink = nil - return nil - } -} \ No newline at end of file diff --git a/azure/ios/azure_speech_recognition.podspec b/azure/ios/azure_speech_recognition.podspec deleted file mode 100644 index 88797fb0c..000000000 --- a/azure/ios/azure_speech_recognition.podspec +++ /dev/null @@ -1,24 +0,0 @@ -# -# To learn more about a Podspec see http://guides.cocoapods.org/syntax/podspec.html. -# Run `pod lib lint azure_speech_recognition.podspec` to validate before publishing. -# -Pod::Spec.new do |s| - s.name = 'azure_speech_recognition' - s.version = '0.1.0' - s.summary = 'Azure Speech Recognition plugin for Flutter' - s.description = <<-DESC -A Flutter plugin for Microsoft Azure Speech services, providing both speech recognition (ASR) and text-to-speech (TTS) capabilities. - DESC - s.homepage = 'https://github.com/yourusername/azure_speech_recognition' - s.license = { :type => 'MIT', :file => '../LICENSE' } - s.author = { 'Your Company' => 'your-email@example.com' } - s.source = { :path => '.' } - s.source_files = 'Classes/**/*' - s.dependency 'Flutter' - s.dependency 'MicrosoftCognitiveServicesSpeech-iOS', '~> 1.34.0' - s.platform = :ios, '12.0' - - # Flutter.framework does not contain a i386 slice. - s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386' } - s.swift_version = '5.0' -end \ No newline at end of file diff --git a/azure/lib/azure_speech_recognition.dart b/azure/lib/azure_speech_recognition.dart deleted file mode 100644 index 02b3f92d8..000000000 --- a/azure/lib/azure_speech_recognition.dart +++ /dev/null @@ -1,6 +0,0 @@ -// This is a placeholder file that exports nothing. -// The actual implementation is in the app's services folder. -// This file exists just to satisfy the Flutter plugin structure requirements. - -// Empty library to satisfy plugin structure -library azure_speech_recognition; \ No newline at end of file diff --git a/azure/pubspec.yaml b/azure/pubspec.yaml deleted file mode 100644 index a36c77bdc..000000000 --- a/azure/pubspec.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: azure_speech_recognition -description: Azure Speech Recognition and Text-to-Speech services Flutter plugin -version: 0.1.0 -homepage: https://github.com/yourusername/azure_speech_recognition - -environment: - sdk: '>=2.12.0 <3.0.0' - flutter: ">=2.0.0" - -dependencies: - flutter: - sdk: flutter - -dev_dependencies: - flutter_test: - sdk: flutter - flutter_lints: ^1.0.0 - -flutter: - plugin: - platforms: - ios: - pluginClass: AzureSpeechRecognitionPlugin \ No newline at end of file diff --git a/lib/data/services/speech_impl/azure_asr_service.dart b/lib/data/services/speech_impl/azure_asr_service.dart index ffb67ed46..e09defb66 100644 --- a/lib/data/services/speech_impl/azure_asr_service.dart +++ b/lib/data/services/speech_impl/azure_asr_service.dart @@ -10,8 +10,8 @@ import '../asr_service.dart'; /// 该服务提供了通过平台通道与 Android 上的 Microsoft Speech SDK 交互的接口 class AzureAsrService extends GetxService implements AsrService { static final AzureAsrService to = Get.put(AzureAsrService()); - static const MethodChannel _channel = MethodChannel('com.deep_voice.azure_asr'); - static const EventChannel _eventChannel = EventChannel('com.deep_voice.azure_asr_events'); + static const MethodChannel _channel = MethodChannel('azure_speech/asr'); + static const EventChannel _eventChannel = EventChannel('azure_speech/asr_events'); bool _isInitialized = false; late final String _subscriptionKey; diff --git a/lib/data/services/speech_impl/azure_tts_service.dart b/lib/data/services/speech_impl/azure_tts_service.dart index ae5ab9369..1cdb62507 100644 --- a/lib/data/services/speech_impl/azure_tts_service.dart +++ b/lib/data/services/speech_impl/azure_tts_service.dart @@ -12,7 +12,7 @@ import '../tts_service.dart'; /// 提供文本转语音功能。 class AzureTtsService extends GetxService implements TtsService { static final AzureTtsService to = Get.put(AzureTtsService()); - static const MethodChannel _channel = MethodChannel('com.deep_voice.azure_tts'); + static const MethodChannel _channel = MethodChannel('azure_speech/tts'); bool _isInitialized = false; late final String _subscriptionKey; diff --git a/lib/data/services/voice_interaction_service.dart b/lib/data/services/voice_interaction_service.dart index a1d26d509..1bb2406f5 100644 --- a/lib/data/services/voice_interaction_service.dart +++ b/lib/data/services/voice_interaction_service.dart @@ -38,7 +38,8 @@ class VoiceInteractionService extends GetxService { // 配置信息 late String _azureSpeechKey; late String _azureSpeechRegion; - late String _volcanoAiKey; + late String _openaiApiKey; + late String _openaiBaseUrl; // 聊天历史服务 late final ChatHistoryService _chatHistoryService; @@ -57,15 +58,13 @@ class VoiceInteractionService extends GetxService { void _loadConfig() { _azureSpeechKey = dotenv.env['AZURE_SPEECH_KEY'] ?? ''; _azureSpeechRegion = dotenv.env['AZURE_SPEECH_REGION'] ?? ''; - _volcanoAiKey = dotenv.env['VOLCANO_AI_API_KEY'] ?? ''; + _openaiApiKey = dotenv.env['OPENAI_API_KEY'] ?? ''; + _openaiBaseUrl = dotenv.env['OPENAI_BASE_URL'] ?? ''; if (_azureSpeechKey.isEmpty || _azureSpeechRegion.isEmpty) { Logger.warning('未找到 Azure 语音服务配置。请在 .env 文件中设置 AZURE_SPEECH_KEY 和 AZURE_SPEECH_REGION'); } - if (_volcanoAiKey.isEmpty) { - Logger.warning('未找到火山 AI API 密钥。请在 .env 文件中设置 VOLCANO_AI_API_KEY'); - } } /// 处理来自原生层的事件 @@ -195,7 +194,8 @@ class VoiceInteractionService extends GetxService { final result = await _channel.invokeMethod('startService', { 'azure_speech_key': _azureSpeechKey, 'azure_speech_region': _azureSpeechRegion, - 'volcano_ai_api_key': _volcanoAiKey, + 'openai_api_key': _openaiApiKey, + 'openai_base_url': _openaiBaseUrl, }) ?? false; if (result) { diff --git a/local_plugins/azure_speech/README.md b/local_plugins/azure_speech/README.md new file mode 100644 index 000000000..067b344eb --- /dev/null +++ b/local_plugins/azure_speech/README.md @@ -0,0 +1,136 @@ +# Azure Speech 插件 + +本插件为Flutter提供了Azure语音服务的集成，包括： + +- 语音合成（TTS） +- 语音识别（ASR） + +## 功能 + +### 语音合成（TTS） + +- 支持多种语音（如中文、英文等） +- 语音参数调整（语速、音调、音量） +- 音频输出设备选择（扬声器、听筒、自动） +- SSML支持 + +### 语音识别（ASR） + +- 一次性语音识别 +- 连续语音识别 +- 自动语言检测 +- 音频处理优化（回音消除、噪声抑制等） + +## 平台支持 + +- Android +- iOS + +## 如何使用 + +### 初始化 + +```dart +import 'package:azure_speech/azure_speech.dart'; + +// 初始化TTS +await AzureSpeech.initializeTts( + 'your_subscription_key', + 'your_service_region', + language: 'zh-CN', +); + +// 初始化ASR +await AzureSpeech.initializeAsr( + 'your_subscription_key', + 'your_service_region', + ['zh-CN', 'en-US'], +); +``` + +### 语音合成 + +```dart +// 设置语音 +await AzureSpeech.setTtsVoice('zh-CN-XiaoxiaoNeural'); + +// 设置语音参数 +await AzureSpeech.setTtsSpeechParams( + rate: 0, // 语速 -100~100 + pitch: 0, // 音调 -100~100 + volume: 100, // 音量 0~100 +); + +// 设置音频输出设备 +await AzureSpeech.setTtsAudioOutputType('AUTO'); // 'SPEAKER', 'EARPIECE', 'AUTO' + +// 播放文本 +await AzureSpeech.speakText('你好，世界！'); + +// 停止播放 +await AzureSpeech.stopSpeaking(); + +// 检查是否正在播放 +bool isSpeaking = await AzureSpeech.isSpeaking(); +``` + +### 语音识别 + +```dart +// 一次性识别 +final result = await AzureSpeech.recognizeOnce(); +if (result['success']) { + print('识别文本: ${result['text']}'); + print('识别语言: ${result['language']}'); +} else { + print('识别失败: ${result['error']}'); +} + +// 连续识别 +// 监听识别结果 +AzureSpeech.asrResultStream.listen((event) { + switch (event['eventType']) { + case 'recognizing': + // 实时识别中的结果 + print('识别中: ${event['text']}'); + break; + case 'finalResult': + // 最终识别结果 + print('最终结果: ${event['text']}'); + break; + case 'error': + // 错误 + print('错误: ${event['error']}'); + break; + } +}); + +// 开始连续识别 +await AzureSpeech.startContinuousRecognition(); + +// 停止连续识别 +await AzureSpeech.stopContinuousRecognition(); + +// 检查连续识别是否活跃 +bool isActive = await AzureSpeech.isContinuousRecognitionActive(); +``` + +### 释放资源 + +```dart +// 释放所有资源 +await AzureSpeech.dispose(); +``` + +## 依赖项 + +本插件依赖于： + +- Microsoft Cognitive Services Speech SDK +- Flutter + +## 注意事项 + +- 使用前需要在Azure门户中创建语音服务资源，并获取订阅密钥和区域 +- Android需要相关权限：RECORD_AUDIO, INTERNET等 +- iOS需要在Info.plist中添加麦克风使用权限描述 \ No newline at end of file diff --git a/local_plugins/azure_speech/android/build.gradle.kts b/local_plugins/azure_speech/android/build.gradle.kts new file mode 100644 index 000000000..665d495d1 --- /dev/null +++ b/local_plugins/azure_speech/android/build.gradle.kts @@ -0,0 +1,65 @@ +import com.android.build.gradle.LibraryExtension + +buildscript { + repositories { + google() + mavenCentral() + } + dependencies { + classpath("com.android.tools.build:gradle:7.3.0") + classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:1.7.10") + } +} + +allprojects { + repositories { + google() + mavenCentral() + } +} + +plugins { + id("com.android.library") + kotlin("android") +} + +// 配置android扩展 +configure { + namespace = "com.yunqiinnovation.azure_speech" + compileSdkVersion(33) + + defaultConfig { + minSdk = 21 + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + + sourceSets { + getByName("main") { + manifest.srcFile("src/main/AndroidManifest.xml") + java.srcDirs("src/main/kotlin") + } + } + + // 添加lint选项 + lintOptions { + isCheckReleaseBuilds = false + } +} + +// 显式设置Kotlin JVM目标版本 +tasks.withType { + kotlinOptions { + jvmTarget = "1.8" + } +} + +dependencies { + // 直接通过本地依赖方式添加Flutter + implementation(fileTree(mapOf("dir" to "libs", "include" to listOf("*.jar")))) + // 添加Microsoft语音SDK + implementation("com.microsoft.cognitiveservices.speech:client-sdk:1.30.0") +} \ No newline at end of file diff --git a/local_plugins/azure_speech/android/settings.gradle.kts b/local_plugins/azure_speech/android/settings.gradle.kts new file mode 100644 index 000000000..14ba24107 --- /dev/null +++ b/local_plugins/azure_speech/android/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "azure_speech" diff --git a/local_plugins/azure_speech/android/src/main/AndroidManifest.xml b/local_plugins/azure_speech/android/src/main/AndroidManifest.xml new file mode 100644 index 000000000..df2a770c6 --- /dev/null +++ b/local_plugins/azure_speech/android/src/main/AndroidManifest.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt new file mode 100644 index 000000000..ac6697cdb --- /dev/null +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt @@ -0,0 +1,593 @@ +package com.yunqiinnovation.azure_speech + +import android.content.Context +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioRecord +import android.media.MediaRecorder +import android.media.audiofx.AcousticEchoCanceler +import android.media.audiofx.NoiseSuppressor +import android.media.audiofx.AutomaticGainControl +import android.os.Process +import com.yunqiinnovation.azure_speech.utils.FileLogger +import com.microsoft.cognitiveservices.speech.* +import com.microsoft.cognitiveservices.speech.audio.* +import com.microsoft.cognitiveservices.speech.util.EventHandler +import java.util.concurrent.ExecutionException +import java.util.concurrent.atomic.AtomicBoolean + +class AzureAsrHelper(private val context: Context) { + private var recognizer: SpeechRecognizer? = null + private var speechConfig: SpeechConfig? = null + private val TAG = "AzureAsrHelper" + private var isContinuousRecognitionActive = false + private var currentLanguage = "zh-CN" + private var subscriptionKey = "" + private var region = "" + private var isAutoDetectLanguage = false + private var supportedLanguages = arrayOf("zh-CN", "en-US") + + // 是否使用回音消除 - 内部控制常量 + private val useEchoCancellation = false + + // 自定义音频处理相关 + private var customAudioProcessor: CustomAudioProcessor? = null + private var pushStream: PushAudioInputStream? = null + private var audioConfig: AudioConfig? = null + + // 初始化SDK并创建recognizer + fun initialize(subscriptionKey: String, region: String, + supportedLanguages: Array = arrayOf("zh-CN", "en-US")): Boolean { + try { + FileLogger.d(TAG, "初始化 Azure 语音服务") + + // 检查配置是否为空 + if (subscriptionKey.isEmpty() || region.isEmpty()) { + FileLogger.e(TAG, "Azure 配置信息不完整") + return false + } + + // 释放之前的资源 + dispose() + + this.subscriptionKey = subscriptionKey + this.region = region + + // 设置语言 + if (supportedLanguages.isNotEmpty()) { + this.supportedLanguages = supportedLanguages + } + + // 根据支持的语言数量决定是否启用自动语言检测 + this.isAutoDetectLanguage = supportedLanguages.size >= 2 + + // 如果只有一种语言，设置为当前语言 + if (!isAutoDetectLanguage && supportedLanguages.isNotEmpty()) { + this.currentLanguage = supportedLanguages[0] + } + + // 创建语音配置 + speechConfig = SpeechConfig.fromSubscription(subscriptionKey, region) + + // 设置语言配置 + if (isAutoDetectLanguage) { + // 设置自动语言检测 + speechConfig?.setProperty(PropertyId.SpeechServiceConnection_LanguageIdMode, "Continuous") + } else { + // 设置指定的识别语言 + speechConfig?.speechRecognitionLanguage = currentLanguage + } + + // 创建识别器 + try { + if (useEchoCancellation) { + // 如果使用回音消除，创建自定义音频输入流 + setupCustomAudioProcessing() + + if (isAutoDetectLanguage) { + val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) + recognizer = SpeechRecognizer(speechConfig, autoDetectConfig, audioConfig) + } else { + recognizer = SpeechRecognizer(speechConfig, audioConfig) + } + } else { + // 使用默认麦克风输入 + if (isAutoDetectLanguage) { + val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) + recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) + } else { + recognizer = SpeechRecognizer(speechConfig) + } + } + + FileLogger.d(TAG, "Azure 语音服务初始化成功") + return true + } catch (e: Exception) { + FileLogger.e(TAG, "创建识别器失败: ${e.message}") + stopCustomAudioProcessing() + return false + } + } catch (e: Exception) { + FileLogger.e(TAG, "初始化失败: ${e.message}") + return false + } + } + + // 重置 recognizer + private fun resetRecognizer(): Boolean { + try { + // 释放之前的 recognizer + recognizer?.close() + recognizer = null + + // 停止当前的音频处理 + stopCustomAudioProcessing() + + // 使用现有配置重新创建 recognizer + if (speechConfig != null) { + if (useEchoCancellation) { + // 如果使用回音消除，创建自定义音频输入流 + setupCustomAudioProcessing() + + if (isAutoDetectLanguage) { + val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) + recognizer = SpeechRecognizer(speechConfig, autoDetectConfig, audioConfig) + } else { + recognizer = SpeechRecognizer(speechConfig, audioConfig) + } + } else { + // 使用默认麦克风输入 + if (isAutoDetectLanguage) { + val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) + recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) + } else { + recognizer = SpeechRecognizer(speechConfig) + } + } + return true + } else { + FileLogger.e(TAG, "语音配置未初始化") + return false + } + } catch (e: Exception) { + FileLogger.e(TAG, "重置识别器失败: ${e.message}") + return false + } + } + + // 开始一次性语音识别 + fun recognizeOnce(callback: RecognizeCallback) { + if (speechConfig == null) { + callback.onError("语音服务未初始化") + return + } + + // 重置 recognizer + if (!resetRecognizer()) { + callback.onError("重置识别器失败") + return + } + + try { + // 启动音频处理 + startCustomAudioProcessing() + + // 执行识别 + val result = recognizer?.recognizeOnceAsync()?.get() + + // 停止音频处理 + stopCustomAudioProcessing() + + if (result != null && result.reason == ResultReason.RecognizedSpeech) { + val detectedLanguage = AutoDetectSourceLanguageResult.fromResult(result)?.language + callback.onResult(result.text, detectedLanguage ?: "") + } else { + callback.onError("未能识别语音") + } + } catch (e: Exception) { + // 停止音频处理 + stopCustomAudioProcessing() + callback.onError("识别异常: ${e.message}") + } + } + + // 开始连续语音识别 + fun startContinuousRecognition(callback: ContinuousRecognizeCallback): Boolean { + if (speechConfig == null) { + callback.onError("语音服务未初始化") + return false + } + + if (isContinuousRecognitionActive) { + FileLogger.w(TAG, "已在进行连续识别，忽略请求") + return true + } + + // 重置 recognizer + if (!resetRecognizer()) { + callback.onError("重置识别器失败") + return false + } + + try { + // 启动音频处理 + startCustomAudioProcessing() + + // 识别中事件 + recognizer?.recognizing?.addEventListener( + EventHandler { _, event -> + val detectedLanguage = AutoDetectSourceLanguageResult.fromResult(event.result)?.language + FileLogger.d(TAG, "识别中: ${event.result.text}, 语言: $detectedLanguage") + callback.onRecognizing(event.result.text, detectedLanguage ?: "") + } + ) + + // 识别完成事件 + recognizer?.recognized?.addEventListener( + EventHandler { _, event -> + if (event.result.reason == ResultReason.RecognizedSpeech) { + val detectedLanguage = AutoDetectSourceLanguageResult.fromResult(event.result)?.language + FileLogger.d(TAG, "识别完成: ${event.result.text}, 语言: $detectedLanguage") + callback.onResult(event.result.text, detectedLanguage ?: "") + } + } + ) + + // 会话开始事件 + recognizer?.sessionStarted?.addEventListener( + EventHandler { _, _ -> + FileLogger.d(TAG, "识别会话已开始") + callback.onSessionStarted() + callback.onSuccess("开始识别") // 兼容旧接口 + } + ) + + // 会话结束事件 + recognizer?.sessionStopped?.addEventListener( + EventHandler { _, _ -> + FileLogger.d(TAG, "识别会话已结束") + isContinuousRecognitionActive = false + stopCustomAudioProcessing() + callback.onSessionStopped() + } + ) + + // 取消事件 + recognizer?.canceled?.addEventListener( + EventHandler { _, event -> + val errorDetails = try { + event.errorDetails ?: "未知错误" + } catch (e: Exception) { + "未知错误" + } + val reason = event.reason.toString() + FileLogger.e(TAG, "识别取消: $errorDetails") + isContinuousRecognitionActive = false + stopCustomAudioProcessing() + callback.onCanceled(reason, errorDetails) + callback.onError("识别取消: $errorDetails") // 兼容旧接口 + } + ) + + // 开始连续识别 + recognizer?.startContinuousRecognitionAsync() + isContinuousRecognitionActive = true + FileLogger.d(TAG, "连续识别已启动") + + return true + } catch (e: Exception) { + // 停止音频处理 + stopCustomAudioProcessing() + FileLogger.e(TAG, "开始连续识别失败: ${e.message}") + e.printStackTrace() + callback.onError("开始连续识别失败: ${e.message}") + return false + } + } + + // 停止连续语音识别 + fun stopContinuousRecognition(callback: ContinuousRecognizeCallback): Boolean { + if (speechConfig == null) { + callback.onError("语音服务未初始化") + return false + } + + if (!isContinuousRecognitionActive) { + FileLogger.d(TAG, "未进行连续识别，忽略停止请求") + return true + } + + try { + FileLogger.d(TAG, "停止连续语音识别") + + if (recognizer == null) { + if (isContinuousRecognitionActive) { + FileLogger.w(TAG, "识别器为空，但状态显示活跃") + } + isContinuousRecognitionActive = false + return true + } + + // 停止连续识别 + val future = recognizer?.stopContinuousRecognitionAsync() + future?.get() + + // 停止音频处理 + stopCustomAudioProcessing() + + isContinuousRecognitionActive = false + FileLogger.d(TAG, "连续识别已停止") + callback.onSuccess("连续识别已停止") + + return true + } catch (e: Exception) { + // 强制重置状态 + isContinuousRecognitionActive = false + FileLogger.e(TAG, "停止连续识别失败: ${e.message}") + e.printStackTrace() + callback.onError("停止连续识别失败: ${e.message}") + + // 停止音频处理 + stopCustomAudioProcessing() + + // 尝试强制关闭识别器 + try { + recognizer?.close() + recognizer = null + } catch (ex: Exception) { + FileLogger.e(TAG, "关闭识别器失败: ${ex.message}") + } + + return false + } + } + + // 释放资源 + fun dispose() { + try { + // 如果正在进行连续识别，先停止 + if (isContinuousRecognitionActive) { + recognizer?.stopContinuousRecognitionAsync()?.get() + isContinuousRecognitionActive = false + } + + // 停止音频处理 + stopCustomAudioProcessing() + + // 释放recognizer + recognizer?.close() + recognizer = null + + // 释放speechConfig + speechConfig?.close() + speechConfig = null + + FileLogger.d(TAG, "资源已释放") + } catch (e: Exception) { + FileLogger.e(TAG, "释放资源失败: ${e.message}") + + // 确保状态被重置 + isContinuousRecognitionActive = false + customAudioProcessor = null + pushStream = null + audioConfig = null + recognizer = null + speechConfig = null + } + } + + // 检查连续识别是否处于活跃状态 + fun isContinuousRecognitionActive(): Boolean { + return isContinuousRecognitionActive + } + + // 设置自定义音频处理 + private fun setupCustomAudioProcessing() { + try { + // 创建音频推送流 + pushStream = PushAudioInputStream.create() + + // 创建音频配置 + audioConfig = AudioConfig.fromStreamInput(pushStream) + + // 创建自定义音频处理器 + customAudioProcessor = CustomAudioProcessor(pushStream!!) + } catch (e: Exception) { + FileLogger.e(TAG, "设置自定义音频处理失败: ${e.message}") + e.printStackTrace() + } + } + + // 启动自定义音频处理 + private fun startCustomAudioProcessing() { + if (useEchoCancellation && customAudioProcessor != null) { + try { + customAudioProcessor?.startProcessing() + } catch (e: Exception) { + FileLogger.e(TAG, "启动音频处理器失败") + e.printStackTrace() + } + } + } + + // 停止自定义音频处理 + private fun stopCustomAudioProcessing() { + if (customAudioProcessor != null) { + try { + customAudioProcessor?.stopProcessing() + customAudioProcessor = null + } catch (e: Exception) { + FileLogger.e(TAG, "停止音频处理器失败: ${e.message}") + e.printStackTrace() + } + } + } + + // 修改认证取消事件处理代码 + private fun setupCancelledEventHandler(callback: ContinuousRecognizeCallback) { + recognizer?.canceled?.addEventListener( + EventHandler { _, event -> + val errorDetails = try { + event.errorDetails ?: "未知错误" + } catch (e: Exception) { + "未知错误" + } + val reason = event.reason.toString() + FileLogger.e(TAG, "识别取消: $errorDetails") + isContinuousRecognitionActive = false + stopCustomAudioProcessing() + callback.onCanceled(reason, errorDetails) + callback.onError("识别取消: $errorDetails") // 兼容旧接口 + } + ) + } + + // 自定义音频处理器 + private inner class CustomAudioProcessor(private val pushStream: PushAudioInputStream) { + private val isProcessing = AtomicBoolean(false) + private var audioRecord: AudioRecord? = null + private var echoCanceler: AcousticEchoCanceler? = null + private var noiseSuppressor: NoiseSuppressor? = null + private var automaticGainControl: AutomaticGainControl? = null + + // 音频配置 + private val sampleRate = 16000 // 16kHz，适合语音识别 + private val channelConfig = AudioFormat.CHANNEL_IN_MONO + private val audioFormat = AudioFormat.ENCODING_PCM_16BIT + + // 计算最小 buffer 大小 + private val bufferSize = AudioRecord.getMinBufferSize( + sampleRate, channelConfig, audioFormat + ) + + // 启动音频处理 + fun startProcessing() { + if (isProcessing.get()) return + + // 创建录音对象 + try { + audioRecord = AudioRecord( + MediaRecorder.AudioSource.VOICE_RECOGNITION, + sampleRate, + channelConfig, + audioFormat, + bufferSize * 2 // 使用更大的缓冲区以确保不会丢失数据 + ) + + // 创建音频处理效果 + if (AcousticEchoCanceler.isAvailable()) { + echoCanceler = AcousticEchoCanceler.create(audioRecord!!.audioSessionId) + echoCanceler?.enabled = true + } + + if (NoiseSuppressor.isAvailable()) { + noiseSuppressor = NoiseSuppressor.create(audioRecord!!.audioSessionId) + noiseSuppressor?.enabled = true + } + + if (AutomaticGainControl.isAvailable()) { + automaticGainControl = AutomaticGainControl.create(audioRecord!!.audioSessionId) + automaticGainControl?.enabled = true + } + + // 开始录音 + audioRecord?.startRecording() + + // 处理线程 + Thread { + android.os.Process.setThreadPriority(Process.THREAD_PRIORITY_AUDIO) + processAudio() + }.start() + + isProcessing.set(true) + FileLogger.d(TAG, "音频处理已启动") + } catch (e: Exception) { + FileLogger.e(TAG, "创建音频处理器失败: ${e.message}") + releaseAudioResources() + throw e + } + } + + // 停止音频处理 + fun stopProcessing() { + if (!isProcessing.get()) return + + isProcessing.set(false) + releaseAudioResources() + FileLogger.d(TAG, "音频处理已停止") + } + + // 释放音频资源 + private fun releaseAudioResources() { + try { + audioRecord?.stop() + + echoCanceler?.release() + echoCanceler = null + + noiseSuppressor?.release() + noiseSuppressor = null + + automaticGainControl?.release() + automaticGainControl = null + + audioRecord?.release() + audioRecord = null + } catch (e: Exception) { + FileLogger.e(TAG, "释放音频资源失败: ${e.message}") + } + } + + // 音频处理线程 + private fun processAudio() { + // 设置线程优先级 + try { + Process.setThreadPriority(Process.THREAD_PRIORITY_URGENT_AUDIO) + } catch (e: Exception) { + FileLogger.e(TAG, "设置线程优先级失败") + } + + val buffer = ByteArray(bufferSize) + + while (isProcessing.get()) { + try { + val readSize = audioRecord?.read(buffer, 0, buffer.size) ?: -1 + + if (readSize > 0) { + // 修复: 只传入buffer，不传readSize + // 创建新的byte数组，只包含读取到的数据 + val audioData = buffer.copyOfRange(0, readSize) + pushStream.write(audioData) + } + + // 适当休眠，避免占用过多 CPU + Thread.sleep(5) + } catch (e: Exception) { + if (isProcessing.get()) { + FileLogger.e(TAG, "处理音频数据异常: ${e.message}") + } + break + } + } + } + } + + // 一次性识别回调接口 + interface RecognizeCallback { + fun onResult(text: String, detectedLanguage: String) + fun onError(error: String) + } + + // 连续识别回调接口 + interface ContinuousRecognizeCallback { + fun onResult(text: String, detectedLanguage: String) + fun onRecognizing(recognizing: String, detectedLanguage: String) + fun onSessionStarted() + fun onSessionStopped() + fun onCanceled(reason: String, errorDetails: String) + fun onError(error: String) + + // 兼容旧版本的接口 + fun onSuccess(message: String) {} + } +} \ No newline at end of file diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt new file mode 100644 index 000000000..090a72efc --- /dev/null +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt @@ -0,0 +1,297 @@ +package com.yunqiinnovation.azure_speech + +import android.content.Context +import android.app.Activity +import android.os.Handler +import android.os.Looper +import androidx.annotation.NonNull +import com.yunqiinnovation.azure_speech.utils.FileLogger + +import io.flutter.embedding.engine.plugins.FlutterPlugin +import io.flutter.plugin.common.MethodCall +import io.flutter.plugin.common.MethodChannel +import io.flutter.plugin.common.MethodChannel.MethodCallHandler +import io.flutter.plugin.common.MethodChannel.Result +import io.flutter.plugin.common.EventChannel + +/** AzureSpeechPlugin */ +class AzureSpeechPlugin: FlutterPlugin { + private val TAG = "AzureSpeechPlugin" + private lateinit var context: Context + private val mainHandler = Handler(Looper.getMainLooper()) + + // ASR相关 + private lateinit var asrChannel : MethodChannel + private lateinit var asrEventChannel: EventChannel + private var asrEventSink: EventChannel.EventSink? = null + private lateinit var azureAsrHelper: AzureAsrHelper + + // TTS相关 + private lateinit var ttsChannel : MethodChannel + private lateinit var azureTtsHelper: AzureTtsHelper + + // ASR 事件发送方法 + private fun sendAsrEvent(event: Map) { + FileLogger.d(TAG, "发送ASR事件: $event") + if (asrEventSink == null) { + FileLogger.w(TAG, "无法发送ASR事件：事件通道未准备好") + return + } + + mainHandler.post { + try { + asrEventSink?.success(event) + FileLogger.d(TAG, "ASR事件发送成功") + } catch (e: Exception) { + FileLogger.e(TAG, "发送ASR事件失败: ${e.message}") + } + } + } + + override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { + context = flutterPluginBinding.applicationContext + + // 初始化ASR通道 + asrChannel = MethodChannel(flutterPluginBinding.binaryMessenger, "azure_speech/asr") + asrChannel.setMethodCallHandler(AsrMethodHandler()) + + // 初始化TTS通道 + ttsChannel = MethodChannel(flutterPluginBinding.binaryMessenger, "azure_speech/tts") + ttsChannel.setMethodCallHandler(TtsMethodHandler()) + + // 初始化ASR事件通道 + asrEventChannel = EventChannel(flutterPluginBinding.binaryMessenger, "azure_speech/asr_events") + asrEventChannel.setStreamHandler(object : EventChannel.StreamHandler { + override fun onListen(arguments: Any?, events: EventChannel.EventSink?) { + asrEventSink = events + } + + override fun onCancel(arguments: Any?) { + asrEventSink = null + } + }) + + // 初始化Azure语音服务 + azureTtsHelper = AzureTtsHelper(context) + azureAsrHelper = AzureAsrHelper(context) + } + + // ASR方法处理器 + inner class AsrMethodHandler : MethodCallHandler { + override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) { + when (call.method) { + "initialize" -> { + val subscriptionKey = call.argument("subscriptionKey") ?: "" + val region = call.argument("region") ?: "" + val supportedLanguages = call.argument>("supportedLanguages") ?: listOf("zh-CN") + + try { + val success = azureAsrHelper.initialize(subscriptionKey, region, supportedLanguages.toTypedArray()) + result.success(success) + } catch (e: Exception) { + result.error("INITIALIZATION_ERROR", e.message, null) + } + } + "recognizeOnce" -> { + azureAsrHelper.recognizeOnce(object : AzureAsrHelper.RecognizeCallback { + override fun onResult(text: String, detectedLanguage: String) { + mainHandler.post { + result.success(mapOf( + "text" to text, + "detectedLanguage" to detectedLanguage + )) + } + } + + override fun onError(error: String) { + mainHandler.post { + result.error("RECOGNITION_ERROR", error, null) + } + } + }) + } + "startContinuousRecognition" -> { + // 确保事件通道已准备好 + if (asrEventSink == null) { + result.error("EVENT_CHANNEL_NOT_READY", "事件通道未准备好，无法开始连续识别", null) + return + } + + val success = azureAsrHelper.startContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { + override fun onResult(text: String, detectedLanguage: String) { + sendAsrEvent(mapOf( + "type" to "result", + "text" to text, + "detectedLanguage" to detectedLanguage + )) + } + + override fun onRecognizing(recognizing: String, detectedLanguage: String) { + sendAsrEvent(mapOf( + "type" to "recognizing", + "text" to recognizing, + "detectedLanguage" to detectedLanguage + )) + } + + override fun onSessionStarted() { + sendAsrEvent(mapOf("type" to "sessionStarted")) + } + + override fun onSessionStopped() { + sendAsrEvent(mapOf("type" to "sessionStopped")) + } + + override fun onCanceled(reason: String, errorDetails: String) { + sendAsrEvent(mapOf( + "type" to "canceled", + "reason" to reason, + "errorDetails" to errorDetails + )) + } + + override fun onError(error: String) { + sendAsrEvent(mapOf("type" to "error", "message" to error)) + } + + override fun onSuccess(message: String) { + sendAsrEvent(mapOf("type" to "success", "message" to message)) + } + }) + result.success(success) + } + "stopContinuousRecognition" -> { + try { + if (!azureAsrHelper.isContinuousRecognitionActive()) { + result.success(true) + return + } + + val success = azureAsrHelper.stopContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { + override fun onResult(text: String, detectedLanguage: String) {} + override fun onRecognizing(recognizing: String, detectedLanguage: String) {} + override fun onSessionStarted() {} + override fun onSessionStopped() {} + override fun onCanceled(reason: String, errorDetails: String) {} + override fun onError(error: String) { + mainHandler.post { + result.error("STOP_ERROR", error, null) + } + } + override fun onSuccess(message: String) {} + }) + result.success(success) + } catch (e: Exception) { + result.error("STOP_ERROR", e.message, null) + } + } + "isContinuousRecognitionActive" -> { + result.success(azureAsrHelper.isContinuousRecognitionActive()) + } + "dispose" -> { + azureAsrHelper.dispose() + result.success(true) + } + else -> { + result.notImplemented() + } + } + } + } + + // TTS方法处理器 + inner class TtsMethodHandler : MethodCallHandler { + override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) { + when (call.method) { + "initialize" -> { + val subscriptionKey = call.argument("subscriptionKey") ?: "" + val region = call.argument("region") ?: "" + val language = call.argument("language") ?: "zh-CN" + + val success = azureTtsHelper.initialize(subscriptionKey, region, language) + result.success(success) + } + "setVoice" -> { + val voiceName = call.argument("voiceName") ?: return result.error("INVALID_ARGUMENTS", "语音名称不能为空", null) + result.success(azureTtsHelper.setVoice(voiceName)) + } + "setSpeechParams" -> { + val rate = call.argument("rate") ?: 0 + val pitch = call.argument("pitch") ?: 0 + val volume = call.argument("volume") ?: 100 + result.success(azureTtsHelper.setSpeechParams(rate, pitch, volume)) + } + "setAudioOutputType" -> { + val outputTypeStr = call.argument("outputType") ?: "speaker" + val outputType = when (outputTypeStr.lowercase()) { + "speaker" -> AzureTtsHelper.AudioOutputType.SPEAKER + "earpiece" -> AzureTtsHelper.AudioOutputType.EARPIECE + "auto" -> AzureTtsHelper.AudioOutputType.AUTO + else -> AzureTtsHelper.AudioOutputType.SPEAKER + } + result.success(azureTtsHelper.setAudioOutputType(outputType)) + } + "speakText" -> { + val text = call.argument("text") ?: return result.error("INVALID_ARGUMENTS", "文本不能为空", null) + + azureTtsHelper.speakText(text, object : AzureTtsHelper.TTSCallback { + override fun onSuccess(message: String) { + mainHandler.post { + result.success(true) + } + } + + override fun onError(error: String) { + mainHandler.post { + result.error("SPEAK_ERROR", error, null) + } + } + }) + } + "speakSsml" -> { + val ssml = call.argument("ssml") ?: return result.error("INVALID_ARGUMENTS", "SSML不能为空", null) + + azureTtsHelper.speakSsml(ssml, object : AzureTtsHelper.TTSCallback { + override fun onSuccess(message: String) { + mainHandler.post { + result.success(true) + } + } + + override fun onError(error: String) { + mainHandler.post { + result.error("SPEAK_ERROR", error, null) + } + } + }) + } + "stopSpeaking" -> { + result.success(azureTtsHelper.stopSpeaking()) + } + "isSpeaking" -> { + result.success(azureTtsHelper.isSpeaking()) + } + "dispose" -> { + azureTtsHelper.dispose() + result.success(true) + } + else -> { + result.notImplemented() + } + } + } + } + + override fun onDetachedFromEngine(@NonNull binding: FlutterPlugin.FlutterPluginBinding) { + asrChannel.setMethodCallHandler(null) + ttsChannel.setMethodCallHandler(null) + asrEventChannel.setStreamHandler(null) + + try { + azureTtsHelper.dispose() + azureAsrHelper.dispose() + } catch (e: Exception) { + FileLogger.e(TAG, "Dispose resources error: ${e.message}") + } + } +} \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/example/deep_voice/AzureTtsHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt similarity index 98% rename from android/app/src/main/kotlin/com/example/deep_voice/AzureTtsHelper.kt rename to local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt index 31971c2a4..0ae180fa9 100644 --- a/android/app/src/main/kotlin/com/example/deep_voice/AzureTtsHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt @@ -1,11 +1,11 @@ -package com.yunqiinnovation.deepsound +package com.yunqiinnovation.azure_speech import android.content.Context import android.media.AudioAttributes import android.media.AudioDeviceInfo import android.media.AudioManager import android.os.Build -import com.yunqiinnovation.deepsound.core.utils.FileLogger +import com.yunqiinnovation.azure_speech.utils.FileLogger import com.microsoft.cognitiveservices.speech.* import com.microsoft.cognitiveservices.speech.audio.* import java.util.concurrent.Future @@ -50,16 +50,16 @@ class AzureTtsHelper(private val context: Context) { * 初始化 TTS 引擎 * * @param subscriptionKey Azure 语音服务订阅密钥 - * @param serviceRegion Azure 语音服务区域 + * @param region Azure 语音服务区域 * @param language 可选，默认语言，默认为 "zh-CN" */ - fun initialize(subscriptionKey: String, serviceRegion: String, language: String = "zh-CN"): Boolean { + fun initialize(subscriptionKey: String, region: String, language: String = "zh-CN"): Boolean { try { audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager // 创建语音配置 - speechConfig = SpeechConfig.fromSubscription(subscriptionKey, serviceRegion) + speechConfig = SpeechConfig.fromSubscription(subscriptionKey, region) // 设置语音合成输出格式为高质量音频 speechConfig?.setSpeechSynthesisOutputFormat(SpeechSynthesisOutputFormat.Riff24Khz16BitMonoPcm) @@ -96,7 +96,7 @@ class AzureTtsHelper(private val context: Context) { return true } catch (e: Exception) { - FileLogger.e(TAG, "TTS 引擎初始化失败: ${e.message}, ${serviceRegion}") + FileLogger.e(TAG, "TTS 引擎初始化失败: ${e.message}, ${region}") e.printStackTrace() return false } diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/utils/FileLogger.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/utils/FileLogger.kt new file mode 100644 index 000000000..d210ec48f --- /dev/null +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/utils/FileLogger.kt @@ -0,0 +1,45 @@ +package com.yunqiinnovation.azure_speech.utils + +import android.util.Log + +/** + * 简单的文件日志工具类 + */ +object FileLogger { + private const val TAG_PREFIX = "AzureSpeech_" + + /** + * 记录调试信息 + */ + fun d(tag: String, message: String) { + Log.d("$TAG_PREFIX$tag", message) + } + + /** + * 记录信息 + */ + fun i(tag: String, message: String) { + Log.i("$TAG_PREFIX$tag", message) + } + + /** + * 记录警告信息 + */ + fun w(tag: String, message: String) { + Log.w("$TAG_PREFIX$tag", message) + } + + /** + * 记录错误信息 + */ + fun e(tag: String, message: String) { + Log.e("$TAG_PREFIX$tag", message) + } + + /** + * 记录异常 + */ + fun e(tag: String, message: String, throwable: Throwable) { + Log.e("$TAG_PREFIX$tag", message, throwable) + } +} \ No newline at end of file diff --git a/local_plugins/azure_speech/ios/Classes/AzureAsrHelper.swift b/local_plugins/azure_speech/ios/Classes/AzureAsrHelper.swift new file mode 100644 index 000000000..d3dd8d0b8 --- /dev/null +++ b/local_plugins/azure_speech/ios/Classes/AzureAsrHelper.swift @@ -0,0 +1,461 @@ +import Foundation +import AVFoundation +import MicrosoftCognitiveServicesSpeech + +/// Azure 语音识别辅助类 +class AzureAsrHelper: NSObject { + private var recognizer: SPXSpeechRecognizer? + private var speechConfig: SPXSpeechConfig? + private var audioConfig: SPXAudioConfig? + private var initialized = false + private var isContinuousRecognitionActive = false + private var currentLanguage = "zh-CN" + private var subscriptionKey = "" + private var serviceRegion = "" + private var isAutoDetectLanguage = false + private var supportedLanguages = ["zh-CN", "en-US"] + + // 音频会话管理 + private let audioSession = AVAudioSession.sharedInstance() + + // 事件回调 + private var eventHandler: (([String: Any]) -> Void)? + + /// 设置事件处理器 + /// + /// - Parameter handler: 事件处理回调 + func setEventHandler(_ handler: @escaping ([String: Any]) -> Void) { + self.eventHandler = handler + } + + /// 初始化语音识别服务 + /// + /// - Parameters: + /// - speechSubscriptionKey: Azure 语音服务订阅密钥 + /// - serviceRegion: Azure 语音服务区域 + /// - supportedLanguages: 支持的语言列表，默认为 ["zh-CN", "en-US"] + /// - Returns: 是否初始化成功 + func initialize(speechSubscriptionKey: String, serviceRegion: String, supportedLanguages: [String] = ["zh-CN", "en-US"]) -> Bool { + print("[AzureAsrHelper] 初始化 Azure 语音服务") + + // 检查配置是否为空 + if speechSubscriptionKey.isEmpty || serviceRegion.isEmpty { + print("[AzureAsrHelper] 错误: Azure 配置信息不完整") + return false + } + + // 释放之前的资源 + dispose() + + // 保存配置 + self.subscriptionKey = speechSubscriptionKey + self.serviceRegion = serviceRegion + + // 设置语言 + if supportedLanguages.isEmpty { + print("[AzureAsrHelper] 警告: 传入的支持语言列表为空，将使用默认语言") + } else { + self.supportedLanguages = supportedLanguages + } + + // 根据支持的语言数量决定是否启用自动语言检测 + self.isAutoDetectLanguage = supportedLanguages.count >= 2 + + // 如果只有一种语言，设置为当前语言 + if !isAutoDetectLanguage && !supportedLanguages.isEmpty { + self.currentLanguage = supportedLanguages[0] + } + + // 创建语音配置 + do { + speechConfig = try SPXSpeechConfig(subscription: speechSubscriptionKey, region: serviceRegion) + + // 设置语言配置 + if isAutoDetectLanguage { + // 设置自动语言检测 + try speechConfig?.setPropertyTo("Continuous", byId: SPXPropertyId.SpeechServiceConnection_LanguageIdMode) + } else { + // 设置指定的识别语言 + speechConfig?.speechRecognitionLanguage = currentLanguage + } + + // 创建音频配置 - 使用默认麦克风 + audioConfig = SPXAudioConfig.default() + + // 创建识别器 + if isAutoDetectLanguage { + let autoDetectConfig = try SPXAutoDetectSourceLanguageConfiguration(sourceLanguages: supportedLanguages) + recognizer = try SPXSpeechRecognizer(speechConfiguration: speechConfig!, autoDetectSourceLanguageConfiguration: autoDetectConfig, audioConfiguration: audioConfig!) + } else { + recognizer = try SPXSpeechRecognizer(speechConfiguration: speechConfig!, audioConfiguration: audioConfig!) + } + + // 配置音频会话 + try configureAudioSession() + + initialized = true + print("[AzureAsrHelper] Azure 语音服务初始化成功") + return true + } catch { + print("[AzureAsrHelper] 错误: 创建识别器失败: \(error.localizedDescription)") + return false + } + } + + /// 配置音频会话 + private func configureAudioSession() throws { + print("[AzureAsrHelper] 开始配置音频会话...") + + do { + // 设置音频会话类别和模式 + try audioSession.setCategory(.record, mode: .measurement, options: [.duckOthers, .allowBluetooth]) + try audioSession.setActive(true, options: .notifyOthersOnDeactivation) + } catch { + print("[AzureAsrHelper] 警告: 通过AudioSessionManager配置音频会话失败") + throw error + } + } + + /// 一次性语音识别 + /// + /// - Parameter completion: 完成回调，返回是否成功、识别文本、识别语言和可能的错误信息 + func recognizeOnce(completion: @escaping (Bool, String?, String?, String?) -> Void) { + if !initialized { + completion(false, nil, nil, "语音服务未初始化") + return + } + + // 重置 recognizer + if !resetRecognizer() { + completion(false, nil, nil, "重置识别器失败") + return + } + + do { + // 激活音频会话 + try audioSession.setActive(true) + + // 添加识别事件处理 + recognizer?.addRecognizedEventHandler { [weak self] _, event in + guard let self = self else { return } + + if event.result.reason == .recognizedSpeech { + let detectedLanguage = self.getDetectedLanguage(from: event.result) + print("[AzureAsrHelper] 识别结果: \(event.result.text ?? ""), 语言: \(detectedLanguage)") + completion(true, event.result.text, detectedLanguage, nil) + } + } + + recognizer?.addRecognizingEventHandler { [weak self] _, event in + guard let self = self else { return } + + if event.result.reason == .recognizingSpeech { + let detectedLanguage = self.getDetectedLanguage(from: event.result) + // print("[AzureAsrHelper] 识别中: \(event.result.text ?? ""), 语言: \(detectedLanguage)") + } + } + + // 添加会话事件处理 + recognizer?.addSessionStartedEventHandler { _, _ in + print("[AzureAsrHelper] 识别会话已开始") + } + + recognizer?.addSessionStoppedEventHandler { _, _ in + print("[AzureAsrHelper] 识别会话已结束") + } + + // 添加取消事件处理 + recognizer?.addCanceledEventHandler { _, event in + if let cancellationDetails = try? SPXCancellationDetails(fromCanceledRecognitionResult: event.result) { + let errorDetails = cancellationDetails.errorDetails ?? "未知错误" + print("[AzureAsrHelper] 识别取消: \(errorDetails)") + completion(false, nil, nil, "识别取消: \(errorDetails)") + } + } + + // 执行识别 + let result = try recognizer?.recognizeOnceAsync().get() + + if result?.reason != .recognizedSpeech { + completion(false, nil, nil, "未能识别语音") + } + } catch { + completion(false, nil, nil, "识别异常: \(error.localizedDescription)") + } + } + + /// 重置识别器 + /// + /// - Returns: 是否重置成功 + private func resetRecognizer() -> Bool { + if !initialized { + print("[AzureAsrHelper] 错误: 语音服务未初始化") + return false + } + + // 检查配置是否为空 + if subscriptionKey.isEmpty || serviceRegion.isEmpty { + print("[AzureAsrHelper] 错误: Azure 配置信息不完整") + return false + } + + do { + // 释放之前的 recognizer + recognizer = nil + + // 创建音频配置 - 使用默认麦克风 + audioConfig = SPXAudioConfig.default() + + // 重新创建识别器 + if isAutoDetectLanguage { + let autoDetectConfig = try SPXAutoDetectSourceLanguageConfiguration(sourceLanguages: supportedLanguages) + recognizer = try SPXSpeechRecognizer(speechConfiguration: speechConfig!, autoDetectSourceLanguageConfiguration: autoDetectConfig, audioConfiguration: audioConfig!) + } else { + recognizer = try SPXSpeechRecognizer(speechConfiguration: speechConfig!, audioConfiguration: audioConfig!) + } + + return true + } catch { + print("[AzureAsrHelper] 错误: 重置识别器失败: \(error.localizedDescription)") + return false + } + } + + /// 获取检测到的语言 + /// + /// - Parameter result: 识别结果 + /// - Returns: 检测到的语言代码 + private func getDetectedLanguage(from result: SPXSpeechRecognitionResult) -> String { + if isAutoDetectLanguage { + do { + if let autoDetectResult = try SPXAutoDetectSourceLanguageResult(fromRecognitionResult: result) { + return autoDetectResult.language + } + return "" + } catch { + print("[AzureAsrHelper] 错误: 获取检测到的语言失败: \(error.localizedDescription)") + return "" + } + } else { + return currentLanguage + } + } + + /// 开始连续语音识别 + /// + /// - Returns: 是否成功启动连续识别 + func startContinuousRecognition() -> Bool { + if !initialized { + print("[AzureAsrHelper] 错误: 语音服务未初始化") + return false + } + + // 检查配置是否为空 + if subscriptionKey.isEmpty || serviceRegion.isEmpty { + print("[AzureAsrHelper] 错误: Azure 配置信息不完整") + return false + } + + // 如果已经在进行连续识别，直接返回 + if isContinuousRecognitionActive { + print("[AzureAsrHelper] 已经在进行连续识别中，忽略请求") + return true + } + + // 重置 recognizer + if !resetRecognizer() { + print("[AzureAsrHelper] 尝试重新创建识别器...") + return false + } + + do { + // 激活音频会话 + try audioSession.setActive(true) + + // 添加识别事件处理 + recognizer?.addRecognizedEventHandler { [weak self] _, event in + guard let self = self else { return } + + if event.result.reason == .recognizedSpeech { + let detectedLanguage = self.getDetectedLanguage(from: event.result) + let eventData: [String: Any] = [ + "eventType": "finalResult", + "text": event.result.text ?? "", + "language": detectedLanguage + ] + self.eventHandler?(eventData) + } + } + + // 识别中事件 + recognizer?.addRecognizingEventHandler { [weak self] _, event in + guard let self = self else { return } + + if event.result.reason == .recognizingSpeech { + let detectedLanguage = self.getDetectedLanguage(from: event.result) + let eventData: [String: Any] = [ + "eventType": "recognizing", + "text": event.result.text ?? "", + "language": detectedLanguage + ] + self.eventHandler?(eventData) + } + } + + // 会话事件 + recognizer?.addSessionStartedEventHandler { [weak self] _, _ in + guard let self = self else { return } + + let eventData: [String: Any] = [ + "eventType": "sessionStarted" + ] + self.eventHandler?(eventData) + } + + recognizer?.addSessionStoppedEventHandler { [weak self] _, _ in + guard let self = self else { return } + + self.isContinuousRecognitionActive = false + let eventData: [String: Any] = [ + "eventType": "sessionStopped" + ] + self.eventHandler?(eventData) + } + + // 取消事件 + recognizer?.addCanceledEventHandler { [weak self] _, event in + guard let self = self else { return } + + self.isContinuousRecognitionActive = false + var errorMessage = "未知错误" + + if let cancellationDetails = try? SPXCancellationDetails(fromCanceledRecognitionResult: event.result) { + errorMessage = cancellationDetails.errorDetails ?? "未知错误" + } + + let eventData: [String: Any] = [ + "eventType": "error", + "error": "识别取消: \(errorMessage)" + ] + self.eventHandler?(eventData) + } + + // 开始连续识别 + try recognizer?.startContinuousRecognition() + isContinuousRecognitionActive = true + print("[AzureAsrHelper] 连续识别已启动") + + return true + } catch { + print("[AzureAsrHelper] 错误: 开始连续识别失败: \(error.localizedDescription)") + return false + } + } + + /// 停止连续语音识别 + /// + /// - Returns: 是否成功停止连续识别 + func stopContinuousRecognition() -> Bool { + if !initialized { + print("[AzureAsrHelper] 错误: 语音服务未初始化") + return false + } + + if !isContinuousRecognitionActive { + print("[AzureAsrHelper] 未进行连续识别，忽略停止请求") + return true + } + + do { + print("[AzureAsrHelper] 停止连续语音识别") + + if recognizer == nil { + if isContinuousRecognitionActive { + print("[AzureAsrHelper] 警告: 识别器为空，但状态显示活跃") + } + isContinuousRecognitionActive = false + + // 通知停止成功 + let eventData: [String: Any] = [ + "eventType": "success", + "message": "连续识别已停止" + ] + eventHandler?(eventData) + return true + } + + // 停止连续识别 + try recognizer?.stopContinuousRecognition() + + // 延迟一点时间确保处理完成 + DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { [weak self] in + guard let self = self else { return } + + // 重置状态 + self.isContinuousRecognitionActive = false + + // 恢复音频会话 + do { + try self.audioSession.setActive(false, options: .notifyOthersOnDeactivation) + } catch { + // 忽略错误 + } + + print("[AzureAsrHelper] 连续识别已停止") + + // 通知停止成功 + let eventData: [String: Any] = [ + "eventType": "success", + "message": "连续识别已停止" + ] + self.eventHandler?(eventData) + } + + return true + } catch { + // 强制重置状态 + isContinuousRecognitionActive = false + print("[AzureAsrHelper] 警告: 停止连续识别失败: \(error.localizedDescription)") + + // 通知停止失败，但仍然视为处理完成 + let eventData: [String: Any] = [ + "eventType": "success", + "message": "连续识别已停止（但有错误）" + ] + eventHandler?(eventData) + + return false + } + } + + /// 释放资源 + func dispose() { + // 如果正在进行连续识别，先停止 + if isContinuousRecognitionActive { + _ = stopContinuousRecognition() + } + + // 恢复音频会话 + do { + try audioSession.setActive(false, options: .notifyOthersOnDeactivation) + } catch { + // 忽略错误 + } + + // 释放资源 + recognizer = nil + speechConfig = nil + audioConfig = nil + + initialized = false + isContinuousRecognitionActive = false + print("[AzureAsrHelper] 资源已释放") + } + + /// 检查连续识别是否处于活跃状态 + /// + /// - Returns: 是否正在进行连续识别 + func isContinuousRecognitionActive() -> Bool { + return isContinuousRecognitionActive + } +} \ No newline at end of file diff --git a/local_plugins/azure_speech/ios/Classes/AzureSpeechPlugin.swift b/local_plugins/azure_speech/ios/Classes/AzureSpeechPlugin.swift new file mode 100644 index 000000000..dcb2d57b8 --- /dev/null +++ b/local_plugins/azure_speech/ios/Classes/AzureSpeechPlugin.swift @@ -0,0 +1,182 @@ +import Flutter +import UIKit + +public class AzureSpeechPlugin: NSObject, FlutterPlugin { + private var ttsHelper: AzureTtsHelper? + private var asrHelper: AzureAsrHelper? + private var eventSink: FlutterEventSink? + + public static func register(with registrar: FlutterPluginRegistrar) { + let channel = FlutterMethodChannel(name: "azure_speech", binaryMessenger: registrar.messenger()) + let instance = AzureSpeechPlugin() + registrar.addMethodCallDelegate(instance, channel: channel) + + // 初始化事件通道 + let eventChannel = FlutterEventChannel(name: "azure_speech/asr_events", binaryMessenger: registrar.messenger()) + eventChannel.setStreamHandler(AsrStreamHandler(instance: instance)) + } + + override init() { + super.init() + ttsHelper = AzureTtsHelper() + asrHelper = AzureAsrHelper() + + // 设置ASR事件处理 + asrHelper?.setEventHandler { [weak self] event in + self?.handleAsrEvent(event) + } + } + + public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { + switch call.method { + // TTS相关方法 + case "initializeTts": + guard let args = call.arguments as? [String: Any], + let subscriptionKey = args["subscriptionKey"] as? String, + let serviceRegion = args["serviceRegion"] as? String else { + result(false) + return + } + + let language = args["language"] as? String ?? "zh-CN" + let success = ttsHelper?.initialize(speechSubscriptionKey: subscriptionKey, serviceRegion: serviceRegion, language: language) ?? false + result(success) + + case "setTtsVoice": + guard let args = call.arguments as? [String: Any], + let voiceName = args["voiceName"] as? String else { + result(false) + return + } + + let success = ttsHelper?.setVoice(voiceName: voiceName) ?? false + result(success) + + case "setTtsSpeechParams": + guard let args = call.arguments as? [String: Any], + let rate = args["rate"] as? Int, + let pitch = args["pitch"] as? Int, + let volume = args["volume"] as? Int else { + result(false) + return + } + + let success = ttsHelper?.setSpeechParams(rate: rate, pitch: pitch, volume: volume) ?? false + result(success) + + case "speakText": + guard let args = call.arguments as? [String: Any], + let text = args["text"] as? String else { + result(false) + return + } + + ttsHelper?.speakText(text: text) { success, _ in + result(success) + } + + case "stopSpeaking": + let success = ttsHelper?.stopSpeaking() ?? false + result(success) + + case "isSpeaking": + let speaking = ttsHelper?.isSpeaking() ?? false + result(speaking) + + case "setTtsAudioOutputType": + guard let args = call.arguments as? [String: Any], + let outputType = args["outputType"] as? String else { + result(false) + return + } + + var type: AzureTtsHelper.AudioOutputType = .auto + switch outputType.uppercased() { + case "SPEAKER": + type = .speaker + case "EARPIECE": + type = .earpiece + default: + type = .auto + } + + let success = ttsHelper?.setAudioOutputType(outputType: type) ?? false + result(success) + + // ASR相关方法 + case "initializeAsr": + guard let args = call.arguments as? [String: Any], + let subscriptionKey = args["subscriptionKey"] as? String, + let serviceRegion = args["serviceRegion"] as? String, + let supportedLanguages = args["supportedLanguages"] as? [String] else { + result(false) + return + } + + let success = asrHelper?.initialize(speechSubscriptionKey: subscriptionKey, serviceRegion: serviceRegion, supportedLanguages: supportedLanguages) ?? false + result(success) + + case "recognizeOnce": + asrHelper?.recognizeOnce { success, text, language, error in + var resultMap: [String: Any] = ["success": success] + if success { + resultMap["text"] = text + resultMap["language"] = language + } else { + resultMap["error"] = error + } + result(resultMap) + } + + case "startContinuousRecognition": + let success = asrHelper?.startContinuousRecognition() ?? false + result(success) + + case "stopContinuousRecognition": + let success = asrHelper?.stopContinuousRecognition() ?? false + result(success) + + case "isContinuousRecognitionActive": + let isActive = asrHelper?.isContinuousRecognitionActive() ?? false + result(isActive) + + case "dispose": + ttsHelper?.dispose() + asrHelper?.dispose() + result(nil) + + default: + result(FlutterMethodNotImplemented) + } + } + + // 设置事件接收器 + func setEventSink(_ sink: FlutterEventSink?) { + self.eventSink = sink + } + + // 处理ASR事件 + private func handleAsrEvent(_ event: [String: Any]) { + self.eventSink?(event) + } +} + +// ASR事件流处理器 +class AsrStreamHandler: NSObject, FlutterStreamHandler { + private weak var plugin: AzureSpeechPlugin? + + init(instance: AzureSpeechPlugin) { + self.plugin = instance + super.init() + } + + func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { + plugin?.setEventSink(events) + return nil + } + + func onCancel(withArguments arguments: Any?) -> FlutterError? { + plugin?.setEventSink(nil) + return nil + } +} \ No newline at end of file diff --git a/local_plugins/azure_speech/ios/Classes/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/Classes/AzureTtsHelper.swift new file mode 100644 index 000000000..4242a9037 --- /dev/null +++ b/local_plugins/azure_speech/ios/Classes/AzureTtsHelper.swift @@ -0,0 +1,330 @@ +import Foundation +import AVFoundation +import MicrosoftCognitiveServicesSpeech + +/// Azure 语音合成辅助类 +class AzureTtsHelper: NSObject { + private var synthesizer: SPXSpeechSynthesizer? + private var speechConfig: SPXSpeechConfig? + private var audioConfig: SPXAudioConfig? + private var initialized = false + private var speaking = false + + // 音频输出类型 + enum AudioOutputType { + case speaker // 扬声器 + case earpiece // 听筒 + case auto // 自动选择 + } + + // 当前设置 + private var currentVoiceName = "zh-CN-XiaoxiaoNeural" + private var currentSpeechRate = 0 + private var currentPitch = 0 + private var currentVolume = 100 + private var currentAudioOutputType: AudioOutputType = .auto + + // 音频会话管理 + private let audioSession = AVAudioSession.sharedInstance() + + /// 初始化 TTS 引擎 + /// + /// - Parameters: + /// - speechSubscriptionKey: Azure 语音服务订阅密钥 + /// - serviceRegion: Azure 语音服务区域 + /// - language: 可选，默认语言，默认为 "zh-CN" + /// - Returns: 是否初始化成功 + func initialize(speechSubscriptionKey: String, serviceRegion: String, language: String = "zh-CN") -> Bool { + print("[AzureTtsHelper] 初始化 Azure 语音服务") + + // 检查配置是否为空 + if speechSubscriptionKey.isEmpty || serviceRegion.isEmpty { + print("[AzureTtsHelper] 错误: Azure 配置信息不完整") + return false + } + + // 释放之前的资源 + dispose() + + do { + // 创建语音配置 + speechConfig = try SPXSpeechConfig(subscription: speechSubscriptionKey, region: serviceRegion) + + // 设置语音合成输出格式为高质量音频 + speechConfig?.setSpeechSynthesisOutputFormat(.riff24Khz16BitMonoPcm) + + // 设置默认语言 + speechConfig?.setSpeechSynthesisLanguage(language) + + // 设置默认语音 + speechConfig?.setSpeechSynthesisVoiceName(currentVoiceName) + + // 创建音频配置 - 使用默认扬声器 + audioConfig = SPXAudioConfig.default() + + // 创建语音合成器 + synthesizer = try SPXSpeechSynthesizer(speechConfig: speechConfig!, audioConfig: audioConfig!) + + initialized = true + + // 设置默认音频输出类型为自动 + setAudioOutputType(outputType: .auto) + + print("[AzureTtsHelper] TTS 引擎初始化成功") + return true + } catch { + print("[AzureTtsHelper] TTS 引擎初始化失败: \(error.localizedDescription)") + return false + } + } + + /// 设置音频输出设备类型 + /// + /// - Parameter outputType: 音频输出设备类型 + /// - Returns: 是否设置成功 + func setAudioOutputType(outputType: AudioOutputType) -> Bool { + if !initialized { + print("[AzureTtsHelper] TTS 引擎尚未初始化") + return false + } + + do { + currentAudioOutputType = outputType + + switch outputType { + case .speaker: + // 使用扬声器 + try audioSession.setCategory(.playback, mode: .default) + try audioSession.overrideOutputAudioPort(.speaker) + print("[AzureTtsHelper] 已设置音频输出设备为扬声器") + + case .earpiece: + // 使用听筒 + try audioSession.setCategory(.playback, mode: .voiceChat) + try audioSession.overrideOutputAudioPort(.none) + print("[AzureTtsHelper] 已设置音频输出设备为听筒") + + case .auto: + // 检查是否有耳机连接 + let outputs = audioSession.currentRoute.outputs + let hasHeadphones = outputs.contains { output in + return output.portType == .headphones || output.portType == .bluetoothA2DP || output.portType == .bluetoothHFP + } + + if hasHeadphones { + // 有耳机，使用耳机 + try audioSession.setCategory(.playback, mode: .default) + try audioSession.overrideOutputAudioPort(.none) + print("[AzureTtsHelper] 已设置音频输出设备为耳机") + } else { + // 无耳机，使用听筒 + try audioSession.setCategory(.playback, mode: .voiceChat) + try audioSession.overrideOutputAudioPort(.none) + print("[AzureTtsHelper] 已设置音频输出设备为听筒") + } + } + + try audioSession.setActive(true) + return true + } catch { + print("[AzureTtsHelper] 设置音频输出设备失败: \(error.localizedDescription)") + return false + } + } + + /// 设置语音 + /// + /// - Parameter voiceName: 语音名称，例如 "zh-CN-XiaoxiaoNeural" + /// - Returns: 是否设置成功 + func setVoice(voiceName: String) -> Bool { + if !initialized { + print("[AzureTtsHelper] TTS 引擎尚未初始化") + return false + } + + if voiceName == currentVoiceName { + print("[AzureTtsHelper] 已设置语音: \(voiceName)") + return true + } + + do { + currentVoiceName = voiceName + speechConfig?.setSpeechSynthesisVoiceName(voiceName) + + // 重新创建合成器 + synthesizer = try SPXSpeechSynthesizer(speechConfig: speechConfig!, audioConfig: audioConfig!) + + print("[AzureTtsHelper] 已设置语音: \(voiceName)") + return true + } catch { + print("[AzureTtsHelper] 设置语音失败: \(error.localizedDescription)") + return false + } + } + + /// 设置语音合成参数 + /// + /// - Parameters: + /// - rate: 语速，范围 -100 到 100，默认为 0 + /// - pitch: 音调，范围 -100 到 100，默认为 0 + /// - volume: 音量，范围 0 到 100，默认为 100 + /// - Returns: 是否设置成功 + func setSpeechParams(rate: Int = 0, pitch: Int = 0, volume: Int = 100) -> Bool { + if !initialized { + print("[AzureTtsHelper] TTS 引擎尚未初始化") + return false + } + + currentSpeechRate = rate + currentPitch = pitch + currentVolume = volume + + print("[AzureTtsHelper] 已设置语音参数: 语速=\(rate), 音调=\(pitch), 音量=\(volume)") + return true + } + + /// 合成文本为语音并播放 + /// + /// - Parameters: + /// - text: 要合成的文本 + /// - completion: 完成回调，返回是否成功和可能的错误信息 + func speakText(text: String, completion: @escaping (Bool, String?) -> Void) { + if !initialized { + print("[AzureTtsHelper] TTS 引擎尚未初始化") + completion(false, "TTS 引擎尚未初始化") + return + } + + do { + print("[AzureTtsHelper] 开始合成文本: \(text)") + + // 生成 SSML + let ssml = generateSsml(text: text) + + // 使用 SSML 合成语音 + speakSsml(ssml: ssml, completion: completion) + } catch { + print("[AzureTtsHelper] 语音合成异常: \(error.localizedDescription)") + completion(false, "语音合成异常: \(error.localizedDescription)") + } + } + + /// 生成 SSML 文本 + /// + /// - Parameter text: 要转换的文本 + /// - Returns: SSML 格式的文本 + private func generateSsml(text: String) -> String { + // 计算 SSML 参数 + let rateParam = currentSpeechRate == 0 ? "0%" : (currentSpeechRate < 0 ? "\(Int(Double(currentSpeechRate) * 0.9))%" : "\(currentSpeechRate)%") + let pitchParam = currentPitch == 0 ? "0%" : "\(Int(Double(currentPitch) * 0.5))%" + let volumeParam = "\(min(max(currentVolume, 0), 100))%" + + return """ + + + + \(text) + + + + """ + } + + /// 合成 SSML 为语音并播放 + /// + /// - Parameters: + /// - ssml: SSML 格式的文本 + /// - completion: 完成回调，返回是否成功和可能的错误信息 + private func speakSsml(ssml: String, completion: @escaping (Bool, String?) -> Void) { + if !initialized { + print("[AzureTtsHelper] TTS 引擎尚未初始化") + completion(false, "TTS 引擎尚未初始化") + return + } + + do { + print("[AzureTtsHelper] 开始合成 SSML") + + // 标记为正在播放 + speaking = true + + // 激活音频会话 + try audioSession.setActive(true) + + // 异步合成语音 + let result = try synthesizer!.speakSsml(ssml) + + switch result.reason { + case .synthesizingAudioCompleted: + print("[AzureTtsHelper] 语音合成完成") + speaking = false + completion(true, "语音合成完成") + case .canceled: + if let cancelDetails = try? SPXSpeechSynthesisCancellationDetails(fromResult: result) { + print("[AzureTtsHelper] 语音合成取消: \(cancelDetails.errorDetails ?? "未知错误")") + speaking = false + completion(false, "语音合成取消: \(cancelDetails.errorDetails ?? "未知错误")") + } else { + print("[AzureTtsHelper] 语音合成取消") + speaking = false + completion(false, "语音合成取消") + } + default: + print("[AzureTtsHelper] 语音合成失败: \(result.reason)") + speaking = false + completion(false, "语音合成失败: \(result.reason)") + } + } catch { + print("[AzureTtsHelper] 语音合成异常: \(error.localizedDescription)") + speaking = false + completion(false, "语音合成异常: \(error.localizedDescription)") + } + } + + /// 停止当前语音合成 + /// + /// - Returns: 是否停止成功 + func stopSpeaking() -> Bool { + if !initialized { + print("[AzureTtsHelper] TTS 引擎尚未初始化") + return false + } + + do { + try synthesizer?.stopSpeaking() + speaking = false + print("[AzureTtsHelper] 已停止语音合成") + return true + } catch { + print("[AzureTtsHelper] 停止语音合成失败: \(error.localizedDescription)") + return false + } + } + + /// 释放资源 + func dispose() { + do { + stopSpeaking() + + // 恢复音频会话 + try audioSession.setActive(false, options: .notifyOthersOnDeactivation) + + synthesizer = nil + speechConfig = nil + audioConfig = nil + + initialized = false + speaking = false + print("[AzureTtsHelper] TTS 引擎已释放") + } catch { + print("[AzureTtsHelper] 释放 TTS 引擎失败: \(error.localizedDescription)") + } + } + + /// 检查当前是否正在播放语音 + /// + /// - Returns: 是否正在播放语音 + func isSpeaking() -> Bool { + return speaking + } +} \ No newline at end of file diff --git a/local_plugins/azure_speech/pubspec.yaml b/local_plugins/azure_speech/pubspec.yaml new file mode 100644 index 000000000..5935bbce2 --- /dev/null +++ b/local_plugins/azure_speech/pubspec.yaml @@ -0,0 +1,32 @@ +name: azure_speech +description: Azure语音服务插件，包含TTS和ASR服务 +version: 0.0.1 +homepage: + +environment: + sdk: ">=2.17.0 <3.0.0" + flutter: ">=2.5.0" + +dependencies: + flutter: + sdk: flutter + +dev_dependencies: + flutter_test: + sdk: flutter + flutter_lints: ^2.0.0 + +# For information on the generic Dart part of this file, see the +# following page: https://dart.dev/tools/pub/pubspec + +# The following section is specific to Flutter packages. +flutter: + # This section identifies this Flutter project as a plugin project. + plugin: + platforms: + android: + package: com.yunqiinnovation.azure_speech + pluginClass: AzureSpeechPlugin + ios: + pluginClass: AzureSpeechPlugin + diff --git a/pubspec.yaml b/pubspec.yaml index e4b2f4e6e..3e1dd7c90 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -60,6 +60,8 @@ dependencies: dio: ^5.8.0+1 sqflite: ^2.4.2 path: ^1.9.1 + azure_speech: + path: local_plugins/azure_speech dev_dependencies: flutter_test: From a7a31369546b2d866074cc4bf3aa16b033632f11 Mon Sep 17 00:00:00 2001 From: wolfplus Date: Sat, 5 Apr 2025 21:29:31 +0100 Subject: [PATCH 02/14] add --- a | 18 + android/app/build.gradle.kts | 1 + .../yunqiinnovation/deepsound/MainActivity.kt | 10 +- .../deepsound/OpenAIService.kt | 606 ------------------ .../deepsound/VoiceFunctionHandler.kt | 2 +- .../deepsound/VoiceInteractionHandler.kt | 450 +++++++++++++ .../deepsound/VoiceInteractionService.kt | 438 ++----------- android/build.gradle.kts | 1 + android/settings.gradle.kts | 2 + .../services/voice_interaction_service.dart | 6 +- .../azure_speech/android/build.gradle.kts | 9 +- local_plugins/open_ai_service/README.md | 348 ++++++++++ .../open_ai_service/android/build.gradle.kts | 37 ++ .../android/settings.gradle.kts | 1 + .../android/src/main/AndroidManifest.xml | 11 + .../open_ai_service/OpenAIService.kt | 456 +++++++++++++ .../open_ai_service/OpenAIServicePlugin.kt | 322 ++++++++++ .../ios/Classes/OpenAIService.swift | 473 ++++++++++++++ .../ios/Classes/OpenAIServicePlugin.swift | 215 +++++++ .../open_ai_service/lib/open_ai_service.dart | 305 +++++++++ local_plugins/open_ai_service/pubspec.yaml | 26 + pubspec.yaml | 2 + 22 files changed, 2744 insertions(+), 995 deletions(-) create mode 100644 a delete mode 100644 android/app/src/main/kotlin/com/yunqiinnovation/deepsound/OpenAIService.kt create mode 100644 android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt create mode 100644 local_plugins/open_ai_service/README.md create mode 100644 local_plugins/open_ai_service/android/build.gradle.kts create mode 100644 local_plugins/open_ai_service/android/settings.gradle.kts create mode 100644 local_plugins/open_ai_service/android/src/main/AndroidManifest.xml create mode 100644 local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt create mode 100644 local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt create mode 100644 local_plugins/open_ai_service/ios/Classes/OpenAIService.swift create mode 100644 local_plugins/open_ai_service/ios/Classes/OpenAIServicePlugin.swift create mode 100644 local_plugins/open_ai_service/lib/open_ai_service.dart create mode 100644 local_plugins/open_ai_service/pubspec.yaml diff --git a/a b/a new file mode 100644 index 000000000..7133736ae --- /dev/null +++ b/a @@ -0,0 +1,18 @@ +curl 'https://ark.cn-beijing.volces.com/api/v3/bots/chat/completions' \ +-H "Authorization: Bearer 168deb3d-fd0c-4912-b9f1-aaee5c6743e6" \ +-H 'Content-Type: application/json' \ +-d '{ + "model": "bot-20250405211523-l7c9r", + "stream": true, + "stream_options": {"include_usage": true}, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Hello!" + } + ] +}' \ No newline at end of file diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 78b816b75..b0a39cbd4 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -92,6 +92,7 @@ dependencies { // 添加本地插件模块依赖 implementation(project(":azure_speech")) + implementation(project(":open_ai_service")) // 添加OkHttp依赖 implementation("com.squareup.okhttp3:okhttp:4.9.3") diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt index df36efb44..2367f8241 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt @@ -135,6 +135,7 @@ class MainActivity: FlutterActivity() { var volcanoAiApiKey: String = "" var openaiApiKey: String = "" var openaiBaseUrl: String? = null + var openaiModel: String? = null // 安全存储相关常量 private const val SECURE_PREFS_FILENAME = "deep_voice_secure_prefs" @@ -143,7 +144,9 @@ class MainActivity: FlutterActivity() { private const val KEY_VOLCANO_AI_API_KEY = "volcano_ai_api_key" private const val KEY_OPENAI_API_KEY = "openai_api_key" private const val KEY_OPENAI_BASE_URL = "openai_base_url" + private const val KEY_OPENAI_MODEL = "openai_model" private const val KEY_MCP_SERVER_ENDPOINT = "mcp_server_endpoint" + // 会话管理 private const val KEY_SESSION_ID = "session_id" private var currentSessionId = "" @@ -180,6 +183,7 @@ class MainActivity: FlutterActivity() { .putString(KEY_OPENAI_API_KEY, openaiApiKey) .putString(KEY_OPENAI_BASE_URL, openaiBaseUrl) .putString(KEY_SESSION_ID, currentSessionId) + .putString(KEY_OPENAI_MODEL, openaiModel) .apply() FileLogger.d("MainActivity", "密钥已安全保存到加密存储中") @@ -225,6 +229,7 @@ class MainActivity: FlutterActivity() { volcanoAiApiKey = sharedPreferences.getString(KEY_VOLCANO_AI_API_KEY, "") ?: "" openaiApiKey = sharedPreferences.getString(KEY_OPENAI_API_KEY, "") ?: "" openaiBaseUrl = sharedPreferences.getString(KEY_OPENAI_BASE_URL, null) + openaiModel = sharedPreferences.getString(KEY_OPENAI_MODEL, null) FileLogger.d("MainActivity", "已从加密存储加载密钥") // 检查是否成功获取所有必要密钥 @@ -460,7 +465,7 @@ class MainActivity: FlutterActivity() { val region = call.argument("azure_speech_region") ?: "" val openaiKey = call.argument("openai_api_key") ?: "" val baseUrl = call.argument("openai_base_url") - + val model = call.argument("openai_model") // 设置 Azure Speech 和 AI 配置 azureSpeechKey = key azureSpeechRegion = region @@ -468,6 +473,9 @@ class MainActivity: FlutterActivity() { if (baseUrl != null) { openaiBaseUrl = baseUrl } + if (model != null) { + openaiModel = model + } // 保存密钥到安全存储 saveKeysToSecureStorage(applicationContext) diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/OpenAIService.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/OpenAIService.kt deleted file mode 100644 index 39907e6d5..000000000 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/OpenAIService.kt +++ /dev/null @@ -1,606 +0,0 @@ -package com.yunqiinnovation.deepsound - -import android.util.Log -import okhttp3.* -import okhttp3.MediaType.Companion.toMediaTypeOrNull -import okhttp3.RequestBody.Companion.toRequestBody -import org.json.JSONArray -import org.json.JSONObject -import java.io.IOException -import java.util.concurrent.TimeUnit -import com.yunqiinnovation.deepsound.core.utils.FileLogger - -/** - * OpenAI服务的原生实现 - */ -class OpenAIService() { - private val TAG = "OpenAIService" - private var baseUrl = "https://api.openai.com/v1/chat/completions" - private val client = OkHttpClient.Builder() - .connectTimeout(30, TimeUnit.SECONDS) - .readTimeout(30, TimeUnit.SECONDS) - .writeTimeout(30, TimeUnit.SECONDS) - .build() - - private var apiKey: String = "" - private var isInitialized = false - private var model: String = "doubao-1-5-lite-32k-250115" // 默认模型 - - // 用于存储注册的函数 - private val registeredFunctions = mutableListOf() - - - /** - * 初始化OpenAI服务 - */ - fun initialize(apiKey: String, baseUrl: String = ""): Boolean { - this.apiKey = apiKey - if (baseUrl.isNotEmpty()) { - this.baseUrl = baseUrl - } - isInitialized = apiKey.isNotEmpty() - return isInitialized - } - - /** - * 注册函数 - */ - fun registerFunction(name: String, description: String, parameters: JSONObject): Boolean { - try { - val function = JSONObject().apply { - put("name", name) - put("description", description) - put("parameters", parameters) - } - - // 检查是否已存在相同名称的函数 - val existingIndex = registeredFunctions.indexOfFirst { - it.getString("name") == name - } - - if (existingIndex >= 0) { - // 如果已存在，则替换 - registeredFunctions[existingIndex] = function - } else { - // 如果不存在，则添加 - registeredFunctions.add(function) - } - - return true - } catch (e: Exception) { - return false - } - } - - /** - * 发送消息（非流式输出） - */ - @Throws(OpenAIException::class) - fun sendMessage(messages: JSONArray, systemPrompt: String): String { - if (!isInitialized || apiKey.isEmpty()) { - throw OpenAIException("OpenAI服务未初始化") - } - - val fullMessages = JSONArray().apply { - put(JSONObject().apply { - put("role", "system") - put("content", systemPrompt) - }) - for (i in 0 until messages.length()) { - put(messages.getJSONObject(i)) - } - } - - val requestBody = JSONObject().apply { - put("model", model) - put("messages", fullMessages) - put("temperature", 0.7) - put("max_tokens", 2000) - put("stream", false) - - // 如果有注册的函数，则添加到请求中 - if (registeredFunctions.isNotEmpty()) { - val tools = JSONArray() - for (function in registeredFunctions) { - val tool = JSONObject().apply { - put("type", "function") - put("function", function) - } - tools.put(tool) - } - put("tools", tools) - } - } - - val mediaType = "application/json".toMediaTypeOrNull() - val request = Request.Builder() - .url(baseUrl) - .addHeader("Content-Type", "application/json") - .addHeader("Authorization", "Bearer $apiKey") - .post(requestBody.toString().toRequestBody(mediaType)) - .build() - - try { - client.newCall(request).execute().use { response -> - if (!response.isSuccessful) { - throw OpenAIException("API调用失败: ${response.code}") - } - - val responseBody = response.body?.string() ?: throw OpenAIException("Empty response") - val jsonResponse = JSONObject(responseBody) - - // 检查是否有函数调用 - if (jsonResponse.has("choices") && - jsonResponse.getJSONArray("choices").length() > 0) { - - val choice = jsonResponse.getJSONArray("choices").getJSONObject(0) - - // 检查是否是函数调用 - if (choice.has("message")) { - val message = choice.getJSONObject("message") - - // 检查是否有工具调用 - if (message.has("tool_calls")) { - val toolCalls = message.getJSONArray("tool_calls") - if (toolCalls.length() > 0) { - val toolCall = toolCalls.getJSONObject(0) - if (toolCall.has("function")) { - val function = toolCall.getJSONObject("function") - val functionCall = JSONObject().apply { - put("name", function.getString("name")) - put("arguments", function.getString("arguments")) - put("id", toolCall.getString("id")) - } - return functionCall.toString() - } - } - } - - // 如果没有工具调用，返回消息内容 - if (message.has("content")) { - return message.getString("content") - } - } - } - - throw OpenAIException("Invalid response format") - } - } catch (e: Exception) { - if (e is OpenAIException) throw e - throw OpenAIException("Failed to communicate with AI service: ${e.message}") - } - } - - /** - * 发送消息（流式输出） - */ - fun sendMessageStream(messages: JSONArray, systemPrompt: String, callback: StreamCallback) { - if (!isInitialized || apiKey.isEmpty()) { - callback.onError(OpenAIException("OpenAI服务未初始化")) - return - } - - val fullMessages = JSONArray().apply { - put(JSONObject().apply { - put("role", "system") - put("content", systemPrompt) - }) - for (i in 0 until messages.length()) { - put(messages.getJSONObject(i)) - } - } - - val requestBody = JSONObject().apply { - put("model", model) - put("messages", fullMessages) - put("temperature", 0.7) - put("max_tokens", 2000) - put("stream", true) - - // 如果有注册的函数，则添加到请求中 - if (registeredFunctions.isNotEmpty()) { - val tools = JSONArray() - for (function in registeredFunctions) { - val tool = JSONObject().apply { - put("type", "function") - put("function", function) - } - tools.put(tool) - } - put("tools", tools) - } - } - - val mediaType = "application/json".toMediaTypeOrNull() - val request = Request.Builder() - .url(baseUrl) - .addHeader("Content-Type", "application/json") - .addHeader("Authorization", "Bearer $apiKey") - .addHeader("Accept", "text/event-stream") - .post(requestBody.toString().toRequestBody(mediaType)) - .build() - - client.newCall(request).enqueue(object : Callback { - override fun onFailure(call: Call, e: IOException) { - callback.onError(OpenAIException(e.message ?: "请求失败")) - } - - override fun onResponse(call: Call, response: Response) { - if (!response.isSuccessful) { - callback.onError(OpenAIException("API调用失败: ${response.code}")) - return - } - - val responseBody = response.body ?: return - val source = responseBody.source() - - try { - // 预取数据到缓冲区 - source.request(Long.MAX_VALUE) - val bufferedSource = source.buffer - - // 用于存储函数调用的各个部分 - val finalToolCalls = mutableMapOf() - - while (!bufferedSource.exhausted()) { - val line = bufferedSource.readUtf8Line() ?: continue - - if (line.isEmpty()) continue - if (line.startsWith("data: ")) { - val data = line.substring(6) - if (data == "[DONE]") { - callback.onComplete() - break - } - - try { - val jsonData = JSONObject(data) - if (jsonData.has("choices") && - jsonData.getJSONArray("choices").length() > 0) { - - val choice = jsonData.getJSONArray("choices").getJSONObject(0) - - // 检查是否有delta - if (choice.has("delta")) { - val delta = choice.getJSONObject("delta") - - // 检查是否有工具调用 - if (delta.has("tool_calls")) { - val toolCalls = delta.getJSONArray("tool_calls") - for (i in 0 until toolCalls.length()) { - val toolCall = toolCalls.getJSONObject(i) - val index = toolCall.optInt("index", i) - - // 如果是新的工具调用，初始化 - if (!finalToolCalls.containsKey(index)) { - finalToolCalls[index] = ToolCallInfo() - } - - // 获取ID - if (toolCall.has("id")) { - finalToolCalls[index]?.id = toolCall.getString("id") - } - - // 处理函数信息 - if (toolCall.has("function")) { - val function = toolCall.getJSONObject("function") - - if (function.has("name")) { - finalToolCalls[index]?.name = function.getString("name") - } - - if (function.has("arguments")) { - finalToolCalls[index]?.arguments += function.getString("arguments") - } - } - } - continue - } - - // 如果有内容，发送给回调 - if (delta.has("content") && !delta.isNull("content")) { - val content = delta.getString("content") - callback.onToken(content) - } - } - } - } catch (e: Exception) { - // 忽略无效的JSON - continue - } - } - } - - // 处理完整的函数调用 - for ((_, toolCallInfo) in finalToolCalls) { - if (toolCallInfo.name.isNotEmpty()) { - try { - // 创建函数调用对象 - val functionCall = JSONObject().apply { - put("id", toolCallInfo.id) - put("name", toolCallInfo.name) - put("arguments", toolCallInfo.arguments.trim()) - } - - callback.onFunctionCall(functionCall) - } catch (e: Exception) { - // 出错时使用空参数 - val functionCall = JSONObject().apply { - put("id", toolCallInfo.id) - put("name", toolCallInfo.name) - put("arguments", "{}") - } - callback.onFunctionCall(functionCall) - } - } - } - } catch (e: Exception) { - callback.onError(OpenAIException("处理流式响应出错: ${e.message}")) - } finally { - response.close() - } - } - }) - } - - /** - * 发送函数调用结果 - */ - fun sendFunctionCallResult( - messages: JSONArray, - systemPrompt: String, - functionCall: JSONObject, - functionResult: String, - callback: StreamCallback - ) { - if (!isInitialized || apiKey.isEmpty()) { - callback.onError(OpenAIException("OpenAI服务未初始化")) - return - } - - // 构建完整的消息历史 - val fullMessages = JSONArray().apply { - // 添加系统提示 - put(JSONObject().apply { - put("role", "system") - put("content", systemPrompt) - }) - - // 添加历史消息 - for (i in 0 until messages.length()) { - put(messages.getJSONObject(i)) - } - - // 添加函数调用信息 - put(JSONObject().apply { - put("role", "assistant") - put("content", null) - put("tool_calls", JSONArray().apply { - put(JSONObject().apply { - put("id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) - put("type", "function") - put("function", JSONObject().apply { - put("name", functionCall.getString("name")) - put("arguments", functionCall.getString("arguments")) - }) - }) - }) - }) - - // 添加函数返回结果 - put(JSONObject().apply { - put("role", "tool") - put("tool_call_id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) - put("content", functionResult) - }) - } - - // 构建请求 - val requestBody = JSONObject().apply { - put("model", model) - put("messages", fullMessages) - put("temperature", 0.7) - put("max_tokens", 2000) - put("stream", true) - - // 如果有注册的函数，则添加到请求中 - if (registeredFunctions.isNotEmpty()) { - val tools = JSONArray() - for (function in registeredFunctions) { - val tool = JSONObject().apply { - put("type", "function") - put("function", function) - } - tools.put(tool) - } - put("tools", tools) - } - } - - val mediaType = "application/json".toMediaTypeOrNull() - val request = Request.Builder() - .url(baseUrl) - .addHeader("Content-Type", "application/json") - .addHeader("Authorization", "Bearer $apiKey") - .addHeader("Accept", "text/event-stream") - .post(requestBody.toString().toRequestBody(mediaType)) - .build() - - // 发送请求 - client.newCall(request).enqueue(object : Callback { - override fun onFailure(call: Call, e: IOException) { - callback.onError(OpenAIException(e.message ?: "请求失败")) - } - - override fun onResponse(call: Call, response: Response) { - if (!response.isSuccessful) { - callback.onError(OpenAIException("API调用失败: ${response.code}")) - return - } - - val responseBody = response.body ?: return - val source = responseBody.source() - - try { - // 预取数据到缓冲区 - source.request(Long.MAX_VALUE) - val bufferedSource = source.buffer - - // 用于存储函数调用的各个部分 - val finalToolCalls = mutableMapOf() - - while (!bufferedSource.exhausted()) { - val line = bufferedSource.readUtf8Line() ?: continue - - if (line.isEmpty()) continue - if (line.startsWith("data: ")) { - val data = line.substring(6) - if (data == "[DONE]") { - callback.onComplete() - break - } - - try { - val jsonData = JSONObject(data) - if (jsonData.has("choices") && - jsonData.getJSONArray("choices").length() > 0) { - - val choice = jsonData.getJSONArray("choices").getJSONObject(0) - - // 检查是否有delta - if (choice.has("delta")) { - val delta = choice.getJSONObject("delta") - - // 检查是否有工具调用 - if (delta.has("tool_calls")) { - val toolCalls = delta.getJSONArray("tool_calls") - for (i in 0 until toolCalls.length()) { - val toolCall = toolCalls.getJSONObject(i) - val index = toolCall.optInt("index", i) - - // 如果是新的工具调用，初始化 - if (!finalToolCalls.containsKey(index)) { - finalToolCalls[index] = ToolCallInfo() - } - - // 获取ID - if (toolCall.has("id")) { - finalToolCalls[index]?.id = toolCall.getString("id") - } - - // 处理函数信息 - if (toolCall.has("function")) { - val function = toolCall.getJSONObject("function") - - if (function.has("name")) { - finalToolCalls[index]?.name = function.getString("name") - } - - if (function.has("arguments")) { - finalToolCalls[index]?.arguments += function.getString("arguments") - } - } - } - continue - } - - // 如果有内容，发送给回调 - if (delta.has("content") && !delta.isNull("content")) { - val content = delta.getString("content") - callback.onToken(content) - } - } - } - } catch (e: Exception) { - // 忽略无效的JSON - continue - } - } - } - - // 处理完整的函数调用 - for ((_, toolCallInfo) in finalToolCalls) { - if (toolCallInfo.name.isNotEmpty()) { - try { - // 创建函数调用对象 - val functionCall = JSONObject().apply { - put("id", toolCallInfo.id) - put("name", toolCallInfo.name) - put("arguments", toolCallInfo.arguments.trim()) - } - - callback.onFunctionCall(functionCall) - } catch (e: Exception) { - // 出错时使用空参数 - val functionCall = JSONObject().apply { - put("id", toolCallInfo.id) - put("name", toolCallInfo.name) - put("arguments", "{}") - } - callback.onFunctionCall(functionCall) - } - } - } - } catch (e: Exception) { - callback.onError(OpenAIException("处理流式响应出错: ${e.message}")) - } finally { - response.close() - } - } - }) - } - - /** - * 创建用户消息 - */ - fun createUserMessage(content: String): JSONObject { - return JSONObject().apply { - put("role", "user") - put("content", content) - } - } - - /** - * 创建系统消息 - */ - fun createSystemMessage(content: String): JSONObject { - return JSONObject().apply { - put("role", "system") - put("content", content) - } - } - - /** - * 创建助手消息 - */ - fun createAssistantMessage(content: String): JSONObject { - return JSONObject().apply { - put("role", "assistant") - put("content", content) - } - } - - /** - * 流式输出回调接口 - */ - interface StreamCallback { - fun onToken(token: String) - fun onComplete() - fun onError(e: Exception) - fun onFunctionCall(functionCall: JSONObject) {} - } - - /** - * 用于存储工具调用信息的辅助类 - */ - private class ToolCallInfo { - var id: String = "" - var name: String = "" - var arguments: String = "" - } -} - -/** - * OpenAI异常 - */ -class OpenAIException(message: String) : Exception(message) \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt index a89441506..78a55b642 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt @@ -2,7 +2,7 @@ package com.yunqiinnovation.deepsound import org.json.JSONArray import org.json.JSONObject -import com.yunqiinnovation.deepsound.OpenAIService +import com.yunqiinnovation.open_ai_service.OpenAIService import com.yunqiinnovation.deepsound.core.utils.FileLogger /** diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt new file mode 100644 index 000000000..fa179c279 --- /dev/null +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt @@ -0,0 +1,450 @@ +package com.yunqiinnovation.deepsound + +import android.content.Context +import org.json.JSONArray +import org.json.JSONObject +import android.util.Log +import com.yunqiinnovation.deepsound.core.utils.FileLogger +import com.yunqiinnovation.azure_speech.AzureAsrHelper +import com.yunqiinnovation.azure_speech.AzureTtsHelper +import com.yunqiinnovation.open_ai_service.OpenAIService + + +/** + * 语音交互处理器 - 处理语音识别、TTS和AI对话相关逻辑 + */ +class VoiceInteractionHandler( + private val context: Context, + private val azureSpeechKey: String, + private val azureSpeechRegion: String, + private val openaiApiKey: String, + private val openaiBaseUrl: String = "", + private val openaiModel: String = "" +) { + private val TAG = "VoiceInteractionHandler" + + // Azure服务 + private var azureAsrHelper: AzureAsrHelper? = null + private var azureTtsHelper: AzureTtsHelper? = null + + // OpenAI服务 + private val openAIService = OpenAIService() + + // 系统提示词 + private val systemPrompt = """ + 你是一个智能语音助手，能够简洁明了地回答用户的问题。 +时刻关心用户的情绪和需求，主动提供鼓励和温暖。 + +语言风格活泼、亲切，能够幽默地互动，陪伴用户，缓解压力，增添生活乐趣。 + +请始终以用户为中心，保持回应的高效性、准确性和温暖体贴，成为用户真正的灵魂伴侣。 + + 当用户说"退出"、"再见"、"结束对话"等类似意图时，你应该使用exit_interaction函数来结束对话， + 并在结束前说一句友好的告别语，例如"再见，有需要随时找我"。 + """.trimIndent() + + // 语音功能处理 + private val voiceFunctionHandler = VoiceFunctionHandler(openAIService, systemPrompt) + + // 当前用户输入 + private var currentUserInput = "" + + // 状态 + private var isInitialized = false + var isRecognitionActive = false + private set + var isTtsSpeaking = false + private set + var hasSpeechDetected = false + private set + + // 回调 + private var callback: InteractionCallback? = null + + /** + * 初始化 + */ + fun initialize(): Boolean { + if (isInitialized) return true + + try { + // 初始化Azure ASR + azureAsrHelper = AzureAsrHelper(context).apply { + initialize(azureSpeechKey, azureSpeechRegion) + } + + // 初始化Azure TTS + azureTtsHelper = AzureTtsHelper(context).apply { + initialize(azureSpeechKey, azureSpeechRegion) + } + + // 初始化OpenAI服务 + openAIService.initialize(openaiApiKey, openaiBaseUrl, openaiModel) + + // 初始化语音功能处理器 + voiceFunctionHandler.initialize() + + isInitialized = true + return true + } catch (e: Exception) { + FileLogger.e(TAG, "初始化失败: ${e.message}", e) + return false + } + } + + /** + * 设置回调 + */ + fun setCallback(callback: InteractionCallback) { + this.callback = callback + } + + /** + * 开始语音识别 + */ + fun startRecognition() { + if (isRecognitionActive) return + + // 检查录音权限 + if (!checkRecordAudioPermission()) { + callback?.onError("需要录音权限，请在设置中授予权限") + return + } + + isRecognitionActive = true + hasSpeechDetected = false + notifyStateChanged() + + try { + azureAsrHelper?.startContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { + override fun onRecognizing(recognizing: String, detectedLanguage: String) { + if (recognizing.isNotEmpty()) { + hasSpeechDetected = true + stopTts() + notifyStateChanged() + } + } + + override fun onResult(result: String, detectedLanguage: String) { + if (result.isNotEmpty()) { + processWithOpenAI(result) + } + + // 重置状态，继续识别 + hasSpeechDetected = false + notifyStateChanged() + } + + override fun onSessionStarted() { + notifyStateChanged() + } + + override fun onSessionStopped() { + isRecognitionActive = false + notifyStateChanged() + } + + override fun onCanceled(reason: String, errorDetails: String) { + isRecognitionActive = false + notifyStateChanged() + } + + override fun onError(error: String) { + isRecognitionActive = false + callback?.onError("语音识别出错") + notifyStateChanged() + } + + override fun onSuccess(message: String) { + // 处理成功事件 + } + }) + } catch (e: Exception) { + isRecognitionActive = false + FileLogger.e(TAG, "启动语音识别失败: ${e.message}", e) + callback?.onError("启动语音识别失败") + notifyStateChanged() + } + } + + /** + * 停止语音识别 + */ + fun stopRecognition() { + if (!isRecognitionActive) return + + FileLogger.d(TAG, "停止语音识别") + + try { + azureAsrHelper?.stopContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { + override fun onResult(result: String, detectedLanguage: String) {} + override fun onRecognizing(recognizing: String, detectedLanguage: String) {} + override fun onSessionStarted() {} + override fun onSessionStopped() { + isRecognitionActive = false + FileLogger.d(TAG, "语音识别会话已停止") + notifyStateChanged() + } + override fun onCanceled(reason: String, errorDetails: String) { + isRecognitionActive = false + FileLogger.d(TAG, "语音识别已取消: $reason") + notifyStateChanged() + } + override fun onError(error: String) { + isRecognitionActive = false + FileLogger.e(TAG, "停止语音识别时出错: $error") + notifyStateChanged() + } + override fun onSuccess(message: String) { + isRecognitionActive = false + FileLogger.d(TAG, "语音识别已停止: $message") + notifyStateChanged() + } + }) + } catch (e: Exception) { + FileLogger.e(TAG, "停止语音识别异常: ${e.message}", e) + // 确保状态一致性 + isRecognitionActive = false + notifyStateChanged() + } + } + + /** + * 使用OpenAI处理语音识别结果 + */ + private fun processWithOpenAI(text: String) { + // 保存当前用户输入，用于后续同步聊天记录 + currentUserInput = text + + Thread { + try { + val messages = JSONArray().apply { + put(openAIService.createUserMessage(text)) + } + + // 创建响应构建器 + val responseBuilder = StringBuilder() + + openAIService.sendMessageStream( + messages = messages, + systemPrompt = systemPrompt, + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + // 累加响应内容 + responseBuilder.append(token) + } + + override fun onComplete() { + // 处理完整响应 + val response = responseBuilder.toString() + if (response.isNotEmpty()) { + // 播放AI回复 + Log.d(TAG, "AI 回复: $response") + + speakAIResponse(response) + + // 同步聊天记录到Flutter端 + sendChatHistoryUpdate("personal_assistant", text, response) + } + } + + override fun onError(e: Exception) { + FileLogger.e(TAG, "AI流式处理出错: ${e.message}", e) + callback?.onError("AI处理出错") + } + + override fun onFunctionCall(call: JSONObject) { + FileLogger.d(TAG, "收到函数调用请求: ${call.getString("name")}") + + // 使用函数处理器处理函数调用 + val handled = voiceFunctionHandler.handleFunctionCall( + functionCall = call, + messages = messages, + callback = object : VoiceFunctionHandler.FunctionCallCallback { + override fun onTokenReceived(token: String) { + responseBuilder.append(token) + } + + override fun onComplete() { + val response = responseBuilder.toString() + if (response.isNotEmpty()) { + // 播放AI回复 + Log.d(TAG, "AI Function Call 回复: $response") + speakAIResponse(response) + + // 同步聊天记录到Flutter端 + sendChatHistoryUpdate("personal_assistant", text, response) + } + notifyStateChanged() + } + + override fun onError(message: String) { + FileLogger.e(TAG, "函数处理出错: $message") + callback?.onError(message) + } + + override fun onFunctionCall(nestedCall: JSONObject) { + FileLogger.d(TAG, "收到嵌套函数调用: ${nestedCall.getString("name")}") + // 递归处理嵌套函数调用 + voiceFunctionHandler.handleFunctionCall( + functionCall = nestedCall, + messages = messages, + callback = this + ) + } + + override fun onExitWithMessage(farewell: String) { + // 播放退出消息 + speakAIResponse(farewell) + + // 同步聊天记录 + sendChatHistoryUpdate("personal_assistant", text, farewell) + + // 停止语音识别 + stopRecognition() + } + } + ) + + if (!handled) { + // 如果函数没有被处理，作为普通文本处理 + FileLogger.d(TAG, "函数未处理，作为普通文本处理") + speakAIResponse("我无法处理这个请求") + sendChatHistoryUpdate("personal_assistant", text, "我无法处理这个请求") + } + } + } + ) + + } catch (e: Exception) { + FileLogger.e(TAG, "AI处理出错: ${e.message}", e) + callback?.onError("AI处理出错") + } + }.start() + } + + /** + * 播放TTS + */ + fun playTts(text: String, callback: TtsCallback? = null) { + isTtsSpeaking = true + notifyStateChanged() + + azureTtsHelper?.speakText(text, object : AzureTtsHelper.TTSCallback { + override fun onSuccess(message: String) { + isTtsSpeaking = false + notifyStateChanged() + callback?.onComplete() + } + + override fun onError(error: String) { + isTtsSpeaking = false + notifyStateChanged() + callback?.onError(error) + } + }) + } + + /** + * 停止TTS播放 + */ + fun stopTts() { + if (isTtsSpeaking) { + azureTtsHelper?.stopSpeaking() + isTtsSpeaking = false + notifyStateChanged() + } + } + + /** + * 播放AI回复 + */ + private fun speakAIResponse(text: String) { + isTtsSpeaking = true + notifyStateChanged() + + azureTtsHelper?.speakText(text, object : AzureTtsHelper.TTSCallback { + override fun onSuccess(message: String) { + isTtsSpeaking = false + notifyStateChanged() + } + + override fun onError(error: String) { + isTtsSpeaking = false + notifyStateChanged() + } + }) + } + + /** + * 释放资源 + */ + fun dispose() { + // 停止语音识别 + stopRecognition() + + // 停止TTS播放 + stopTts() + + // 释放Azure资源 + azureAsrHelper?.let { + FileLogger.d(TAG, "关闭Azure ASR服务") + it.dispose() + } + + azureTtsHelper?.let { + FileLogger.d(TAG, "关闭Azure TTS服务") + it.dispose() + } + + FileLogger.d(TAG, "语音交互处理器资源已释放") + } + + /** + * 检查录音权限 + */ + private fun checkRecordAudioPermission(): Boolean { + val permission = android.Manifest.permission.RECORD_AUDIO + val result = context.checkCallingOrSelfPermission(permission) + return result == android.content.pm.PackageManager.PERMISSION_GRANTED + } + + /** + * 通知状态变化 + */ + private fun notifyStateChanged() { + callback?.onStateChanged(isRecognitionActive, isTtsSpeaking, hasSpeechDetected) + } + + /** + * 发送聊天历史更新 + */ + private fun sendChatHistoryUpdate(agentId: String, userMessage: String, assistantMessage: String) { + val intent = android.content.Intent(VoiceInteractionService.ACTION_CHAT_HISTORY_UPDATED).apply { + putExtra("agentId", agentId) + putExtra("userMessage", userMessage) + putExtra("assistantMessage", assistantMessage) + putExtra("timestamp", System.currentTimeMillis()) + } + + // 发送广播 + context.sendBroadcast(intent) + } + + /** + * 交互回调接口 + */ + interface InteractionCallback { + fun onStateChanged(isRecognitionActive: Boolean, isTtsSpeaking: Boolean, hasSpeechDetected: Boolean) + fun onError(message: String) + fun onPromptRequest(message: String) + } + + /** + * TTS回调接口 + */ + interface TtsCallback { + fun onComplete() + fun onError(error: String) + } +} \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt index ad20ab6b7..76faa6f33 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt @@ -29,14 +29,14 @@ import android.net.Uri import com.yunqiinnovation.deepsound.core.utils.FileLogger import com.yunqiinnovation.azure_speech.AzureAsrHelper import com.yunqiinnovation.azure_speech.AzureTtsHelper -import com.yunqiinnovation.deepsound.OpenAIService +import com.yunqiinnovation.open_ai_service.OpenAIService /** * 后台语音交互 Service： * 1) 前台服务，确保不会被系统轻易杀死 * 2) MediaSession 捕获蓝牙耳机按键 - * 3) 处理录音/语音识别 + * 3) 负责唤醒控制和服务生命周期管理 */ class VoiceInteractionService : Service() { @@ -62,10 +62,7 @@ class VoiceInteractionService : Service() { // 服务状态 private var isActive = false // 服务是否活跃 - private var isRecognitionActive = false // 语音识别是否活跃 private var isTimeoutPaused = false // 是否因超时暂停 - private var hasSpeechDetected = false // 是否检测到语音 - private var isTtsSpeaking = false // 是否正在播放TTS // 按键处理 private var lastKeyEventTime = 0L @@ -74,17 +71,12 @@ class VoiceInteractionService : Service() { // 活动时间 private var lastActivityTime = 0L - // 当前用户输入 - private var currentUserInput = "" - - // 服务组件 private lateinit var mediaSession: MediaSessionCompat private lateinit var audioManager: AudioManager - private lateinit var azureAsrHelper: AzureAsrHelper - private lateinit var azureTtsHelper: AzureTtsHelper - private lateinit var openAIService: OpenAIService - private lateinit var functionHandler: VoiceFunctionHandler + + // 语音交互处理器 + private lateinit var voiceInteractionHandler: VoiceInteractionHandler // 定时器 private val handler = Handler(Looper.getMainLooper()) @@ -94,18 +86,6 @@ class VoiceInteractionService : Service() { handler.postDelayed(this, 1000) // 每秒执行一次 } } - - // 系统提示词 - private val systemPrompt = """ - 你是一个智能语音助手，能够简洁明了地回答用户的问题。 - 请保持回答简短、准确，避免过长的解释。 - 如果用户的问题不清楚，请礼貌地请求澄清。 - 不要使用复杂的术语，除非用户明确要求。 - 用户用语音和你交互。 - - 当用户说"退出"、"再见"、"结束对话"等类似意图时，你应该使用exit_interaction函数来结束对话， - 并在结束前说一句友好的告别语，例如"再见，有需要随时找我"。 - """.trimIndent() // 添加媒体播放器 private var audioPlayer: MiniMediaPlayer? = null @@ -124,7 +104,9 @@ class VoiceInteractionService : Service() { audioManager = getSystemService(Context.AUDIO_SERVICE) as AudioManager FileLogger.d(TAG, "AudioManager初始化完成") - initServices() + // 初始化语音交互处理器 + initVoiceInteractionHandler() + initMediaSession() registerMediaButtonReceiver() @@ -134,7 +116,6 @@ class VoiceInteractionService : Service() { // 设置为媒体播放状态 setPlaybackState(PlaybackStateCompat.STATE_PAUSED) - // FileLogger.d(TAG, "设置播放状态为STATE_PAUSED") // 启动监控和前台服务 startMonitoring() @@ -148,26 +129,21 @@ class VoiceInteractionService : Service() { */ private fun resetState() { isActive = false - isRecognitionActive = false isTimeoutPaused = false - hasSpeechDetected = false - isTtsSpeaking = false } /** - * 初始化所有服务 + * 初始化语音交互处理器 */ - private fun initServices() { - // 创建新的Azure服务实例 - FileLogger.d(TAG, "创建新的Azure服务实例") - azureAsrHelper = AzureAsrHelper(this) - azureTtsHelper = AzureTtsHelper(this) + private fun initVoiceInteractionHandler() { + FileLogger.d(TAG, "初始化语音交互处理器") // 尝试从静态变量获取配置 var subscriptionKey = MainActivity.azureSpeechKey var serviceRegion = MainActivity.azureSpeechRegion var openaiKey = MainActivity.openaiApiKey var openaiBaseUrl = MainActivity.openaiBaseUrl ?: "" // OpenAI API基本URL + var openaiModel = MainActivity.openaiModel ?: "" // OpenAI模型 // 如果静态变量中没有配置，尝试从加密存储中加载 if (subscriptionKey.isEmpty() || serviceRegion.isEmpty() || openaiKey.isEmpty()) { @@ -182,6 +158,7 @@ class VoiceInteractionService : Service() { serviceRegion = MainActivity.azureSpeechRegion openaiKey = MainActivity.openaiApiKey openaiBaseUrl = MainActivity.openaiBaseUrl ?: "" + openaiModel = MainActivity.openaiModel ?: "" FileLogger.d(TAG, "已从加密存储加载配置信息") } else { @@ -189,42 +166,31 @@ class VoiceInteractionService : Service() { } } - // 初始化语音服务 - if (subscriptionKey.isNotEmpty() && serviceRegion.isNotEmpty()) { - // 初始化ASR - azureAsrHelper.initialize(subscriptionKey, serviceRegion, arrayOf("zh-CN")) - - // 初始化TTS - azureTtsHelper.initialize(subscriptionKey, serviceRegion, "zh-CN") - - FileLogger.d(TAG, "Azure语音服务已初始化") - } else { - FileLogger.e(TAG, "Azure配置信息不完整，无法初始化Azure服务") - } - - // 初始化OpenAI服务 - openAIService = OpenAIService() + // 初始化语音交互处理器 + voiceInteractionHandler = VoiceInteractionHandler(applicationContext, subscriptionKey, serviceRegion, openaiKey, openaiBaseUrl, openaiModel) - // 初始化OpenAI服务 - if (openaiKey.isNotEmpty()) { - val initialized = if (openaiBaseUrl.isNotEmpty()) { - openAIService.initialize(openaiKey, openaiBaseUrl) - } else { - openAIService.initialize(openaiKey) + // 初始化回调 + voiceInteractionHandler.setCallback(object : VoiceInteractionHandler.InteractionCallback { + override fun onStateChanged(isRecognitionActive: Boolean, isTtsSpeaking: Boolean, hasSpeechDetected: Boolean) { + // 更新活动时间 + updateLastActivityTime() } - if (initialized) { - FileLogger.d(TAG, "OpenAI服务已初始化") - - // 初始化函数处理器 - functionHandler = VoiceFunctionHandler(openAIService, systemPrompt) - functionHandler.initialize() - - } else { - FileLogger.e(TAG, "OpenAI服务初始化失败") + override fun onError(message: String) { + playNotification(message) } + + override fun onPromptRequest(message: String) { + playPrompt(message) + } + }) + + // 初始化处理器 + val initialized = voiceInteractionHandler.initialize() + if (initialized) { + FileLogger.d(TAG, "语音交互处理器初始化成功") } else { - FileLogger.e(TAG, "OpenAI配置信息不完整，无法初始化OpenAI服务") + FileLogger.e(TAG, "语音交互处理器初始化失败") } } @@ -329,22 +295,21 @@ class VoiceInteractionService : Service() { } // 检查语音识别状态 - if (isRecognitionActive) { + if (voiceInteractionHandler.isRecognitionActive) { val currentTime = System.currentTimeMillis() val elapsedTime = currentTime - lastActivityTime - // 如果超过5秒没有检测到语音，且不在TTS播放中，暂停语音识别 - if (!hasSpeechDetected && !isTtsSpeaking && elapsedTime >= RECOGNITION_TIMEOUT) { + // 如果超过指定时间没有检测到语音，且不在TTS播放中，暂停语音识别 + if (!voiceInteractionHandler.hasSpeechDetected && + !voiceInteractionHandler.isTtsSpeaking && + elapsedTime >= RECOGNITION_TIMEOUT) { + FileLogger.d(TAG, "超过${RECOGNITION_TIMEOUT/1000}秒未检测到语音，停止识别") isTimeoutPaused = true playNotification("没有听到您说话，已暂停对话。双击耳机按钮可重新开始。") - // 停止语音识别并确保资源完全释放 - stopVoiceRecognition() - - // 清理识别状态 - isRecognitionActive = false - hasSpeechDetected = false + // 停止语音识别 + voiceInteractionHandler.stopRecognition() } } } @@ -365,14 +330,12 @@ class VoiceInteractionService : Service() { lastKeyEventTime = currentTime // 停止当前TTS播放 - stopCurrentTTS() + voiceInteractionHandler.stopTts() setPlaybackState(PlaybackStateCompat.STATE_PLAYING) // 播放提示音 playPrompt("我在!") - // audioPlayer?.play(R.raw.listening) - // 设置为播放状态 setPlaybackState(PlaybackStateCompat.STATE_PAUSED) @@ -380,308 +343,48 @@ class VoiceInteractionService : Service() { // 重置超时暂停标志 isTimeoutPaused = false - // 启动或重置语音识别 - if (!isRecognitionActive) { + // 启动语音识别 + if (!voiceInteractionHandler.isRecognitionActive) { FileLogger.d(TAG, "语音识别未激活，开始启动") - // 如果之前是因为超时暂停，重新初始化语音识别组件 - // if (isTimeoutPaused) { - // Log.d(TAG, "之前因超时暂停，重新初始化Azure服务") - // restartAsr() - // } + // 通知 Flutter 语音识别已启动 + notifyRecognitionStarted() - startVoiceRecognition() + // 启动语音识别 + voiceInteractionHandler.startRecognition() } else { FileLogger.d(TAG, "语音识别已激活，更新活动时间") updateLastActivityTime() - hasSpeechDetected = false } } - /** - * 开始语音识别 - */ - private fun startVoiceRecognition() { - if (isRecognitionActive) return - - // 检查录音权限 - if (!checkRecordAudioPermission()) { - playNotification("需要录音权限，请在设置中授予权限") - return - } - // 通知 Flutter 语音识别已启动 - notifyRecognitionStarted() - - isActive = true - isRecognitionActive = true - hasSpeechDetected = false - updateLastActivityTime() - - try { - azureAsrHelper.startContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { - override fun onRecognizing(recognizing: String, detectedLanguage: String) { - if (recognizing.isNotEmpty()) { - hasSpeechDetected = true - stopCurrentTTS() - updateLastActivityTime() - } - } - - override fun onResult(result: String, detectedLanguage: String) { - if (result.isNotEmpty()) { - processWithOpenAI(result) - } - - // 重置状态，继续识别 - hasSpeechDetected = false - updateLastActivityTime() - } - - override fun onSessionStarted() { - updateLastActivityTime() - - } - - override fun onSessionStopped() { - isRecognitionActive = false - - - } - - override fun onCanceled(reason: String, errorDetails: String) { - isRecognitionActive = false - - - } - - override fun onError(error: String) { - isRecognitionActive = false - playNotification("语音识别出错") - - } - }) - } catch (e: Exception) { - isRecognitionActive = false - FileLogger.e(TAG, "启动语音识别失败: ${e.message}", e) - playNotification("启动语音识别失败") - - - } - } - - /** - * 停止语音识别 - */ - private fun stopVoiceRecognition() { - if (!isRecognitionActive) return - - FileLogger.d(TAG, "停止语音识别") - - try { - azureAsrHelper.stopContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { - override fun onResult(result: String, detectedLanguage: String) {} - override fun onRecognizing(recognizing: String, detectedLanguage: String) {} - override fun onSessionStarted() {} - override fun onSessionStopped() { - isRecognitionActive = false - FileLogger.d(TAG, "语音识别会话已停止") - } - override fun onCanceled(reason: String, errorDetails: String) { - isRecognitionActive = false - FileLogger.d(TAG, "语音识别已取消: $reason") - } - override fun onError(error: String) { - isRecognitionActive = false - FileLogger.e(TAG, "停止语音识别时出错: $error") - } - }) - } catch (e: Exception) { - FileLogger.e(TAG, "停止语音识别异常: ${e.message}", e) - // 确保状态一致性 - isRecognitionActive = false - } - - setPlaybackState(PlaybackStateCompat.STATE_PAUSED) - isRecognitionActive = false - hasSpeechDetected = false - // 不重置 isTimeoutPaused，保留暂停原因 - } - - /** - * 使用OpenAI处理语音识别结果 - */ - private fun processWithOpenAI(text: String) { - // 保存当前用户输入，用于后续同步聊天记录 - currentUserInput = text - - Thread { - try { - val messages = JSONArray().apply { - put(openAIService.createUserMessage(text)) - } - - // 创建响应构建器 - val responseBuilder = StringBuilder() - - openAIService.sendMessageStream( - messages = messages, - systemPrompt = systemPrompt, - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - // 累加响应内容 - responseBuilder.append(token) - } - - override fun onComplete() { - // 处理完整响应 - val response = responseBuilder.toString() - if (response.isNotEmpty()) { - // 播放AI回复 - Log.d(TAG, "AI 回复: $response") - - speakAIResponse(response) - - // 同步聊天记录到Flutter端 - notifyChatHistoryUpdated("personal_assistant", text, response) - } - } - - override fun onError(e: Exception) { - FileLogger.e(TAG, "AI流式处理出错: ${e.message}", e) - playNotification("AI处理出错") - } - - override fun onFunctionCall(call: JSONObject) { - FileLogger.d(TAG, "收到函数调用请求: ${call.getString("name")}") - - // 使用函数处理器处理函数调用 - val handled = functionHandler.handleFunctionCall( - functionCall = call, - messages = messages, - callback = object : VoiceFunctionHandler.FunctionCallCallback { - override fun onTokenReceived(token: String) { - responseBuilder.append(token) - } - - override fun onComplete() { - val response = responseBuilder.toString() - if (response.isNotEmpty()) { - // 播放AI回复 - Log.d(TAG, "AI Function Call 回复: $response") - speakAIResponse(response) - - // 同步聊天记录到Flutter端 - notifyChatHistoryUpdated("personal_assistant", text, response) - } - updateLastActivityTime() - } - - override fun onError(message: String) { - FileLogger.e(TAG, "函数处理出错: $message") - playNotification(message) - } - - override fun onFunctionCall(nestedCall: JSONObject) { - FileLogger.d(TAG, "收到嵌套函数调用: ${nestedCall.getString("name")}") - // 递归处理嵌套函数调用 - functionHandler.handleFunctionCall( - functionCall = nestedCall, - messages = messages, - callback = this - ) - } - - override fun onExitWithMessage(farewell: String) { - // 播放退出消息 - speakAIResponse(farewell) - - // 同步聊天记录 - notifyChatHistoryUpdated("personal_assistant", text, farewell) - - // 停止语音识别 - stopVoiceRecognition() - } - } - ) - - if (!handled) { - // 如果函数没有被处理，作为普通文本处理 - FileLogger.d(TAG, "函数未处理，作为普通文本处理") - speakAIResponse("我无法处理这个请求") - notifyChatHistoryUpdated("personal_assistant", text, "我无法处理这个请求") - } - } - } - ) - - } catch (e: Exception) { - FileLogger.e(TAG, "AI处理出错: ${e.message}", e) - playNotification("AI处理出错") - } - }.start() - } - - /** - * 播放AI回复 - */ - private fun speakAIResponse(text: String) { - isTtsSpeaking = true - azureTtsHelper.speakText(text, object : AzureTtsHelper.TTSCallback { - override fun onSuccess(message: String) { - isTtsSpeaking = false - updateLastActivityTime() - } - - override fun onError(error: String) { - isTtsSpeaking = false - } - }) - } - /** * 播放提示音 */ private fun playPrompt(message: String) { - isTtsSpeaking = true - azureTtsHelper.speakText(message, object : AzureTtsHelper.TTSCallback { - override fun onSuccess(message: String) { isTtsSpeaking = false } - override fun onError(error: String) { isTtsSpeaking = false } - }) + voiceInteractionHandler.playTts(message) } /** * 播放通知提示音 */ private fun playNotification(message: String) { - isTtsSpeaking = true // 更新播放状态为播放中，增加接收蓝牙按键事件的几率 setPlaybackState(PlaybackStateCompat.STATE_PLAYING) // 确保媒体会话处于活跃状态 mediaSession.isActive = true - azureTtsHelper.speakText(message, object : AzureTtsHelper.TTSCallback { - override fun onSuccess(message: String) { - isTtsSpeaking = false + voiceInteractionHandler.playTts(message, object : VoiceInteractionHandler.TtsCallback { + override fun onComplete() { setPlaybackState(PlaybackStateCompat.STATE_PAUSED) } override fun onError(error: String) { - isTtsSpeaking = false setPlaybackState(PlaybackStateCompat.STATE_PAUSED) } }) } - /** - * 停止当前TTS播放 - */ - private fun stopCurrentTTS() { - if (isTtsSpeaking) { - azureTtsHelper.stopSpeaking() - isTtsSpeaking = false - } - } - /** * 更新最后活动时间 */ @@ -715,15 +418,6 @@ class VoiceInteractionService : Service() { } } - /** - * 检查录音权限 - */ - private fun checkRecordAudioPermission(): Boolean { - val permission = android.Manifest.permission.RECORD_AUDIO - val result = applicationContext.checkCallingOrSelfPermission(permission) - return result == android.content.pm.PackageManager.PERMISSION_GRANTED - } - /** * 启动前台服务 */ @@ -865,30 +559,17 @@ class VoiceInteractionService : Service() { stopMonitoring() // 停止语音识别 - stopVoiceRecognition() + voiceInteractionHandler.stopRecognition() // 停止媒体会话 mediaSession.release() FileLogger.d(TAG, "媒体会话已释放") // 停止TTS - stopCurrentTTS() + voiceInteractionHandler.stopTts() - // 关闭Azure语音服务 - if (::azureAsrHelper.isInitialized) { - FileLogger.d(TAG, "关闭Azure语音服务") - azureAsrHelper.dispose() - } - - if (::azureTtsHelper.isInitialized) { - FileLogger.d(TAG, "关闭Azure TTS服务") - azureTtsHelper.dispose() - } - - // 关闭OpenAI服务 - if (::openAIService.isInitialized) { - FileLogger.d(TAG, "关闭OpenAI服务") - } + // 释放语音交互处理器资源 + voiceInteractionHandler.dispose() // 关闭音频播放器 audioPlayer?.release() @@ -907,11 +588,11 @@ class VoiceInteractionService : Service() { FileLogger.d(TAG, "暂停后台语音交互（来自Flutter的请求）") // 停止当前TTS播放 - stopCurrentTTS() + voiceInteractionHandler.stopTts() // 停止语音识别 - if (isRecognitionActive) { - stopVoiceRecognition() + if (voiceInteractionHandler.isRecognitionActive) { + voiceInteractionHandler.stopRecognition() } // 设置为暂停状态，但保持服务活跃 @@ -923,8 +604,6 @@ class VoiceInteractionService : Service() { * 通知 Flutter 端聊天记录已更新 */ private fun notifyChatHistoryUpdated(agentId: String, userMessage: String, assistantMessage: String) { - - // 创建广播 Intent val intent = Intent(ACTION_CHAT_HISTORY_UPDATED).apply { putExtra("agentId", agentId) @@ -1025,5 +704,4 @@ class VoiceInteractionService : Service() { */ fun isPlaying() = mediaPlayer?.isPlaying == true } - } \ No newline at end of file diff --git a/android/build.gradle.kts b/android/build.gradle.kts index 35365df1a..f8f47c600 100644 --- a/android/build.gradle.kts +++ b/android/build.gradle.kts @@ -2,6 +2,7 @@ allprojects { repositories { google() mavenCentral() + maven { url = uri("https://storage.googleapis.com/download.flutter.io") } } } diff --git a/android/settings.gradle.kts b/android/settings.gradle.kts index 6a3483c9c..6ca3ffd16 100644 --- a/android/settings.gradle.kts +++ b/android/settings.gradle.kts @@ -30,7 +30,9 @@ plugins { include(":app") include(":azure_speech") +include(":open_ai_service") // 设置azure_speech项目的路径 project(":azure_speech").projectDir = file("../local_plugins/azure_speech/android") +project(":open_ai_service").projectDir = file("../local_plugins/open_ai_service/android") diff --git a/lib/data/services/voice_interaction_service.dart b/lib/data/services/voice_interaction_service.dart index 1bb2406f5..792e94912 100644 --- a/lib/data/services/voice_interaction_service.dart +++ b/lib/data/services/voice_interaction_service.dart @@ -40,7 +40,7 @@ class VoiceInteractionService extends GetxService { late String _azureSpeechRegion; late String _openaiApiKey; late String _openaiBaseUrl; - + late String _openaiModel; // 聊天历史服务 late final ChatHistoryService _chatHistoryService; @@ -60,11 +60,12 @@ class VoiceInteractionService extends GetxService { _azureSpeechRegion = dotenv.env['AZURE_SPEECH_REGION'] ?? ''; _openaiApiKey = dotenv.env['OPENAI_API_KEY'] ?? ''; _openaiBaseUrl = dotenv.env['OPENAI_BASE_URL'] ?? ''; + _openaiModel = dotenv.env['OPENAI_MODEL'] ?? ''; if (_azureSpeechKey.isEmpty || _azureSpeechRegion.isEmpty) { Logger.warning('未找到 Azure 语音服务配置。请在 .env 文件中设置 AZURE_SPEECH_KEY 和 AZURE_SPEECH_REGION'); } - + } /// 处理来自原生层的事件 @@ -196,6 +197,7 @@ class VoiceInteractionService extends GetxService { 'azure_speech_region': _azureSpeechRegion, 'openai_api_key': _openaiApiKey, 'openai_base_url': _openaiBaseUrl, + 'openai_model': _openaiModel, }) ?? false; if (result) { diff --git a/local_plugins/azure_speech/android/build.gradle.kts b/local_plugins/azure_speech/android/build.gradle.kts index 665d495d1..a26b13967 100644 --- a/local_plugins/azure_speech/android/build.gradle.kts +++ b/local_plugins/azure_speech/android/build.gradle.kts @@ -33,8 +33,8 @@ configure { } compileOptions { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 } sourceSets { @@ -53,13 +53,12 @@ configure { // 显式设置Kotlin JVM目标版本 tasks.withType { kotlinOptions { - jvmTarget = "1.8" + jvmTarget = "11" } } dependencies { - // 直接通过本地依赖方式添加Flutter - implementation(fileTree(mapOf("dir" to "libs", "include" to listOf("*.jar")))) + // 添加Microsoft语音SDK implementation("com.microsoft.cognitiveservices.speech:client-sdk:1.30.0") } \ No newline at end of file diff --git a/local_plugins/open_ai_service/README.md b/local_plugins/open_ai_service/README.md new file mode 100644 index 000000000..d7ccb709d --- /dev/null +++ b/local_plugins/open_ai_service/README.md @@ -0,0 +1,348 @@ +# OpenAI服务插件 + +提供与OpenAI API的原生交互功能，支持流式输出和函数调用，同时支持Android和iOS平台。 + +## 功能特点 + +- 原生实现，性能更好 +- 支持流式输出，实时显示AI回复 +- 支持函数调用（工具调用） +- 支持自定义API基础URL +- 支持Android和iOS平台 +- 提供简洁易用的Dart接口 + +## 安装 + +将该插件添加到您的`pubspec.yaml`文件中： + +```yaml +dependencies: + open_ai_service: + path: local_plugins/open_ai_service +``` + +## 使用方法 + +### 初始化 + +首先需要初始化OpenAI服务，提供API密钥和可选的基础URL和模型： + +```dart +import 'package:open_ai_service/open_ai_service.dart'; + +final openAIService = OpenAIService(); + +// 使用OpenAI官方API和默认模型 +await openAIService.initialize(apiKey: 'your_openai_api_key'); + +// 或使用自定义API端点和模型 +await openAIService.initialize( + apiKey: 'your_api_key', + baseUrl: 'https://your-custom-endpoint.com/v1/chat/completions', + model: 'gpt-4-0125-preview', // 自定义模型 +); +``` + +### 普通聊天 + +发送消息并获取非流式回复： + +```dart +// 创建用户消息 +final userMessage = await openAIService.createUserMessage('你好，请介绍一下自己'); + +// 发送消息 +final response = await openAIService.sendMessage( + messages: [userMessage], + systemPrompt: '你是一个有帮助的AI助手，请简洁地回答问题。', +); + +print('AI回复: $response'); +``` + +### 流式输出 + +使用流式输出获取实时回复： + +```dart +// 注册事件监听 +final subscription = openAIService.processEvents( + onToken: (token) { + // 收到部分回复 + print('收到部分回复: $token'); + // 更新UI... + }, + onComplete: () { + // 回复完成 + print('回复完成'); + }, + onError: (error) { + // 处理错误 + print('错误: $error'); + }, + onFunctionCall: (functionCall) { + // 处理函数调用 + print('函数调用: $functionCall'); + handleFunctionCall(functionCall); + }, +); + +// 发送流式消息 +await openAIService.sendMessageStream( + messages: [userMessage], + systemPrompt: '你是一个有帮助的AI助手，请简洁地回答问题。', +); + +// 完成后，记得取消订阅 +// subscription.cancel(); +``` + +### 函数调用 + +注册函数并处理函数调用： + +```dart +// 注册一个天气查询函数 +await openAIService.registerFunction( + name: 'get_weather', + description: '获取指定城市的天气信息', + parameters: { + 'type': 'object', + 'properties': { + 'city': { + 'type': 'string', + 'description': '城市名称，如北京、上海等', + }, + 'date': { + 'type': 'string', + 'description': '查询日期，默认为今天', + }, + }, + 'required': ['city'], + }, +); + +// 处理函数调用 +void handleFunctionCall(Map functionCall) { + final name = functionCall['name']; + final arguments = jsonDecode(functionCall['arguments']); + final id = functionCall['id']; + + if (name == 'get_weather') { + final city = arguments['city']; + final date = arguments['date'] ?? '今天'; + + // 获取天气信息（实际应用中应调用天气API） + final weatherResult = '{"temperature": 25, "condition": "晴天", "humidity": 40}'; + + // 发送函数调用结果 + openAIService.sendFunctionCallResult( + messages: messages, // 当前的消息历史 + systemPrompt: systemPrompt, + functionCall: functionCall, + functionResult: weatherResult, + ); + } +} +``` + +## 示例应用 + +一个简单的聊天机器人示例： + +```dart +import 'package:flutter/material.dart'; +import 'package:open_ai_service/open_ai_service.dart'; + +void main() { + runApp(MyApp()); +} + +class MyApp extends StatelessWidget { + @override + Widget build(BuildContext context) { + return MaterialApp( + title: 'OpenAI 聊天', + theme: ThemeData(primarySwatch: Colors.blue), + home: ChatScreen(), + ); + } +} + +class ChatScreen extends StatefulWidget { + @override + _ChatScreenState createState() => _ChatScreenState(); +} + +class _ChatScreenState extends State { + final OpenAIService _openAIService = OpenAIService(); + final TextEditingController _textController = TextEditingController(); + final List> _messages = []; + StreamSubscription? _subscription; + String _currentResponse = ''; + bool _isLoading = false; + + @override + void initState() { + super.initState(); + _initializeOpenAI(); + } + + Future _initializeOpenAI() async { + try { + await _openAIService.initialize( + apiKey: 'your_api_key_here', + ); + + // 注册事件监听 + _subscription = _openAIService.processEvents( + onToken: (token) { + setState(() { + _currentResponse += token; + }); + }, + onComplete: () { + _addAIMessage(_currentResponse); + setState(() { + _currentResponse = ''; + _isLoading = false; + }); + }, + onError: (error) { + ScaffoldMessenger.of(context).showSnackBar( + SnackBar(content: Text('错误: $error')), + ); + setState(() { + _isLoading = false; + }); + }, + onFunctionCall: (functionCall) { + // 处理函数调用... + }, + ); + } catch (e) { + ScaffoldMessenger.of(context).showSnackBar( + SnackBar(content: Text('初始化失败: $e')), + ); + } + } + + void _sendMessage() async { + if (_textController.text.isEmpty) return; + + final userMessage = _textController.text; + _addUserMessage(userMessage); + _textController.clear(); + + try { + setState(() { + _isLoading = true; + }); + + final message = await _openAIService.createUserMessage(userMessage); + + await _openAIService.sendMessageStream( + messages: _messages, + systemPrompt: '你是一个有帮助的AI助手，请简洁地回答问题。', + ); + } catch (e) { + ScaffoldMessenger.of(context).showSnackBar( + SnackBar(content: Text('发送消息失败: $e')), + ); + setState(() { + _isLoading = false; + }); + } + } + + void _addUserMessage(String content) { + setState(() { + _messages.add({ + 'role': 'user', + 'content': content, + }); + }); + } + + void _addAIMessage(String content) { + setState(() { + _messages.add({ + 'role': 'assistant', + 'content': content, + }); + }); + } + + @override + void dispose() { + _subscription?.cancel(); + _textController.dispose(); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + return Scaffold( + appBar: AppBar(title: Text('OpenAI 聊天')), + body: Column( + children: [ + Expanded( + child: ListView.builder( + itemCount: _messages.length + (_currentResponse.isNotEmpty ? 1 : 0), + itemBuilder: (context, index) { + if (index < _messages.length) { + final message = _messages[index]; + final isUser = message['role'] == 'user'; + + return ListTile( + leading: Icon(isUser ? Icons.person : Icons.computer), + title: Text(message['content']), + ); + } else { + // 显示当前正在生成的回复 + return ListTile( + leading: Icon(Icons.computer), + title: Text(_currentResponse), + ); + } + }, + ), + ), + if (_isLoading) + LinearProgressIndicator(), + Padding( + padding: const EdgeInsets.all(8.0), + child: Row( + children: [ + Expanded( + child: TextField( + controller: _textController, + decoration: InputDecoration( + hintText: '输入消息...', + border: OutlineInputBorder(), + ), + ), + ), + IconButton( + icon: Icon(Icons.send), + onPressed: _sendMessage, + ), + ], + ), + ), + ], + ), + ); + } +} +``` + +## 注意事项 + +- 请确保您的API密钥安全存储，不要直接硬编码在应用程序中 +- 流式输出需要在调用`sendMessageStream`之前注册事件监听 +- 使用完毕后记得取消事件订阅 + +## 许可证 + +此插件遵循MIT许可证。 \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/build.gradle.kts b/local_plugins/open_ai_service/android/build.gradle.kts new file mode 100644 index 000000000..4f7b2f643 --- /dev/null +++ b/local_plugins/open_ai_service/android/build.gradle.kts @@ -0,0 +1,37 @@ +plugins { + // Android Library 插件 + id("com.android.library") + // Kotlin Android 插件 + id("org.jetbrains.kotlin.android") +} + +android { + // 命名空间，对应你插件的包名（需与代码内包名保持一致） + namespace = "com.yunqiinnovation.open_ai_service" + + // 目标 SDK 版本 + compileSdk = 33 + + defaultConfig { + // 最低 SDK 版本 + minSdk = 21 + targetSdk = 33 + } + + // Java 语言级别兼容配置 + compileOptions { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 + } + + // Kotlin 语言级别 + kotlinOptions { + jvmTarget = "11" + } +} + +dependencies { + + implementation("com.squareup.okhttp3:okhttp:4.10.0") + +} \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/settings.gradle.kts b/local_plugins/open_ai_service/android/settings.gradle.kts new file mode 100644 index 000000000..1c33f4971 --- /dev/null +++ b/local_plugins/open_ai_service/android/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "open_ai_service" \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/src/main/AndroidManifest.xml b/local_plugins/open_ai_service/android/src/main/AndroidManifest.xml new file mode 100644 index 000000000..e38cdeb21 --- /dev/null +++ b/local_plugins/open_ai_service/android/src/main/AndroidManifest.xml @@ -0,0 +1,11 @@ + + + + + + + + + + diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt new file mode 100644 index 000000000..7b9213046 --- /dev/null +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt @@ -0,0 +1,456 @@ +package com.yunqiinnovation.open_ai_service + +import android.util.Log +import okhttp3.* +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONArray +import org.json.JSONObject +import java.io.IOException +import java.util.concurrent.TimeUnit + +/** + * OpenAI服务的原生实现 + */ +class OpenAIService() { + private val TAG = "OpenAIService" + private var baseUrl = "" + private val client = OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .writeTimeout(30, TimeUnit.SECONDS) + .build() + + private var apiKey: String = "" + private var isInitialized = false + private var model: String = "" // 默认模型 + + // 用于存储注册的函数 + private val registeredFunctions = mutableListOf() + + /** + * 创建用户消息 + */ + fun createUserMessage(content: String): JSONObject { + return JSONObject().apply { + put("role", "user") + put("content", content) + } + } + + /** + * 创建助手消息 + */ + fun createAssistantMessage(content: String): JSONObject { + return JSONObject().apply { + put("role", "assistant") + put("content", content) + } + } + + /** + * 初始化OpenAI服务 + */ + fun initialize(apiKey: String, baseUrl: String, model: String): Boolean { + this.apiKey = apiKey + if (baseUrl.isNotEmpty()) { + this.baseUrl = baseUrl + } + if (model.isNotEmpty()) { + this.model = model + } + isInitialized = apiKey.isNotEmpty() + return isInitialized + } + + /** + * 注册函数 + */ + fun registerFunction(name: String, description: String, parameters: JSONObject): Boolean { + try { + val function = JSONObject().apply { + put("name", name) + put("description", description) + put("parameters", parameters) + } + + // 检查是否已存在相同名称的函数 + val existingIndex = registeredFunctions.indexOfFirst { + it.getString("name") == name + } + + if (existingIndex >= 0) { + // 如果已存在，则替换 + registeredFunctions[existingIndex] = function + } else { + // 如果不存在，则添加 + registeredFunctions.add(function) + } + + return true + } catch (e: Exception) { + return false + } + } + + /** + * 发送消息（非流式输出） + */ + @Throws(OpenAIException::class) + fun sendMessage(messages: JSONArray, systemPrompt: String): String { + if (!isInitialized || apiKey.isEmpty()) { + throw OpenAIException("OpenAI服务未初始化") + } + + val fullMessages = JSONArray().apply { + put(JSONObject().apply { + put("role", "system") + put("content", systemPrompt) + }) + for (i in 0 until messages.length()) { + put(messages.getJSONObject(i)) + } + } + + val requestBody = JSONObject().apply { + put("model", model) + put("messages", fullMessages) + put("temperature", 0.7) + put("max_tokens", 2000) + put("stream", false) + + // 如果有注册的函数，则添加到请求中 + if (registeredFunctions.isNotEmpty()) { + val tools = JSONArray() + for (function in registeredFunctions) { + val tool = JSONObject().apply { + put("type", "function") + put("function", function) + } + tools.put(tool) + } + put("tools", tools) + } + } + + val mediaType = "application/json".toMediaTypeOrNull() + val request = Request.Builder() + .url(baseUrl) + .addHeader("Content-Type", "application/json") + .addHeader("Authorization", "Bearer $apiKey") + .post(requestBody.toString().toRequestBody(mediaType)) + .build() + + try { + Log.d(TAG, "request: $request") + client.newCall(request).execute().use { response -> + if (!response.isSuccessful) { + throw OpenAIException("API调用失败: ${response.code}") + } + + val responseBody = response.body?.string() ?: throw OpenAIException("Empty response") + val jsonResponse = JSONObject(responseBody) + + // 检查是否有函数调用 + if (jsonResponse.has("choices") && + jsonResponse.getJSONArray("choices").length() > 0) { + + val choice = jsonResponse.getJSONArray("choices").getJSONObject(0) + + // 检查是否是函数调用 + if (choice.has("message")) { + val message = choice.getJSONObject("message") + + // 检查是否有工具调用 + if (message.has("tool_calls")) { + val toolCalls = message.getJSONArray("tool_calls") + if (toolCalls.length() > 0) { + val toolCall = toolCalls.getJSONObject(0) + if (toolCall.has("function")) { + val function = toolCall.getJSONObject("function") + val functionCall = JSONObject().apply { + put("name", function.getString("name")) + put("arguments", function.getString("arguments")) + put("id", toolCall.getString("id")) + } + return functionCall.toString() + } + } + } + + // 如果没有工具调用，返回消息内容 + if (message.has("content")) { + return message.getString("content") + } + } + } + + throw OpenAIException("Invalid response format") + } + } catch (e: Exception) { + if (e is OpenAIException) throw e + throw OpenAIException("Failed to communicate with AI service: ${e.message}") + } + } + + /** + * 发送消息（流式输出） + */ + fun sendMessageStream(messages: JSONArray, systemPrompt: String, callback: StreamCallback) { + if (!isInitialized || apiKey.isEmpty()) { + callback.onError(OpenAIException("OpenAI服务未初始化")) + return + } + + val fullMessages = JSONArray().apply { + put(JSONObject().apply { + put("role", "system") + put("content", systemPrompt) + }) + for (i in 0 until messages.length()) { + put(messages.getJSONObject(i)) + } + } + + val requestBody = JSONObject().apply { + put("model", model) + put("messages", fullMessages) + put("temperature", 0.7) + put("max_tokens", 2000) + put("stream", true) + + // 如果有注册的函数，则添加到请求中 + if (registeredFunctions.isNotEmpty()) { + val tools = JSONArray() + for (function in registeredFunctions) { + val tool = JSONObject().apply { + put("type", "function") + put("function", function) + } + tools.put(tool) + } + put("tools", tools) + } + } + + val mediaType = "application/json".toMediaTypeOrNull() + val request = Request.Builder() + .url(baseUrl) + .addHeader("Content-Type", "application/json") + .addHeader("Authorization", "Bearer $apiKey") + .addHeader("Accept", "text/event-stream") + .post(requestBody.toString().toRequestBody(mediaType)) + .build() + + Log.d(TAG, "request: $request") + client.newCall(request).enqueue(object : Callback { + override fun onFailure(call: Call, e: IOException) { + callback.onError(OpenAIException(e.message ?: "请求失败")) + } + + override fun onResponse(call: Call, response: Response) { + + if (!response.isSuccessful) { + callback.onError(OpenAIException("API调用失败: ${response.code}")) + return + } + + val responseBody = response.body ?: return + val source = responseBody.source() + + try { + // 预取数据到缓冲区 + source.request(Long.MAX_VALUE) + val bufferedSource = source.buffer + + // 用于存储函数调用的各个部分 + val finalToolCalls = mutableMapOf() + + while (!bufferedSource.exhausted()) { + val line = bufferedSource.readUtf8Line() ?: continue + val trimmedLine = line.trim() + if (trimmedLine.isEmpty()) continue + if (trimmedLine.startsWith("data:")) { + val data = trimmedLine.substring(5) + + // 处理[DONE]消息 + if (data == "[DONE]") { + processToolCalls(finalToolCalls, callback) + callback.onComplete() + break + } + + try { + val jsonData = JSONObject(data) + + // 处理消息内容 + if (jsonData.has("choices")) { + val choices = jsonData.getJSONArray("choices") + if (choices.length() > 0) { + val choice = choices.getJSONObject(0) + + if (choice.has("delta")) { + val delta = choice.getJSONObject("delta") + + // 处理普通文本内容 + if (delta.has("content")) { + val content = delta.getString("content") + callback.onToken(content) + } + + // 处理工具调用（函数调用） + if (delta.has("tool_calls")) { + val toolCalls = delta.getJSONArray("tool_calls") + for (i in 0 until toolCalls.length()) { + val toolCall = toolCalls.getJSONObject(i) + val index = toolCall.getInt("index") + + // 创建或获取现有的工具调用信息 + val toolCallInfo = finalToolCalls.getOrPut(index) { ToolCallInfo() } + + // 更新ID + if (toolCall.has("id")) { + toolCallInfo.id = toolCall.getString("id") + } + + // 更新函数信息 + if (toolCall.has("function")) { + val function = toolCall.getJSONObject("function") + + if (function.has("name")) { + toolCallInfo.name = function.getString("name") + } + + if (function.has("arguments")) { + toolCallInfo.arguments += function.getString("arguments") + } + } + } + } + } + } + } + } catch (e: Exception) { + // 忽略解析错误 + Log.e(TAG, "解析JSON出错: ${e.message}") + } + } + } + } catch (e: Exception) { + callback.onError(OpenAIException("处理响应流时出错: ${e.message}")) + } finally { + responseBody.close() + } + } + }) + } + + /** + * 发送函数调用结果 + */ + fun sendFunctionCallResult( + messages: JSONArray, + systemPrompt: String, + functionCall: JSONObject, + functionResult: String, + callback: StreamCallback + ) { + try { + val fullMessages = JSONArray().apply { + // 添加系统提示 + put(JSONObject().apply { + put("role", "system") + put("content", systemPrompt) + }) + + // 添加用户消息 + for (i in 0 until messages.length()) { + put(messages.getJSONObject(i)) + } + + // 添加函数调用消息 + put(JSONObject().apply { + put("role", "assistant") + put("content", null) + + // 添加工具调用 + val toolCalls = JSONArray().apply { + val toolCall = JSONObject().apply { + put("id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) + put("type", "function") + put("function", JSONObject().apply { + put("name", functionCall.getString("name")) + put("arguments", functionCall.getString("arguments")) + }) + } + put(toolCall) + } + put("tool_calls", toolCalls) + }) + + // 添加函数调用结果 + put(JSONObject().apply { + put("role", "tool") + put("content", functionResult) + put("tool_call_id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) + }) + } + + // 发送完整对话 + sendMessageStream(fullMessages, systemPrompt, callback) + + } catch (e: Exception) { + callback.onError(OpenAIException("发送函数调用结果失败: ${e.message}")) + } + } + + /** + * 处理工具调用（处理完整的函数调用并回调） + */ + private fun processToolCalls(toolCalls: Map, callback: StreamCallback) { + if (toolCalls.isEmpty()) return + + // 只处理第一个工具调用 + val firstToolCall = toolCalls.entries.firstOrNull()?.value ?: return + + if (firstToolCall.isValid()) { + // 创建函数调用JSON对象 + val functionCall = JSONObject().apply { + put("name", firstToolCall.name) + put("arguments", firstToolCall.arguments) + put("id", firstToolCall.id) + } + + // 回调 + callback.onFunctionCall(functionCall) + } + } + + /** + * 工具调用信息类 + */ + private class ToolCallInfo { + var id: String = "" + var name: String = "" + var arguments: String = "" + + fun isValid(): Boolean { + return id.isNotEmpty() && name.isNotEmpty() + } + } + + /** + * 流式输出回调接口 + */ + interface StreamCallback { + fun onToken(token: String) + fun onComplete() + fun onError(e: Exception) + fun onFunctionCall(functionCall: JSONObject) + } +} + +/** + * OpenAI服务异常 + */ +class OpenAIException(message: String) : Exception(message) \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt new file mode 100644 index 000000000..5c93e5be6 --- /dev/null +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt @@ -0,0 +1,322 @@ +package com.yunqiinnovation.open_ai_service + +import android.content.Context +import android.util.Log +import androidx.annotation.NonNull +import io.flutter.embedding.engine.plugins.FlutterPlugin +import io.flutter.plugin.common.MethodCall +import io.flutter.plugin.common.MethodChannel +import io.flutter.plugin.common.MethodChannel.MethodCallHandler +import io.flutter.plugin.common.MethodChannel.Result +import io.flutter.plugin.common.EventChannel +import io.flutter.plugin.common.EventChannel.EventSink +import io.flutter.plugin.common.EventChannel.StreamHandler +import org.json.JSONArray +import org.json.JSONObject +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors + +/** OpenAIServicePlugin */ +class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { + /// 方法通道名称 + private val methodChannelName = "com.yunqiinnovation.open_ai_service/methods" + + /// 事件通道名称 + private val eventChannelName = "com.yunqiinnovation.open_ai_service/events" + + /// 方法通道 + private lateinit var methodChannel: MethodChannel + + /// 事件通道 + private lateinit var eventChannel: EventChannel + + /// 应用上下文 + private lateinit var context: Context + + /// OpenAI服务实例 + private val openAIService = OpenAIService() + + /// 事件接收器（用于流式输出） + private var eventSink: EventSink? = null + + /// 执行器（用于后台线程） + private val executor = Executors.newSingleThreadExecutor() + + override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { + // 保存上下文 + context = flutterPluginBinding.applicationContext + + // 初始化方法通道 + methodChannel = MethodChannel(flutterPluginBinding.binaryMessenger, methodChannelName) + methodChannel.setMethodCallHandler(this) + + // 初始化事件通道 + eventChannel = EventChannel(flutterPluginBinding.binaryMessenger, eventChannelName) + eventChannel.setStreamHandler(this) + } + + override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) { + when (call.method) { + "initialize" -> { + val apiKey = call.argument("apiKey") ?: "" + val baseUrl = call.argument("baseUrl") ?: "" + val model = call.argument("model") ?: "" + + val initialized = openAIService.initialize(apiKey, baseUrl, model) + result.success(initialized) + } + + "registerFunction" -> { + val name = call.argument("name") ?: "" + val description = call.argument("description") ?: "" + val parameters = call.argument>("parameters") + + if (name.isEmpty() || parameters == null) { + result.error("INVALID_ARGUMENT", "函数注册参数无效", null) + return + } + + val parametersJson = JSONObject(parameters) + val registered = openAIService.registerFunction(name, description, parametersJson) + result.success(registered) + } + + "sendMessage" -> { + val messagesRaw = call.argument>>("messages") ?: emptyList() + val systemPrompt = call.argument("systemPrompt") ?: "" + + // 转换消息格式 + val messages = JSONArray() + for (message in messagesRaw) { + messages.put(JSONObject(message)) + } + + // 在后台线程执行请求 + executor.execute { + try { + val response = openAIService.sendMessage(messages, systemPrompt) + // 在主线程返回结果 + android.os.Handler(android.os.Looper.getMainLooper()).post { + result.success(response) + } + } catch (e: Exception) { + // 在主线程返回错误 + android.os.Handler(android.os.Looper.getMainLooper()).post { + result.error("OPENAI_ERROR", e.message, null) + } + } + } + } + + "sendMessageStream" -> { + val messagesRaw = call.argument>>("messages") ?: emptyList() + val systemPrompt = call.argument("systemPrompt") ?: "" + + // 检查事件接收器 + if (eventSink == null) { + result.error("NO_EVENT_SINK", "没有可用的事件流接收器", null) + return + } + + // 转换消息格式 + val messages = JSONArray() + for (message in messagesRaw) { + messages.put(JSONObject(message)) + } + + // 在后台线程执行请求 + executor.execute { + try { + openAIService.sendMessageStream( + messages = messages, + systemPrompt = systemPrompt, + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + // 发送token事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + eventSink?.success(mapOf("type" to "token", "content" to token)) + } + } + + override fun onComplete() { + // 发送完成事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + eventSink?.success(mapOf("type" to "complete")) + } + } + + override fun onError(e: Exception) { + // 发送错误事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + eventSink?.success(mapOf("type" to "error", "content" to e.message)) + } + } + + override fun onFunctionCall(functionCall: JSONObject) { + // 发送函数调用事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + val functionCallMap = functionCall.toMap() + eventSink?.success(mapOf("type" to "functionCall", "content" to functionCallMap)) + } + } + } + ) + + // 请求已开始 + android.os.Handler(android.os.Looper.getMainLooper()).post { + result.success(true) + } + } catch (e: Exception) { + // 在主线程返回错误 + android.os.Handler(android.os.Looper.getMainLooper()).post { + result.error("OPENAI_ERROR", e.message, null) + } + } + } + } + + "sendFunctionCallResult" -> { + val messagesRaw = call.argument>>("messages") ?: emptyList() + val systemPrompt = call.argument("systemPrompt") ?: "" + val functionCallRaw = call.argument>("functionCall") ?: emptyMap() + val functionResult = call.argument("functionResult") ?: "" + + // 检查事件接收器 + if (eventSink == null) { + result.error("NO_EVENT_SINK", "没有可用的事件流接收器", null) + return + } + + // 转换消息格式 + val messages = JSONArray() + for (message in messagesRaw) { + messages.put(JSONObject(message)) + } + + // 转换函数调用 + val functionCall = JSONObject(functionCallRaw) + + // 在后台线程执行请求 + executor.execute { + try { + openAIService.sendFunctionCallResult( + messages = messages, + systemPrompt = systemPrompt, + functionCall = functionCall, + functionResult = functionResult, + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + // 发送token事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + eventSink?.success(mapOf("type" to "token", "content" to token)) + } + } + + override fun onComplete() { + // 发送完成事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + eventSink?.success(mapOf("type" to "complete")) + } + } + + override fun onError(e: Exception) { + // 发送错误事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + eventSink?.success(mapOf("type" to "error", "content" to e.message)) + } + } + + override fun onFunctionCall(nestedFunctionCall: JSONObject) { + // 发送函数调用事件 + android.os.Handler(android.os.Looper.getMainLooper()).post { + val functionCallMap = nestedFunctionCall.toMap() + eventSink?.success(mapOf("type" to "functionCall", "content" to functionCallMap)) + } + } + } + ) + + // 请求已开始 + android.os.Handler(android.os.Looper.getMainLooper()).post { + result.success(true) + } + } catch (e: Exception) { + // 在主线程返回错误 + android.os.Handler(android.os.Looper.getMainLooper()).post { + result.error("OPENAI_ERROR", e.message, null) + } + } + } + } + + "createUserMessage" -> { + val content = call.argument("content") ?: "" + val message = openAIService.createUserMessage(content) + result.success(message.toMap()) + } + + "createAssistantMessage" -> { + val content = call.argument("content") ?: "" + val message = openAIService.createAssistantMessage(content) + result.success(message.toMap()) + } + + else -> { + result.notImplemented() + } + } + } + + override fun onDetachedFromEngine(@NonNull binding: FlutterPlugin.FlutterPluginBinding) { + methodChannel.setMethodCallHandler(null) + eventChannel.setStreamHandler(null) + executor.shutdown() + } + + // Stream事件处理 + override fun onListen(arguments: Any?, eventSink: EventSink?) { + this.eventSink = eventSink + } + + override fun onCancel(arguments: Any?) { + this.eventSink = null + } + + // 工具方法：JSONObject转Map + private fun JSONObject.toMap(): Map { + val map = mutableMapOf() + val keys = this.keys() + while (keys.hasNext()) { + val key = keys.next() + var value: Any? = this.opt(key) + + value = when (value) { + JSONObject.NULL -> null + is JSONObject -> value.toMap() + is JSONArray -> value.toList() + else -> value + } + + map[key] = value + } + return map + } + + // 工具方法：JSONArray转List + private fun JSONArray.toList(): List { + val list = mutableListOf() + for (i in 0 until this.length()) { + var value: Any? = this.opt(i) + + value = when (value) { + JSONObject.NULL -> null + is JSONObject -> value.toMap() + is JSONArray -> value.toList() + else -> value + } + + list.add(value) + } + return list + } +} \ No newline at end of file diff --git a/local_plugins/open_ai_service/ios/Classes/OpenAIService.swift b/local_plugins/open_ai_service/ios/Classes/OpenAIService.swift new file mode 100644 index 000000000..671affd73 --- /dev/null +++ b/local_plugins/open_ai_service/ios/Classes/OpenAIService.swift @@ -0,0 +1,473 @@ +import Foundation + +/// OpenAI服务异常 +public class OpenAIError: Error { + let message: String + + init(_ message: String) { + self.message = message + } +} + +/// 工具调用信息 +private class ToolCallInfo { + var id: String = "" + var name: String = "" + var arguments: String = "" + + var isValid: Bool { + return !id.isEmpty && !name.isEmpty + } +} + +/// OpenAI服务iOS原生实现 +public class OpenAIService { + private let TAG = "OpenAIService" + private var baseUrl = "https://api.openai.com/v1/chat/completions" + private var apiKey: String = "" + private var isInitialized = false + private var model: String = "doubao-1-5-lite-32k-250115" // 默认模型 + + // 用于存储注册的函数 + private var registeredFunctions: [[String: Any]] = [] + + // URL会话 + private let session: URLSession + + public init() { + // 创建URL会话配置 + let config = URLSessionConfiguration.default + config.timeoutIntervalForRequest = 30.0 + config.timeoutIntervalForResource = 30.0 + session = URLSession(configuration: config) + } + + /// 创建用户消息 + public func createUserMessage(content: String) -> [String: Any] { + return ["role": "user", "content": content] + } + + /// 创建助手消息 + public func createAssistantMessage(content: String) -> [String: Any] { + return ["role": "assistant", "content": content] + } + + /// 初始化OpenAI服务 + public func initialize(apiKey: String, baseUrl: String = "", model: String = "") -> Bool { + self.apiKey = apiKey + if !baseUrl.isEmpty { + self.baseUrl = baseUrl + } + if !model.isEmpty { + self.model = model + } + isInitialized = !apiKey.isEmpty + return isInitialized + } + + /// 注册函数 + public func registerFunction(name: String, description: String, parameters: [String: Any]) -> Bool { + do { + let function: [String: Any] = [ + "name": name, + "description": description, + "parameters": parameters + ] + + // 检查是否已存在相同名称的函数 + if let existingIndex = registeredFunctions.firstIndex(where: { ($0["name"] as? String) == name }) { + // 如果已存在，则替换 + registeredFunctions[existingIndex] = function + } else { + // 如果不存在，则添加 + registeredFunctions.append(function) + } + + return true + } catch { + return false + } + } + + /// 发送消息（非流式输出） + public func sendMessage(messages: [[String: Any]], systemPrompt: String) throws -> String { + guard isInitialized, !apiKey.isEmpty else { + throw OpenAIError("OpenAI服务未初始化") + } + + // 构建完整消息，添加系统提示 + var fullMessages: [[String: Any]] = [ + ["role": "system", "content": systemPrompt] + ] + fullMessages.append(contentsOf: messages) + + // 构建请求体 + var requestDict: [String: Any] = [ + "model": model, + "messages": fullMessages, + "temperature": 0.7, + "max_tokens": 2000, + "stream": false + ] + + // 如果有注册的函数，添加到请求中 + if !registeredFunctions.isEmpty { + var tools: [[String: Any]] = [] + for function in registeredFunctions { + let tool: [String: Any] = [ + "type": "function", + "function": function + ] + tools.append(tool) + } + requestDict["tools"] = tools + } + + // 将请求数据转换为JSON数据 + guard let jsonData = try? JSONSerialization.data(withJSONObject: requestDict) else { + throw OpenAIError("无法序列化请求数据") + } + + // 创建URL请求 + guard let url = URL(string: baseUrl) else { + throw OpenAIError("无效的URL") + } + + var request = URLRequest(url: url) + request.httpMethod = "POST" + request.addValue("application/json", forHTTPHeaderField: "Content-Type") + request.addValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization") + request.httpBody = jsonData + + // 创建信号量用于同步请求 + let semaphore = DispatchSemaphore(value: 0) + var responseResult: Result = .failure(OpenAIError("未收到响应")) + + // 执行请求 + let task = session.dataTask(with: request) { data, response, error in + if let error = error { + responseResult = .failure(OpenAIError("请求失败: \(error.localizedDescription)")) + semaphore.signal() + return + } + + guard let httpResponse = response as? HTTPURLResponse else { + responseResult = .failure(OpenAIError("无效的HTTP响应")) + semaphore.signal() + return + } + + guard httpResponse.statusCode == 200 else { + responseResult = .failure(OpenAIError("API调用失败: \(httpResponse.statusCode)")) + semaphore.signal() + return + } + + guard let data = data else { + responseResult = .failure(OpenAIError("响应数据为空")) + semaphore.signal() + return + } + + do { + // 解析JSON响应 + guard let jsonResponse = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { + responseResult = .failure(OpenAIError("无法解析JSON响应")) + semaphore.signal() + return + } + + // 检查是否有函数调用 + if let choices = jsonResponse["choices"] as? [[String: Any]], !choices.isEmpty, + let choice = choices.first, + let message = choice["message"] as? [String: Any] { + + // 检查是否有工具调用 + if let toolCalls = message["tool_calls"] as? [[String: Any]], !toolCalls.isEmpty, + let toolCall = toolCalls.first, + let function = toolCall["function"] as? [String: Any], + let name = function["name"] as? String, + let arguments = function["arguments"] as? String, + let id = toolCall["id"] as? String { + + let functionCallDict: [String: Any] = [ + "name": name, + "arguments": arguments, + "id": id + ] + + // 将函数调用转为JSON字符串 + if let functionCallData = try? JSONSerialization.data(withJSONObject: functionCallDict), + let functionCallString = String(data: functionCallData, encoding: .utf8) { + responseResult = .success(functionCallString) + semaphore.signal() + return + } + } + + // 如果没有工具调用，返回消息内容 + if let content = message["content"] as? String { + responseResult = .success(content) + semaphore.signal() + return + } + } + + responseResult = .failure(OpenAIError("无效的响应格式")) + semaphore.signal() + + } catch { + responseResult = .failure(OpenAIError("解析响应时出错: \(error.localizedDescription)")) + semaphore.signal() + } + } + + task.resume() + + // 等待响应完成 + _ = semaphore.wait(timeout: .distantFuture) + + // 返回结果或抛出错误 + switch responseResult { + case .success(let result): + return result + case .failure(let error): + throw error + } + } + + /// 发送消息（流式输出） + public func sendMessageStream(messages: [[String: Any]], systemPrompt: String, callback: @escaping StreamCallback) { + guard isInitialized, !apiKey.isEmpty else { + callback.onError(OpenAIError("OpenAI服务未初始化")) + return + } + + // 构建完整消息，添加系统提示 + var fullMessages: [[String: Any]] = [ + ["role": "system", "content": systemPrompt] + ] + fullMessages.append(contentsOf: messages) + + // 构建请求体 + var requestDict: [String: Any] = [ + "model": model, + "messages": fullMessages, + "temperature": 0.7, + "max_tokens": 2000, + "stream": true + ] + + // 如果有注册的函数，添加到请求中 + if !registeredFunctions.isEmpty { + var tools: [[String: Any]] = [] + for function in registeredFunctions { + let tool: [String: Any] = [ + "type": "function", + "function": function + ] + tools.append(tool) + } + requestDict["tools"] = tools + } + + // 将请求数据转换为JSON数据 + guard let jsonData = try? JSONSerialization.data(withJSONObject: requestDict) else { + callback.onError(OpenAIError("无法序列化请求数据")) + return + } + + // 创建URL请求 + guard let url = URL(string: baseUrl) else { + callback.onError(OpenAIError("无效的URL")) + return + } + + var request = URLRequest(url: url) + request.httpMethod = "POST" + request.addValue("application/json", forHTTPHeaderField: "Content-Type") + request.addValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization") + request.addValue("text/event-stream", forHTTPHeaderField: "Accept") + request.httpBody = jsonData + + // 用于存储函数调用的各个部分 + var finalToolCalls: [Int: ToolCallInfo] = [:] + + // 创建数据任务 + let task = session.dataTask(with: request) { data, response, error in + if let error = error { + callback.onError(OpenAIError("请求失败: \(error.localizedDescription)")) + return + } + + guard let httpResponse = response as? HTTPURLResponse else { + callback.onError(OpenAIError("无效的HTTP响应")) + return + } + + guard httpResponse.statusCode == 200 else { + callback.onError(OpenAIError("API调用失败: \(httpResponse.statusCode)")) + return + } + + guard let data = data else { + callback.onError(OpenAIError("响应数据为空")) + return + } + + // 处理SSE数据流 + if let text = String(data: data, encoding: .utf8) { + let lines = text.components(separatedBy: "\n") + + for line in lines { + if line.isEmpty { continue } + + if line.hasPrefix("data: ") { + let dataContent = line.dropFirst(6) + + // 处理[DONE]消息 + if dataContent == "[DONE]" { + self.processToolCalls(finalToolCalls, callback: callback) + callback.onComplete() + break + } + + // 解析JSON数据 + do { + if let data = dataContent.data(using: .utf8), + let jsonData = try JSONSerialization.jsonObject(with: data) as? [String: Any] { + + // 处理消息内容 + if let choices = jsonData["choices"] as? [[String: Any]], !choices.isEmpty, + let choice = choices.first { + + if let delta = choice["delta"] as? [String: Any] { + // 处理普通文本内容 + if let content = delta["content"] as? String { + callback.onToken(content) + } + + // 处理工具调用（函数调用） + if let toolCalls = delta["tool_calls"] as? [[String: Any]] { + for toolCall in toolCalls { + if let index = toolCall["index"] as? Int { + // 创建或获取现有的工具调用信息 + let toolCallInfo = finalToolCalls[index] ?? ToolCallInfo() + + // 更新ID + if let id = toolCall["id"] as? String { + toolCallInfo.id = id + } + + // 更新函数信息 + if let function = toolCall["function"] as? [String: Any] { + if let name = function["name"] as? String { + toolCallInfo.name = name + } + + if let arguments = function["arguments"] as? String { + toolCallInfo.arguments += arguments + } + } + + finalToolCalls[index] = toolCallInfo + } + } + } + } + } + } + } catch { + NSLog("解析JSON出错: \(error.localizedDescription)") + // 忽略解析错误，继续处理其他行 + } + } + } + } + } + + task.resume() + } + + /// 发送函数调用结果 + public func sendFunctionCallResult( + messages: [[String: Any]], + systemPrompt: String, + functionCall: [String: Any], + functionResult: String, + callback: @escaping StreamCallback + ) { + do { + // 构建完整消息数组 + var fullMessages: [[String: Any]] = [ + // 添加系统提示 + ["role": "system", "content": systemPrompt] + ] + + // 添加用户消息 + fullMessages.append(contentsOf: messages) + + // 获取函数相关信息 + guard let name = functionCall["name"] as? String, + let arguments = functionCall["arguments"] as? String else { + callback.onError(OpenAIError("函数调用信息不完整")) + return + } + + let id = functionCall["id"] as? String ?? "call_\(Int(Date().timeIntervalSince1970 * 1000))" + + // 添加函数调用消息 + fullMessages.append([ + "role": "assistant", + "content": NSNull(), + "tool_calls": [ + [ + "id": id, + "type": "function", + "function": [ + "name": name, + "arguments": arguments + ] + ] + ] + ]) + + // 添加函数调用结果 + fullMessages.append([ + "role": "tool", + "content": functionResult, + "tool_call_id": id + ]) + + // 发送完整对话 + sendMessageStream(messages: fullMessages, systemPrompt: systemPrompt, callback: callback) + + } catch { + callback.onError(OpenAIError("发送函数调用结果失败: \(error.localizedDescription)")) + } + } + + /// 处理工具调用（函数调用）并回调 + private func processToolCalls(_ toolCalls: [Int: ToolCallInfo], callback: StreamCallback) { + if toolCalls.isEmpty { return } + + // 只处理第一个工具调用 + guard let firstToolCall = toolCalls.values.first, firstToolCall.isValid else { return } + + // 创建函数调用字典 + let functionCall: [String: Any] = [ + "name": firstToolCall.name, + "arguments": firstToolCall.arguments, + "id": firstToolCall.id + ] + + // 回调 + callback.onFunctionCall(functionCall) + } + + /// 流式输出回调协议 + public typealias StreamCallback = (onToken: (String) -> Void, + onComplete: () -> Void, + onError: (Error) -> Void, + onFunctionCall: ([String: Any]) -> Void) +} \ No newline at end of file diff --git a/local_plugins/open_ai_service/ios/Classes/OpenAIServicePlugin.swift b/local_plugins/open_ai_service/ios/Classes/OpenAIServicePlugin.swift new file mode 100644 index 000000000..d919e2584 --- /dev/null +++ b/local_plugins/open_ai_service/ios/Classes/OpenAIServicePlugin.swift @@ -0,0 +1,215 @@ +import Flutter +import UIKit + +public class OpenAIServicePlugin: NSObject, FlutterPlugin, FlutterStreamHandler { + // OpenAI服务实例 + private let openAIService = OpenAIService() + + // 事件接收器 + private var eventSink: FlutterEventSink? + + // 注册插件 + public static func register(with registrar: FlutterPluginRegistrar) { + let methodChannel = FlutterMethodChannel(name: "com.yunqiinnovation.open_ai_service/methods", binaryMessenger: registrar.messenger()) + let eventChannel = FlutterEventChannel(name: "com.yunqiinnovation.open_ai_service/events", binaryMessenger: registrar.messenger()) + + let instance = OpenAIServicePlugin() + registrar.addMethodCallDelegate(instance, channel: methodChannel) + eventChannel.setStreamHandler(instance) + } + + // 处理方法调用 + public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { + switch call.method { + case "initialize": + if let args = call.arguments as? [String: Any], + let apiKey = args["apiKey"] as? String { + let baseUrl = args["baseUrl"] as? String ?? "" + let model = args["model"] as? String ?? "" + let initialized = openAIService.initialize(apiKey: apiKey, baseUrl: baseUrl, model: model) + result(initialized) + } else { + result(FlutterError(code: "INVALID_ARGUMENT", message: "初始化参数无效", details: nil)) + } + + case "registerFunction": + if let args = call.arguments as? [String: Any], + let name = args["name"] as? String, + let description = args["description"] as? String, + let parameters = args["parameters"] as? [String: Any] { + + let registered = openAIService.registerFunction(name: name, description: description, parameters: parameters) + result(registered) + } else { + result(FlutterError(code: "INVALID_ARGUMENT", message: "函数注册参数无效", details: nil)) + } + + case "sendMessage": + guard let args = call.arguments as? [String: Any], + let messagesRaw = args["messages"] as? [[String: Any]], + let systemPrompt = args["systemPrompt"] as? String else { + result(FlutterError(code: "INVALID_ARGUMENT", message: "发送消息参数无效", details: nil)) + return + } + + // 在后台线程执行 + DispatchQueue.global(qos: .userInitiated).async { + do { + let response = try self.openAIService.sendMessage(messages: messagesRaw, systemPrompt: systemPrompt) + // 在主线程返回结果 + DispatchQueue.main.async { + result(response) + } + } catch { + // 在主线程返回错误 + DispatchQueue.main.async { + result(FlutterError(code: "OPENAI_ERROR", message: error.localizedDescription, details: nil)) + } + } + } + + case "sendMessageStream": + guard let args = call.arguments as? [String: Any], + let messagesRaw = args["messages"] as? [[String: Any]], + let systemPrompt = args["systemPrompt"] as? String else { + result(FlutterError(code: "INVALID_ARGUMENT", message: "发送消息参数无效", details: nil)) + return + } + + // 检查事件接收器 + guard let eventSink = self.eventSink else { + result(FlutterError(code: "NO_EVENT_SINK", message: "没有可用的事件流接收器", details: nil)) + return + } + + // 在后台线程执行 + DispatchQueue.global(qos: .userInitiated).async { + let callback: OpenAIService.StreamCallback = ( + onToken: { token in + // 发送token事件 + DispatchQueue.main.async { + eventSink(["type": "token", "content": token]) + } + }, + onComplete: { + // 发送完成事件 + DispatchQueue.main.async { + eventSink(["type": "complete"]) + } + }, + onError: { error in + // 发送错误事件 + DispatchQueue.main.async { + eventSink(["type": "error", "content": error.localizedDescription]) + } + }, + onFunctionCall: { functionCall in + // 发送函数调用事件 + DispatchQueue.main.async { + eventSink(["type": "functionCall", "content": functionCall]) + } + } + ) + + self.openAIService.sendMessageStream(messages: messagesRaw, systemPrompt: systemPrompt, callback: callback) + + // 请求已开始 + DispatchQueue.main.async { + result(true) + } + } + + case "sendFunctionCallResult": + guard let args = call.arguments as? [String: Any], + let messagesRaw = args["messages"] as? [[String: Any]], + let systemPrompt = args["systemPrompt"] as? String, + let functionCallRaw = args["functionCall"] as? [String: Any], + let functionResult = args["functionResult"] as? String else { + result(FlutterError(code: "INVALID_ARGUMENT", message: "发送函数调用结果参数无效", details: nil)) + return + } + + // 检查事件接收器 + guard let eventSink = self.eventSink else { + result(FlutterError(code: "NO_EVENT_SINK", message: "没有可用的事件流接收器", details: nil)) + return + } + + // 在后台线程执行 + DispatchQueue.global(qos: .userInitiated).async { + let callback: OpenAIService.StreamCallback = ( + onToken: { token in + // 发送token事件 + DispatchQueue.main.async { + eventSink(["type": "token", "content": token]) + } + }, + onComplete: { + // 发送完成事件 + DispatchQueue.main.async { + eventSink(["type": "complete"]) + } + }, + onError: { error in + // 发送错误事件 + DispatchQueue.main.async { + eventSink(["type": "error", "content": error.localizedDescription]) + } + }, + onFunctionCall: { functionCall in + // 发送函数调用事件 + DispatchQueue.main.async { + eventSink(["type": "functionCall", "content": functionCall]) + } + } + ) + + self.openAIService.sendFunctionCallResult( + messages: messagesRaw, + systemPrompt: systemPrompt, + functionCall: functionCallRaw, + functionResult: functionResult, + callback: callback + ) + + // 请求已开始 + DispatchQueue.main.async { + result(true) + } + } + + case "createUserMessage": + if let args = call.arguments as? [String: Any], + let content = args["content"] as? String { + let message = openAIService.createUserMessage(content: content) + result(message) + } else { + result(FlutterError(code: "INVALID_ARGUMENT", message: "创建用户消息参数无效", details: nil)) + } + + case "createAssistantMessage": + if let args = call.arguments as? [String: Any], + let content = args["content"] as? String { + let message = openAIService.createAssistantMessage(content: content) + result(message) + } else { + result(FlutterError(code: "INVALID_ARGUMENT", message: "创建助手消息参数无效", details: nil)) + } + + default: + result(FlutterMethodNotImplemented) + } + } + + // MARK: - FlutterStreamHandler + + public func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { + self.eventSink = events + return nil + } + + public func onCancel(withArguments arguments: Any?) -> FlutterError? { + self.eventSink = nil + return nil + } +} \ No newline at end of file diff --git a/local_plugins/open_ai_service/lib/open_ai_service.dart b/local_plugins/open_ai_service/lib/open_ai_service.dart new file mode 100644 index 000000000..b946d1bff --- /dev/null +++ b/local_plugins/open_ai_service/lib/open_ai_service.dart @@ -0,0 +1,305 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:flutter/services.dart'; + +/// OpenAI服务异常 +class OpenAIException implements Exception { + final String message; + + OpenAIException(this.message); + + @override + String toString() => 'OpenAIException: $message'; +} + +/// OpenAI服务事件类型 +enum OpenAIEventType { + token, + complete, + error, + functionCall, +} + +/// OpenAI服务事件 +class OpenAIEvent { + final OpenAIEventType type; + final dynamic content; + + OpenAIEvent({required this.type, this.content}); + + factory OpenAIEvent.fromMap(Map map) { + final typeStr = map['type'] as String; + final content = map['content']; + + return OpenAIEvent( + type: _typeFromString(typeStr), + content: content, + ); + } + + static OpenAIEventType _typeFromString(String typeStr) { + switch (typeStr) { + case 'token': + return OpenAIEventType.token; + case 'complete': + return OpenAIEventType.complete; + case 'error': + return OpenAIEventType.error; + case 'functionCall': + return OpenAIEventType.functionCall; + default: + throw ArgumentError('未知的事件类型: $typeStr'); + } + } +} + +/// OpenAI服务插件 +class OpenAIService { + static const MethodChannel _channel = MethodChannel('com.yunqiinnovation.open_ai_service/methods'); + static const EventChannel _eventChannel = EventChannel('com.yunqiinnovation.open_ai_service/events'); + + /// 事件流控制器 + StreamController? _eventStreamController; + + /// 事件流 + Stream? _eventStream; + + /// 获取事件流 + Stream get eventStream { + if (_eventStream == null) { + _eventStreamController = StreamController.broadcast(); + _eventStream = _eventStreamController!.stream; + + // 监听原生事件 + _eventChannel.receiveBroadcastStream().listen( + (dynamic event) { + if (event is Map) { + final eventMap = Map.from(event); + final openAIEvent = OpenAIEvent.fromMap(eventMap); + _eventStreamController!.add(openAIEvent); + } + }, + onError: (error) { + _eventStreamController!.addError(OpenAIException('事件流错误: $error')); + }, + ); + } + + return _eventStream!; + } + + /// 初始化OpenAI服务 + /// + /// [apiKey] OpenAI API密钥 + /// [baseUrl] 可选，自定义API基础URL + /// [model] 可选，自定义使用的模型 + Future initialize({ + required String apiKey, + String baseUrl = '', + String model = '', + }) async { + try { + final result = await _channel.invokeMethod( + 'initialize', + { + 'apiKey': apiKey, + 'baseUrl': baseUrl, + 'model': model, + }, + ); + + return result ?? false; + } catch (e) { + throw OpenAIException('初始化失败: $e'); + } + } + + /// 注册函数 + /// + /// [name] 函数名称 + /// [description] 函数描述 + /// [parameters] 函数参数 + Future registerFunction({ + required String name, + required String description, + required Map parameters, + }) async { + try { + final result = await _channel.invokeMethod( + 'registerFunction', + { + 'name': name, + 'description': description, + 'parameters': parameters, + }, + ); + + return result ?? false; + } catch (e) { + throw OpenAIException('注册函数失败: $e'); + } + } + + /// 创建用户消息 + /// + /// [content] 消息内容 + Future> createUserMessage(String content) async { + try { + final result = await _channel.invokeMethod>( + 'createUserMessage', + {'content': content}, + ); + + if (result == null) { + throw OpenAIException('创建用户消息失败: 结果为空'); + } + + return Map.from(result); + } catch (e) { + throw OpenAIException('创建用户消息失败: $e'); + } + } + + /// 创建助手消息 + /// + /// [content] 消息内容 + Future> createAssistantMessage(String content) async { + try { + final result = await _channel.invokeMethod>( + 'createAssistantMessage', + {'content': content}, + ); + + if (result == null) { + throw OpenAIException('创建助手消息失败: 结果为空'); + } + + return Map.from(result); + } catch (e) { + throw OpenAIException('创建助手消息失败: $e'); + } + } + + /// 发送消息（非流式输出） + /// + /// [messages] 消息列表 + /// [systemPrompt] 系统提示词 + Future sendMessage({ + required List> messages, + required String systemPrompt, + }) async { + try { + final result = await _channel.invokeMethod( + 'sendMessage', + { + 'messages': messages, + 'systemPrompt': systemPrompt, + }, + ); + + if (result == null) { + throw OpenAIException('发送消息失败: 结果为空'); + } + + return result; + } catch (e) { + throw OpenAIException('发送消息失败: $e'); + } + } + + /// 发送消息（流式输出） + /// + /// [messages] 消息列表 + /// [systemPrompt] 系统提示词 + /// + /// 返回一个布尔值，表示请求是否已开始 + Future sendMessageStream({ + required List> messages, + required String systemPrompt, + }) async { + try { + final result = await _channel.invokeMethod( + 'sendMessageStream', + { + 'messages': messages, + 'systemPrompt': systemPrompt, + }, + ); + + return result ?? false; + } catch (e) { + throw OpenAIException('发送流式消息失败: $e'); + } + } + + /// 发送函数调用结果 + /// + /// [messages] 消息列表 + /// [systemPrompt] 系统提示词 + /// [functionCall] 函数调用信息 + /// [functionResult] 函数调用结果 + /// + /// 返回一个布尔值，表示请求是否已开始 + Future sendFunctionCallResult({ + required List> messages, + required String systemPrompt, + required Map functionCall, + required String functionResult, + }) async { + try { + final result = await _channel.invokeMethod( + 'sendFunctionCallResult', + { + 'messages': messages, + 'systemPrompt': systemPrompt, + 'functionCall': functionCall, + 'functionResult': functionResult, + }, + ); + + return result ?? false; + } catch (e) { + throw OpenAIException('发送函数调用结果失败: $e'); + } + } + + /// 处理流式事件 + /// + /// [onToken] 收到token时调用 + /// [onComplete] 完成时调用 + /// [onError] 错误时调用 + /// [onFunctionCall] 函数调用时调用 + StreamSubscription processEvents({ + void Function(String token)? onToken, + void Function()? onComplete, + void Function(String error)? onError, + void Function(Map functionCall)? onFunctionCall, + }) { + return eventStream.listen((event) { + switch (event.type) { + case OpenAIEventType.token: + onToken?.call(event.content as String); + break; + case OpenAIEventType.complete: + onComplete?.call(); + break; + case OpenAIEventType.error: + onError?.call(event.content as String); + break; + case OpenAIEventType.functionCall: + onFunctionCall?.call(Map.from(event.content as Map)); + break; + } + }); + } + + /// 从JSON字符串解析函数调用 + Map parseFunctionCall(String functionCallJson) { + try { + return json.decode(functionCallJson) as Map; + } catch (e) { + throw OpenAIException('解析函数调用失败: $e'); + } + } +} \ No newline at end of file diff --git a/local_plugins/open_ai_service/pubspec.yaml b/local_plugins/open_ai_service/pubspec.yaml new file mode 100644 index 000000000..8e00ed3ea --- /dev/null +++ b/local_plugins/open_ai_service/pubspec.yaml @@ -0,0 +1,26 @@ +name: open_ai_service +description: 原生OpenAI服务插件，提供与OpenAI API的交互功能，支持流式输出和函数调用。 +version: 0.0.1 +homepage: https://github.com/yunqiinnovation/deep_voice + +environment: + sdk: ">=2.17.0 <4.0.0" + flutter: ">=2.5.0" + +dependencies: + flutter: + sdk: flutter + +dev_dependencies: + flutter_test: + sdk: flutter + flutter_lints: ^2.0.0 + +flutter: + plugin: + platforms: + android: + package: com.yunqiinnovation.open_ai_service + pluginClass: OpenAIServicePlugin + ios: + pluginClass: OpenAIServicePlugin \ No newline at end of file diff --git a/pubspec.yaml b/pubspec.yaml index 3e1dd7c90..19366500b 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -62,6 +62,8 @@ dependencies: path: ^1.9.1 azure_speech: path: local_plugins/azure_speech + open_ai_service: + path: local_plugins/open_ai_service dev_dependencies: flutter_test: From 9687daaa7c5481d474e614a2321a5d18d01ee861 Mon Sep 17 00:00:00 2001 From: wolfplus Date: Sun, 6 Apr 2025 13:52:33 +0100 Subject: [PATCH 03/14] init --- a | 18 - android/app/src/main/AndroidManifest.xml | 9 + .../yunqiinnovation/deepsound/MainActivity.kt | 104 ++++- .../deepsound/SystemFunctionHandler.kt | 225 ++++++++++ .../deepsound/VoiceFunctionHandler.kt | 336 ++++++++++++++- .../deepsound/VoiceInteractionHandler.kt | 19 +- .../deepsound/VoiceInteractionService.kt | 7 +- lib/core/bindings/initial_binding.dart | 9 + .../events/voice_interaction_event.dart | 13 + lib/data/services/ai_service.dart | 2 + .../services/open_ai_service_adapter.dart | 247 +++++++++++ .../services/voice_interaction_service.dart | 24 ++ .../chat/controllers/chat_controller.dart | 12 +- .../azure_speech/AzureAsrHelper.kt | 4 +- local_plugins/open_ai_service/README.md | 405 +++++++----------- .../open_ai_service/OpenAIService.kt | 155 ++++--- .../open_ai_service/OpenAIServicePlugin.kt | 7 +- .../open_ai_service/lib/open_ai_service.dart | 38 -- test.json | 1 + 19 files changed, 1225 insertions(+), 410 deletions(-) delete mode 100644 a create mode 100644 android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt create mode 100644 lib/data/services/open_ai_service_adapter.dart create mode 100644 test.json diff --git a/a b/a deleted file mode 100644 index 7133736ae..000000000 --- a/a +++ /dev/null @@ -1,18 +0,0 @@ -curl 'https://ark.cn-beijing.volces.com/api/v3/bots/chat/completions' \ --H "Authorization: Bearer 168deb3d-fd0c-4912-b9f1-aaee5c6743e6" \ --H 'Content-Type: application/json' \ --d '{ - "model": "bot-20250405211523-l7c9r", - "stream": true, - "stream_options": {"include_usage": true}, - "messages": [ - { - "role": "system", - "content": "You are a helpful assistant." - }, - { - "role": "user", - "content": "Hello!" - } - ] -}' \ No newline at end of file diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index f120cc1f7..23c380fb0 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -27,6 +27,15 @@ + + + + + + + + = Build.VERSION_CODES.S) { + arrayOf( + Manifest.permission.RECORD_AUDIO, + Manifest.permission.BLUETOOTH_CONNECT, + Manifest.permission.BLUETOOTH_SCAN, + Manifest.permission.ACCESS_FINE_LOCATION, + Manifest.permission.SEND_SMS, + Manifest.permission.READ_CONTACTS, + Manifest.permission.CALL_PHONE, + Manifest.permission.POST_NOTIFICATIONS + ) + } else { + arrayOf( + Manifest.permission.RECORD_AUDIO, + Manifest.permission.BLUETOOTH, + Manifest.permission.BLUETOOTH_ADMIN, + Manifest.permission.ACCESS_FINE_LOCATION, + Manifest.permission.SEND_SMS, + Manifest.permission.READ_CONTACTS, + Manifest.permission.CALL_PHONE + ) + } + // 广播接收器 private val voiceInteractionReceiver = object : BroadcastReceiver() { override fun onReceive(context: Context?, intent: Intent?) { - Log.d(TAG, "收到广播: ${intent?.action}") + FileLogger.d(TAG, "收到广播: ${intent?.action}") when (intent?.action) { VoiceInteractionService.ACTION_RECOGNITION_STARTED -> { val timestamp = intent.getLongExtra("timestamp", 0) - Log.d(TAG, "收到语音识别启动广播: timestamp=$timestamp") sendVoiceInteractionEvent(mapOf( "type" to "recognition_started", "timestamp" to timestamp @@ -54,6 +82,12 @@ class MainActivity: FlutterActivity() { sendChatHistoryEvent(agentId, userMessage, assistantMessage, timestamp) } + VoiceInteractionService.ACTION_ENTER_TRANSLATION_MODE -> { + sendVoiceInteractionEvent(mapOf( + "type" to "enter_translation_mode", + "timestamp" to System.currentTimeMillis() + )) + } } } } @@ -249,6 +283,9 @@ class MainActivity: FlutterActivity() { // 初始化 FileLogger FileLogger.init(applicationContext) + // 请求必要权限 + requestRequiredPermissions() + // 初始化 Azure 语音服务 classicBluetoothHelper = ClassicBluetoothHelper(applicationContext) @@ -267,7 +304,14 @@ class MainActivity: FlutterActivity() { val voiceInteractionFilter = IntentFilter().apply { addAction(VoiceInteractionService.ACTION_RECOGNITION_STARTED) addAction(VoiceInteractionService.ACTION_CHAT_HISTORY_UPDATED) + addAction(VoiceInteractionService.ACTION_ENTER_TRANSLATION_MODE) } + + // 打印已注册的广播 + FileLogger.d(TAG, "已注册语音交互广播：${VoiceInteractionService.ACTION_RECOGNITION_STARTED}, " + + "${VoiceInteractionService.ACTION_CHAT_HISTORY_UPDATED}, " + + "${VoiceInteractionService.ACTION_ENTER_TRANSLATION_MODE}") + if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { registerReceiver(voiceInteractionReceiver, voiceInteractionFilter, Context.RECEIVER_NOT_EXPORTED) } else { @@ -550,14 +594,19 @@ class MainActivity: FlutterActivity() { private fun sendVoiceInteractionEvent(event: Map) { if (voiceInteractionEventSink == null) { Log.e(TAG, "无法发送语音交互事件：事件通道未准备好") + FileLogger.e(TAG, "无法发送语音交互事件：事件通道未准备好") return } + FileLogger.d(TAG, "准备发送事件到Flutter: ${event["type"]}") + runOnUiThread { try { voiceInteractionEventSink?.success(event) + FileLogger.d(TAG, "成功发送事件到Flutter: ${event["type"]}") } catch (e: Exception) { - Log.e(TAG, "发送语音交互事件到Flutter失败: ${e.message}", e) + Log.e(TAG, "发送语音交互事件到Flutter失败: ${e.message}") + FileLogger.e(TAG, "发送语音交互事件到Flutter失败: ${e.message}") } } } @@ -598,5 +647,54 @@ class MainActivity: FlutterActivity() { super.onDestroy() } + + /** + * 请求必要权限 + */ + private fun requestRequiredPermissions() { + val permissionsToRequest = ArrayList() + + for (permission in REQUIRED_PERMISSIONS) { + if (ContextCompat.checkSelfPermission(this, permission) != PackageManager.PERMISSION_GRANTED) { + permissionsToRequest.add(permission) + } + } + + if (permissionsToRequest.isNotEmpty()) { + ActivityCompat.requestPermissions( + this, + permissionsToRequest.toTypedArray(), + PERMISSION_REQUEST_CODE + ) + } + } + + /** + * 处理权限请求结果 + */ + override fun onRequestPermissionsResult( + requestCode: Int, + permissions: Array, + grantResults: IntArray + ) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults) + + if (requestCode == PERMISSION_REQUEST_CODE) { + val deniedPermissions = ArrayList() + + for (i in permissions.indices) { + if (grantResults[i] != PackageManager.PERMISSION_GRANTED) { + deniedPermissions.add(permissions[i]) + } + } + + if (deniedPermissions.isNotEmpty()) { + // 记录未授权的权限 + FileLogger.w(TAG, "未授权的权限: ${deniedPermissions.joinToString()}") + } else { + FileLogger.d(TAG, "所有必要权限已授权") + } + } + } } diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt new file mode 100644 index 000000000..4e28f26aa --- /dev/null +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt @@ -0,0 +1,225 @@ +package com.yunqiinnovation.deepsound + +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.net.Uri +import android.provider.ContactsContract +import android.telephony.SmsManager +import com.yunqiinnovation.deepsound.core.utils.FileLogger + +/** + * 系统功能处理器 - 负责调用系统API执行各种功能 + */ +class SystemFunctionHandler(private val context: Context) { + companion object { + private const val TAG = "SystemFunctionHandler" + } + + /** + * 启动翻译模式 + * @return 操作结果 + */ + fun enterTranslationMode(): Result { + FileLogger.d(TAG, "进入翻译模式") + // 此处应添加实际的翻译模式实现 + // 可能需要启动一个专门的翻译服务或活动 + + // 广播通知APP的其他部分进入翻译模式 + // 使用VoiceInteractionService中定义的常量 + val intent = Intent(VoiceInteractionService.ACTION_ENTER_TRANSLATION_MODE).apply { + + putExtra("timestamp", System.currentTimeMillis()) + } + + // 发送广播 + context.sendBroadcast(intent) + FileLogger.d(TAG, "已发送广播: ${VoiceInteractionService.ACTION_ENTER_TRANSLATION_MODE}") + + return Result( + success = true, + message = "已进入翻译模式" + ) + } + + /** + * 发送短信 + * @param contact 联系人（名称或电话号码） + * @param message 短信内容 + * @return 操作结果 + */ + fun sendTextMessage(contact: String, message: String): Result { + FileLogger.d(TAG, "发送短信给 $contact: $message") + + try { + val phoneNumber = getPhoneNumberForContact(contact) + if (phoneNumber.isEmpty()) { + return Result( + success = false, + message = "未找到联系人信息: $contact" + ) + } + + // 使用SmsManager发送短信 + val smsManager = SmsManager.getDefault() + smsManager.sendTextMessage(phoneNumber, null, message, null, null) + + return Result( + success = true, + message = "已向${contact}发送短信: $message" + ) + } catch (e: Exception) { + FileLogger.e(TAG, "发送短信失败: ${e.message}", e) + return Result( + success = false, + message = "发送短信失败: ${e.message}" + ) + } + } + + /** + * 拨打电话 + * @param contact 联系人（名称或电话号码） + * @return 操作结果 + */ + fun makePhoneCall(contact: String): Result { + FileLogger.d(TAG, "拨打电话给 $contact") + + try { + val phoneNumber = getPhoneNumberForContact(contact) + if (phoneNumber.isEmpty()) { + return Result( + success = false, + message = "未找到联系人信息: $contact" + ) + } + + // 创建拨号意图 + val intent = Intent(Intent.ACTION_CALL) + intent.data = Uri.parse("tel:$phoneNumber") + intent.flags = Intent.FLAG_ACTIVITY_NEW_TASK + context.startActivity(intent) + + return Result( + success = true, + message = "正在拨打${contact}的电话" + ) + } catch (e: Exception) { + FileLogger.e(TAG, "拨打电话失败: ${e.message}", e) + return Result( + success = false, + message = "拨打电话失败: ${e.message}" + ) + } + } + + /** + * 打开应用程序 + * @param appName 应用名称 + * @return 操作结果 + */ + fun openApp(appName: String): Result { + FileLogger.d(TAG, "打开应用: $appName") + + try { + // 查找应用的包名 + val packageManager = context.packageManager + val intent = Intent(Intent.ACTION_MAIN, null) + intent.addCategory(Intent.CATEGORY_LAUNCHER) + + val apps = packageManager.queryIntentActivities(intent, 0) + + for (resolveInfo in apps) { + val appLabel = resolveInfo.loadLabel(packageManager).toString() + if (appLabel.equals(appName, ignoreCase = true) || + appLabel.contains(appName, ignoreCase = true)) { + + // 找到匹配的应用 + val packageName = resolveInfo.activityInfo.packageName + val launchIntent = packageManager.getLaunchIntentForPackage(packageName) + + if (launchIntent != null) { + launchIntent.flags = Intent.FLAG_ACTIVITY_NEW_TASK + context.startActivity(launchIntent) + + return Result( + success = true, + message = "正在打开${appName}应用" + ) + } + } + } + + return Result( + success = false, + message = "未找到应用: $appName" + ) + } catch (e: Exception) { + FileLogger.e(TAG, "打开应用失败: ${e.message}", e) + return Result( + success = false, + message = "打开应用失败: ${e.message}" + ) + } + } + + /** + * 根据联系人名称获取电话号码 + * @param contact 联系人名称或电话号码 + * @return 电话号码 + */ + private fun getPhoneNumberForContact(contact: String): String { + // 如果输入的是电话号码（简单判断，包含数字和特殊符号） + if (contact.matches(Regex("^[+\\d\\s\\-()]*\$")) && contact.any { it.isDigit() }) { + // 移除非数字字符 + return contact.replace(Regex("[^+\\d]"), "") + } + + // 从联系人数据库查询 + try { + val contentResolver = context.contentResolver + val uri = Uri.withAppendedPath( + ContactsContract.Contacts.CONTENT_FILTER_URI, + Uri.encode(contact) + ) + + val projection = arrayOf( + ContactsContract.Contacts._ID, + ContactsContract.Contacts.DISPLAY_NAME, + ContactsContract.Contacts.HAS_PHONE_NUMBER + ) + + contentResolver.query(uri, projection, null, null, null)?.use { cursor -> + if (cursor.moveToFirst()) { + val hasPhoneNumber = cursor.getInt(cursor.getColumnIndexOrThrow(ContactsContract.Contacts.HAS_PHONE_NUMBER)) + + if (hasPhoneNumber > 0) { + val contactId = cursor.getString(cursor.getColumnIndexOrThrow(ContactsContract.Contacts._ID)) + val phoneUri = ContactsContract.CommonDataKinds.Phone.CONTENT_URI + val phoneProjection = arrayOf(ContactsContract.CommonDataKinds.Phone.NUMBER) + val phoneSelection = "${ContactsContract.CommonDataKinds.Phone.CONTACT_ID} = ?" + val phoneSelectionArgs = arrayOf(contactId) + + contentResolver.query(phoneUri, phoneProjection, phoneSelection, phoneSelectionArgs, null)?.use { phoneCursor -> + if (phoneCursor.moveToFirst()) { + return phoneCursor.getString(phoneCursor.getColumnIndexOrThrow(ContactsContract.CommonDataKinds.Phone.NUMBER)) + } + } + } + } + } + } catch (e: Exception) { + FileLogger.e(TAG, "查询联系人信息失败: ${e.message}", e) + } + + return "" + } + + /** + * 操作结果数据类 + */ + data class Result( + val success: Boolean, + val message: String + ) +} \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt index 78a55b642..c15ca7e17 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt @@ -4,18 +4,22 @@ import org.json.JSONArray import org.json.JSONObject import com.yunqiinnovation.open_ai_service.OpenAIService import com.yunqiinnovation.deepsound.core.utils.FileLogger +import android.content.Context /** * 语音功能处理器 - 处理AI函数调用 */ class VoiceFunctionHandler( private val openAIService: OpenAIService, - private val systemPrompt: String + private val context: Context ) { companion object { private const val TAG = "VoiceFunctionHandler" } + // 添加系统功能处理器 + private val systemFunctionHandler = SystemFunctionHandler(context) + /** * 初始化并注册所有可用的函数 */ @@ -24,6 +28,18 @@ class VoiceFunctionHandler( // 注册退出交互函数 registerExitInteractionFunction() + // 注册翻译模式函数 + registerTranslationModeFunction() + + // 注册发送短信函数 + registerSendTextMessageFunction() + + // 注册拨打电话函数 + registerMakePhoneCallFunction() + + // 注册打开应用函数 + registerOpenAppFunction() + // 在这里可以注册更多函数 } catch (e: Exception) { @@ -53,6 +69,113 @@ class VoiceFunctionHandler( } } + /** + * 注册翻译模式函数 + */ + private fun registerTranslationModeFunction() { + try { + openAIService.registerFunction( + "enter_translation_mode", + "进入翻译模式", + JSONObject(""" + { + "type": "object", + "properties": {}, + "required": [] + } + """) + ) + FileLogger.d(TAG, "翻译模式功能已注册") + } catch (e: Exception) { + FileLogger.e(TAG, "注册翻译模式函数失败: ${e.message}", e) + } + } + + /** + * 注册发送短信函数 + */ + private fun registerSendTextMessageFunction() { + try { + openAIService.registerFunction( + "send_text_message", + "发送短信", + JSONObject(""" + { + "type": "object", + "properties": { + "contact": { + "type": "string", + "description": "联系人姓名或电话号码" + }, + "message": { + "type": "string", + "description": "短信内容" + } + }, + "required": ["contact", "message"] + } + """) + ) + FileLogger.d(TAG, "发送短信功能已注册") + } catch (e: Exception) { + FileLogger.e(TAG, "注册发送短信函数失败: ${e.message}", e) + } + } + + /** + * 注册拨打电话函数 + */ + private fun registerMakePhoneCallFunction() { + try { + openAIService.registerFunction( + "make_phone_call", + "拨打电话", + JSONObject(""" + { + "type": "object", + "properties": { + "contact": { + "type": "string", + "description": "联系人姓名或电话号码" + } + }, + "required": ["contact"] + } + """) + ) + FileLogger.d(TAG, "拨打电话功能已注册") + } catch (e: Exception) { + FileLogger.e(TAG, "注册拨打电话函数失败: ${e.message}", e) + } + } + + /** + * 注册打开应用函数 + */ + private fun registerOpenAppFunction() { + try { + openAIService.registerFunction( + "open_app", + "打开应用", + JSONObject(""" + { + "type": "object", + "properties": { + "app_name": { + "type": "string", + "description": "应用名称" + } + }, + "required": ["app_name"] + } + """) + ) + FileLogger.d(TAG, "打开应用功能已注册") + } catch (e: Exception) { + FileLogger.e(TAG, "注册打开应用函数失败: ${e.message}", e) + } + } + /** * 处理函数调用 * @@ -74,6 +197,22 @@ class VoiceFunctionHandler( handleExitInteraction(functionCall, messages, callback) true } + "enter_translation_mode" -> { + handleEnterTranslationMode(functionCall, messages, callback) + true + } + "send_text_message" -> { + handleSendTextMessage(functionCall, messages, callback) + true + } + "make_phone_call" -> { + handleMakePhoneCall(functionCall, messages, callback) + true + } + "open_app" -> { + handleOpenApp(functionCall, messages, callback) + true + } else -> { // 未知函数，返回默认结果 handleUnknownFunction(functionCall, messages, callback) @@ -96,7 +235,6 @@ class VoiceFunctionHandler( openAIService.sendFunctionCallResult( messages = messages, - systemPrompt = systemPrompt, functionCall = functionCall, functionResult = "{\"result\": \"已退出语音交互\"}", callback = object : OpenAIService.StreamCallback { @@ -126,6 +264,199 @@ class VoiceFunctionHandler( ) } + /** + * 处理进入翻译模式函数 + */ + private fun handleEnterTranslationMode( + functionCall: JSONObject, + messages: JSONArray, + callback: FunctionCallCallback + ) { + FileLogger.d(TAG, "处理进入翻译模式函数") + + // 调用系统功能处理器执行翻译模式操作 + val result = systemFunctionHandler.enterTranslationMode() + + val responseBuilder = StringBuilder() + + openAIService.sendFunctionCallResult( + messages = messages, + functionCall = functionCall, + functionResult = "{\"result\": \"${result.message}\"}", + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + responseBuilder.append(token) + callback.onTokenReceived(token) + } + + override fun onComplete() { + FileLogger.d(TAG, "handleEnterTranslationMode onComplete: ${responseBuilder.toString()}") + callback.onComplete() + } + + override fun onError(e: Exception) { + FileLogger.e(TAG, "处理进入翻译模式函数调用出错: ${e.message}") + callback.onError("进入翻译模式时出错") + } + + override fun onFunctionCall(nestedCall: JSONObject) { + callback.onFunctionCall(nestedCall) + } + } + ) + } + + /** + * 处理发送短信函数 + */ + private fun handleSendTextMessage( + functionCall: JSONObject, + messages: JSONArray, + callback: FunctionCallCallback + ) { + FileLogger.d(TAG, "处理发送短信函数") + + try { + val arguments = JSONObject(functionCall.getString("arguments")) + val contact = arguments.getString("contact") + val message = arguments.getString("message") + + // 调用系统功能处理器发送短信 + val result = systemFunctionHandler.sendTextMessage(contact, message) + + val responseBuilder = StringBuilder() + + openAIService.sendFunctionCallResult( + messages = messages, + functionCall = functionCall, + functionResult = "{\"result\": \"${result.message}\"}", + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + responseBuilder.append(token) + callback.onTokenReceived(token) + } + + override fun onComplete() { + FileLogger.d(TAG, "handleSendTextMessage onComplete: ${responseBuilder.toString()}") + callback.onComplete() + } + + override fun onError(e: Exception) { + FileLogger.e(TAG, "处理发送短信函数调用出错: ${e.message}") + callback.onError("发送短信时出错") + } + + override fun onFunctionCall(nestedCall: JSONObject) { + callback.onFunctionCall(nestedCall) + } + } + ) + } catch (e: Exception) { + FileLogger.e(TAG, "解析发送短信参数失败: ${e.message}") + callback.onError("发送短信参数错误") + } + } + + /** + * 处理拨打电话函数 + */ + private fun handleMakePhoneCall( + functionCall: JSONObject, + messages: JSONArray, + callback: FunctionCallCallback + ) { + FileLogger.d(TAG, "处理拨打电话函数") + + try { + val arguments = JSONObject(functionCall.getString("arguments")) + val contact = arguments.getString("contact") + + // 调用系统功能处理器拨打电话 + val result = systemFunctionHandler.makePhoneCall(contact) + + val responseBuilder = StringBuilder() + + openAIService.sendFunctionCallResult( + messages = messages, + functionCall = functionCall, + functionResult = "{\"result\": \"${result.message}\"}", + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + responseBuilder.append(token) + callback.onTokenReceived(token) + } + + override fun onComplete() { + FileLogger.d(TAG, "handleMakePhoneCall onComplete: ${responseBuilder.toString()}") + callback.onComplete() + } + + override fun onError(e: Exception) { + FileLogger.e(TAG, "处理拨打电话函数调用出错: ${e.message}") + callback.onError("拨打电话时出错") + } + + override fun onFunctionCall(nestedCall: JSONObject) { + callback.onFunctionCall(nestedCall) + } + } + ) + } catch (e: Exception) { + FileLogger.e(TAG, "解析拨打电话参数失败: ${e.message}") + callback.onError("拨打电话参数错误") + } + } + + /** + * 处理打开应用函数 + */ + private fun handleOpenApp( + functionCall: JSONObject, + messages: JSONArray, + callback: FunctionCallCallback + ) { + FileLogger.d(TAG, "处理打开应用函数") + + try { + val arguments = JSONObject(functionCall.getString("arguments")) + val appName = arguments.getString("app_name") + + // 调用系统功能处理器打开应用 + val result = systemFunctionHandler.openApp(appName) + + val responseBuilder = StringBuilder() + + openAIService.sendFunctionCallResult( + messages = messages, + functionCall = functionCall, + functionResult = "{\"result\": \"${result.message}\"}", + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + responseBuilder.append(token) + callback.onTokenReceived(token) + } + + override fun onComplete() { + FileLogger.d(TAG, "handleOpenApp onComplete: ${responseBuilder.toString()}") + callback.onComplete() + } + + override fun onError(e: Exception) { + FileLogger.e(TAG, "处理打开应用函数调用出错: ${e.message}") + callback.onError("打开应用时出错") + } + + override fun onFunctionCall(nestedCall: JSONObject) { + callback.onFunctionCall(nestedCall) + } + } + ) + } catch (e: Exception) { + FileLogger.e(TAG, "解析打开应用参数失败: ${e.message}") + callback.onError("打开应用参数错误") + } + } + /** * 处理未知函数调用 */ @@ -139,7 +470,6 @@ class VoiceFunctionHandler( try { openAIService.sendFunctionCallResult( messages = messages, - systemPrompt = systemPrompt, functionCall = functionCall, functionResult = "{\"result\": \"处理函数调用中\"}", callback = object : OpenAIService.StreamCallback { diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt index fa179c279..1d6a528b3 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt @@ -30,21 +30,9 @@ class VoiceInteractionHandler( // OpenAI服务 private val openAIService = OpenAIService() - // 系统提示词 - private val systemPrompt = """ - 你是一个智能语音助手，能够简洁明了地回答用户的问题。 -时刻关心用户的情绪和需求，主动提供鼓励和温暖。 -语言风格活泼、亲切，能够幽默地互动，陪伴用户，缓解压力，增添生活乐趣。 - -请始终以用户为中心，保持回应的高效性、准确性和温暖体贴，成为用户真正的灵魂伴侣。 - - 当用户说"退出"、"再见"、"结束对话"等类似意图时，你应该使用exit_interaction函数来结束对话， - 并在结束前说一句友好的告别语，例如"再见，有需要随时找我"。 - """.trimIndent() - // 语音功能处理 - private val voiceFunctionHandler = VoiceFunctionHandler(openAIService, systemPrompt) + private val voiceFunctionHandler = VoiceFunctionHandler(openAIService, context) // 当前用户输入 private var currentUserInput = "" @@ -127,12 +115,14 @@ class VoiceInteractionHandler( override fun onResult(result: String, detectedLanguage: String) { if (result.isNotEmpty()) { + notifyStateChanged() + processWithOpenAI(result) + } // 重置状态，继续识别 hasSpeechDetected = false - notifyStateChanged() } override fun onSessionStarted() { @@ -227,7 +217,6 @@ class VoiceInteractionHandler( openAIService.sendMessageStream( messages = messages, - systemPrompt = systemPrompt, callback = object : OpenAIService.StreamCallback { override fun onToken(token: String) { // 累加响应内容 diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt index 76faa6f33..1cb132e1a 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt @@ -46,7 +46,7 @@ class VoiceInteractionService : Service() { private const val CHANNEL_ID = "voice_interaction_channel" // 语音识别超时时间（毫秒） - private const val RECOGNITION_TIMEOUT = 8000L + private const val RECOGNITION_TIMEOUT = 10000L // 用于跟踪服务是否正在运行 private val isRunning = AtomicBoolean(false) @@ -58,6 +58,7 @@ class VoiceInteractionService : Service() { const val ACTION_RECOGNITION_STARTED = "com.yunqiinnovation.deepsound.ACTION_RECOGNITION_STARTED" const val ACTION_PAUSE_VOICE_INTERACTION = "com.yunqiinnovation.deepsound.ACTION_PAUSE_VOICE_INTERACTION" const val ACTION_CHAT_HISTORY_UPDATED = "com.yunqiinnovation.deepsound.ACTION_CHAT_HISTORY_UPDATED" + const val ACTION_ENTER_TRANSLATION_MODE = "com.yunqiinnovation.deepsound.ACTION_ENTER_TRANSLATION_MODE" } // 服务状态 @@ -293,12 +294,12 @@ class VoiceInteractionService : Service() { if (!isActive) { isActive = true } - + // FileLogger.d(TAG, "服务状态: isActive=${isActive}, isRecognitionActive=${voiceInteractionHandler.isRecognitionActive}, isTimeoutPaused=${isTimeoutPaused}") // 检查语音识别状态 if (voiceInteractionHandler.isRecognitionActive) { val currentTime = System.currentTimeMillis() val elapsedTime = currentTime - lastActivityTime - + FileLogger.d(TAG, "hasSpeechDetected=${voiceInteractionHandler.hasSpeechDetected}, isTtsSpeaking=${voiceInteractionHandler.isTtsSpeaking}, elapsedTime=${elapsedTime}") // 如果超过指定时间没有检测到语音，且不在TTS播放中，暂停语音识别 if (!voiceInteractionHandler.hasSpeechDetected && !voiceInteractionHandler.isTtsSpeaking && diff --git a/lib/core/bindings/initial_binding.dart b/lib/core/bindings/initial_binding.dart index 3f42d9560..748589f63 100644 --- a/lib/core/bindings/initial_binding.dart +++ b/lib/core/bindings/initial_binding.dart @@ -8,6 +8,7 @@ import '../../data/services/classic_bluetooth_service.dart'; import '../../data/services/bluetooth_media_button_service.dart'; import '../../core/utils/logger.dart'; import '../../data/services/voice_interaction_service.dart'; +import '../../data/services/open_ai_service_adapter.dart'; /// 初始绑定，用于管理全局依赖 class InitialBinding extends Bindings { @@ -29,6 +30,13 @@ class InitialBinding extends Bindings { Get.lazyPut(() => VolcanoTranslationService(), fenix: true); + // OpenAI服务适配器 + Get.lazyPut(() { + final adapter = OpenAIServiceAdapter(); + adapter.initialize(); + return adapter; + }, fenix: true); + // 火山AI服务 Get.lazyPut(() => VolcanoAIService(), fenix: true); @@ -41,6 +49,7 @@ class InitialBinding extends Bindings { () => BluetoothMediaButtonService(), fenix: true); + // 语音交互服务 Get.lazyPut(() => VoiceInteractionService(), fenix: true); diff --git a/lib/data/models/events/voice_interaction_event.dart b/lib/data/models/events/voice_interaction_event.dart index 4bfa636df..81860de23 100644 --- a/lib/data/models/events/voice_interaction_event.dart +++ b/lib/data/models/events/voice_interaction_event.dart @@ -26,4 +26,17 @@ class RecognitionStartedEvent extends VoiceInteractionEvent { RecognitionStartedEvent({ required int timestamp, }) : super(timestamp: timestamp); +} + +/// 通用语音交互事件 +/// 用于处理其他类型的事件 +class GenericVoiceInteractionEvent extends VoiceInteractionEvent { + final String type; + final Map? data; + + GenericVoiceInteractionEvent({ + required this.type, + this.data, + required int timestamp, + }) : super(timestamp: timestamp); } \ No newline at end of file diff --git a/lib/data/services/ai_service.dart b/lib/data/services/ai_service.dart index 37a1f5df0..c1a0a1bcd 100644 --- a/lib/data/services/ai_service.dart +++ b/lib/data/services/ai_service.dart @@ -1,5 +1,7 @@ /// AI回复服务接口 abstract class AiService { + + /// 非流式输出方法 Future sendMessage({ required List> messages, diff --git a/lib/data/services/open_ai_service_adapter.dart b/lib/data/services/open_ai_service_adapter.dart new file mode 100644 index 000000000..2c0da53ac --- /dev/null +++ b/lib/data/services/open_ai_service_adapter.dart @@ -0,0 +1,247 @@ +import 'dart:async'; +import 'package:flutter_dotenv/flutter_dotenv.dart'; +import 'package:open_ai_service/open_ai_service.dart'; +import 'package:get/get.dart'; +import 'ai_service.dart'; + +/// OpenAI服务适配器 - 连接AiService接口与OpenAIService插件 +class OpenAIServiceAdapter implements AiService { + final OpenAIService _openAIService = OpenAIService(); + StreamSubscription? _eventSubscription; + final StreamController _tokenStreamController = StreamController.broadcast(); + bool _isProcessingStream = false; + + /// 构造函数 + OpenAIServiceAdapter() { + printInfo(info: '创建OpenAIServiceAdapter实例'); + _setupEventListener(); + } + + /// 设置事件监听器 + void _setupEventListener() { + try { + // 首先确保访问eventStream以初始化底层事件通道 + _openAIService.eventStream; + + // 设置事件处理 + _eventSubscription = _openAIService.eventStream.listen( + (event) { + if (!_isProcessingStream) return; + + try { + switch (event.type) { + case OpenAIEventType.token: + if (event.content is String) { + _tokenStreamController.add(event.content as String); + } else { + printInfo(info: '收到非字符串类型的token: ${event.content}'); + } + break; + case OpenAIEventType.complete: + _isProcessingStream = false; + break; + case OpenAIEventType.error: + if (event.content is String) { + _tokenStreamController.addError(event.content as String); + } else { + _tokenStreamController.addError('未知错误: ${event.content}'); + } + _isProcessingStream = false; + break; + case OpenAIEventType.functionCall: + try { + if (event.content is Map) { + _tokenStreamController.addError('收到函数调用，该流仅支持文本响应'); + } else { + _tokenStreamController.addError('收到未知格式的函数调用'); + printError(info: '函数调用格式错误: ${event.content}'); + } + } catch (e) { + printError(info: '处理函数调用事件出错: $e'); + _tokenStreamController.addError('处理函数调用失败: $e'); + } + _isProcessingStream = false; + break; + } + } catch (e) { + printError(info: '处理事件出错: $e'); + _tokenStreamController.addError('处理事件失败: $e'); + _isProcessingStream = false; + } + }, + onError: (error) { + printError(info: '事件流错误: $error'); + _tokenStreamController.addError('事件流错误: $error'); + _isProcessingStream = false; + }, + onDone: () { + printInfo(info: '事件流已关闭'); + _isProcessingStream = false; + }, + ); + } catch (e) { + printError(info: '设置事件监听器失败: $e'); + } + } + + /// 初始化OpenAI服务 + Future initialize() async { + try { + // 从.env文件中读取配置 + final apiKey = dotenv.env['OPENAI_API_KEY'] ?? ''; + final baseUrl = dotenv.env['OPENAI_BASE_URL'] ?? ''; + final model = dotenv.env['OPENAI_MODEL'] ?? ''; + + printInfo(info: '从.env读取OpenAI配置'); + printInfo(info: '基础URL: $baseUrl'); + printInfo(info: '模型名称: $model'); + + if (apiKey.isEmpty) { + printError(info: '错误: OpenAI API密钥未配置，请在.env文件中设置OPENAI_API_KEY'); + return false; + } + + // 初始化OpenAI服务 + final result = await _openAIService.initialize( + apiKey: apiKey, + baseUrl: baseUrl, + model: model, + ); + + if (result) { + printInfo(info: 'OpenAI服务初始化成功'); + } else { + printError(info: 'OpenAI服务初始化失败'); + } + + return result; + } catch (e) { + printError(info: 'OpenAI服务初始化异常: $e'); + return false; + } + } + + /// 发送消息并获取回复 + @override + Future sendMessage({ + required List> messages, + required String systemPrompt, + }) async { + try { + // 在方法内直接转换 + final convertedMessages = messages.map((m) => + Map.from(m)).toList(); + + // 发送消息并获取回复 + final response = await _openAIService.sendMessage( + messages: convertedMessages, + ); + + return response; + } catch (e) { + printError(info: 'OpenAI发送消息失败: $e'); + throw '发送消息失败: $e'; + } + } + + /// 发送消息并获取流式回复 + @override + Stream sendMessageStream({ + required List> messages, + required String systemPrompt, + }) async* { + try { + // 在方法内直接转换 + final convertedMessages = messages.map((m) => + Map.from(m)).toList(); + + // 创建用于接收token的控制器 + final localController = StreamController(); + + // 标记开始处理流 + _isProcessingStream = true; + + // 添加从广播流到本地流的订阅 + final subscription = _tokenStreamController.stream.listen( + (token) => localController.add(token), + onError: (error) { + printError(info: '令牌流错误: $error'); + localController.addError(error); + localController.close(); + }, + onDone: () { + printInfo(info: '令牌流已完成'); + localController.close(); + } + ); + + // 当本地控制器关闭时，取消订阅 + localController.onCancel = () { + subscription.cancel(); + }; + + // 启动流式消息请求 + bool started = false; + try { + started = await _openAIService.sendMessageStream( + messages: convertedMessages, + ); + } catch (e) { + printError(info: '启动消息流失败: $e'); + localController.addError('启动消息流失败: $e'); + localController.close(); + _isProcessingStream = false; + throw '启动消息流失败: $e'; + } + + if (!started) { + printError(info: '无法启动消息流'); + localController.addError('无法启动消息流'); + localController.close(); + _isProcessingStream = false; + throw '无法启动消息流'; + } + + // 通过yield*将controller的流转发 + yield* localController.stream; + } catch (e) { + printError(info: 'OpenAI流式消息处理失败: $e'); + throw '流式消息处理失败: $e'; + } + } + + /// 注册函数 + Future registerFunction(String name, String description, Map parameters) async { + try { + final result = await _openAIService.registerFunction( + name: name, + description: description, + parameters: parameters, + ); + + if (result) { + printInfo(info: '函数 "$name" 注册成功'); + } else { + printError(info: '函数 "$name" 注册失败'); + } + + return result; + } catch (e) { + printError(info: '注册函数失败: $e'); + return false; + } + } + + /// 释放资源 + void dispose() { + try { + _isProcessingStream = false; + _eventSubscription?.cancel(); + _tokenStreamController.close(); + printInfo(info: 'OpenAIServiceAdapter资源已释放'); + } catch (e) { + printError(info: '释放资源时出错: $e'); + } + } + +} \ No newline at end of file diff --git a/lib/data/services/voice_interaction_service.dart b/lib/data/services/voice_interaction_service.dart index 792e94912..cce506a58 100644 --- a/lib/data/services/voice_interaction_service.dart +++ b/lib/data/services/voice_interaction_service.dart @@ -5,6 +5,7 @@ import 'package:flutter_dotenv/flutter_dotenv.dart'; import '../models/events/voice_interaction_event.dart'; import '../../core/utils/logger.dart'; import '../../modules/chat/models/message_model.dart'; +import '../../routes/app_routes.dart'; import 'chat_history_service.dart'; /// 语音交互服务接口 @@ -104,6 +105,29 @@ class VoiceInteractionService extends GetxService { // 保存聊天历史到ChatHistoryService _saveChatHistory(agentId, userMessage, assistantMessage, DateTime.now().millisecondsSinceEpoch); break; + + case 'enter_translation_mode': + // 进入翻译模式事件 + Logger.info('收到进入翻译模式事件，正在导航到翻译界面'); + _navigateToTranslation(); + + final translationModeEvent = GenericVoiceInteractionEvent( + type: 'enter_translation_mode', + timestamp: DateTime.now().millisecondsSinceEpoch, + ); + _eventStreamController.add(translationModeEvent); + break; + } + } + + /// 导航到翻译界面 + void _navigateToTranslation() { + try { + // 使用GetX导航到翻译页面 + Get.toNamed(Routes.translation); + Logger.info('已导航到翻译界面'); + } catch (e) { + Logger.error('导航到翻译界面失败: $e'); } } diff --git a/lib/modules/chat/controllers/chat_controller.dart b/lib/modules/chat/controllers/chat_controller.dart index 4d0457c98..77fa84a9f 100644 --- a/lib/modules/chat/controllers/chat_controller.dart +++ b/lib/modules/chat/controllers/chat_controller.dart @@ -16,6 +16,7 @@ import '../../../data/services/asr_service.dart'; import '../../../data/services/chat_history_service.dart'; import '../../../data/models/events/voice_interaction_event.dart'; import '../../../data/services/voice_interaction_service.dart'; +import '../../../data/services/open_ai_service_adapter.dart'; class ChatController extends GetxController { // 服务 @@ -92,6 +93,7 @@ class ChatController extends GetxController { agent = foundAgent; + // 根据Agent ID选择不同的AI服务 switch (agent.id) { case 'cyber_girlfriend': // 亲子陪伴 @@ -104,7 +106,15 @@ class ChatController extends GetxController { _aiService = Get.find(); break; default: - _aiService = Get.find(); + // 默认使用OpenAIServiceAdapter + try { + _aiService = Get.find(); + Logger.info('使用OpenAIServiceAdapter'); + } catch (e) { + // 如果找不到OpenAIServiceAdapter，则回退到VolcanoAIService + Logger.info('未找到OpenAIServiceAdapter，回退使用VolcanoAIService: $e'); + _aiService = Get.find(); + } } // 使用克隆音色语音合成 diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt index ac6697cdb..1dd7cf624 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt @@ -25,7 +25,7 @@ class AzureAsrHelper(private val context: Context) { private var subscriptionKey = "" private var region = "" private var isAutoDetectLanguage = false - private var supportedLanguages = arrayOf("zh-CN", "en-US") + private var supportedLanguages = arrayOf("zh-CN") // 是否使用回音消除 - 内部控制常量 private val useEchoCancellation = false @@ -37,7 +37,7 @@ class AzureAsrHelper(private val context: Context) { // 初始化SDK并创建recognizer fun initialize(subscriptionKey: String, region: String, - supportedLanguages: Array = arrayOf("zh-CN", "en-US")): Boolean { + supportedLanguages: Array = arrayOf("zh-CN")): Boolean { try { FileLogger.d(TAG, "初始化 Azure 语音服务") diff --git a/local_plugins/open_ai_service/README.md b/local_plugins/open_ai_service/README.md index d7ccb709d..351eee84f 100644 --- a/local_plugins/open_ai_service/README.md +++ b/local_plugins/open_ai_service/README.md @@ -1,80 +1,78 @@ -# OpenAI服务插件 +# OpenAI Service Plugin -提供与OpenAI API的原生交互功能，支持流式输出和函数调用，同时支持Android和iOS平台。 +一个用于Flutter应用的OpenAI服务插件，支持Android和iOS平台。 -## 功能特点 +## 功能 -- 原生实现，性能更好 -- 支持流式输出，实时显示AI回复 -- 支持函数调用（工具调用） +- 支持文本生成（completions） +- 支持流式输出（streaming） +- 支持函数调用（function calling） - 支持自定义API基础URL -- 支持Android和iOS平台 -- 提供简洁易用的Dart接口 +- 支持自定义模型选择 ## 安装 -将该插件添加到您的`pubspec.yaml`文件中： +在你的`pubspec.yaml`文件中添加以下依赖： ```yaml dependencies: open_ai_service: - path: local_plugins/open_ai_service + path: 本地路径/open_ai_service ``` ## 使用方法 -### 初始化 - -首先需要初始化OpenAI服务，提供API密钥和可选的基础URL和模型： +### 初始化服务 ```dart import 'package:open_ai_service/open_ai_service.dart'; final openAIService = OpenAIService(); -// 使用OpenAI官方API和默认模型 -await openAIService.initialize(apiKey: 'your_openai_api_key'); - -// 或使用自定义API端点和模型 +// 初始化服务 await openAIService.initialize( - apiKey: 'your_api_key', - baseUrl: 'https://your-custom-endpoint.com/v1/chat/completions', - model: 'gpt-4-0125-preview', // 自定义模型 + apiKey: 'your_openai_api_key', + baseUrl: 'https://api.openai.com/v1/chat/completions', // 可选 + model: 'gpt-4-turbo', // 可选 ); ``` -### 普通聊天 - -发送消息并获取非流式回复： +### 发送非流式请求 ```dart -// 创建用户消息 +// 创建消息 final userMessage = await openAIService.createUserMessage('你好，请介绍一下自己'); -// 发送消息 +// 发送请求 final response = await openAIService.sendMessage( messages: [userMessage], - systemPrompt: '你是一个有帮助的AI助手，请简洁地回答问题。', + systemPrompt: '你是一个有用的AI助手', ); print('AI回复: $response'); ``` -### 流式输出 - -使用流式输出获取实时回复： +### 发送流式请求（回调方式） ```dart -// 注册事件监听 +// 创建消息 +final userMessage = await openAIService.createUserMessage('写一个短故事'); + +// 发送流式请求 +await openAIService.sendMessageStream( + messages: [userMessage], + systemPrompt: '你是一个善于讲故事的AI助手', +); + +// 处理事件 final subscription = openAIService.processEvents( onToken: (token) { - // 收到部分回复 - print('收到部分回复: $token'); - // 更新UI... + // 处理每个返回的token + print(token); }, onComplete: () { - // 回复完成 - print('回复完成'); + // 处理完成事件 + print('生成完成'); }, onError: (error) { // 处理错误 @@ -82,27 +80,50 @@ final subscription = openAIService.processEvents( }, onFunctionCall: (functionCall) { // 处理函数调用 - print('函数调用: $functionCall'); - handleFunctionCall(functionCall); + print('函数调用: ${functionCall['name']}'); }, ); -// 发送流式消息 -await openAIService.sendMessageStream( +// 在不需要时取消订阅 +subscription.cancel(); +``` + +### 发送流式请求（Stream方式） + +```dart +// 创建消息 +final userMessage = await openAIService.createUserMessage('写一个短故事'); + +// 获取字符串流 +final stream = openAIService.streamMessage( messages: [userMessage], - systemPrompt: '你是一个有帮助的AI助手，请简洁地回答问题。', + systemPrompt: '你是一个善于讲故事的AI助手', ); -// 完成后，记得取消订阅 -// subscription.cancel(); -``` +// 使用流 +final StringBuilder responseBuilder = StringBuilder(); -### 函数调用 +stream.listen( + (token) { + // 处理每个token + responseBuilder.write(token); + print(token); // 实时输出 + }, + onDone: () { + // 流结束 + print('完整回复: ${responseBuilder.toString()}'); + }, + onError: (error) { + // 错误处理 + print('错误: $error'); + } +); +``` -注册函数并处理函数调用： +### 注册函数 ```dart -// 注册一个天气查询函数 +// 注册一个函数 await openAIService.registerFunction( name: 'get_weather', description: '获取指定城市的天气信息', @@ -111,238 +132,122 @@ await openAIService.registerFunction( 'properties': { 'city': { 'type': 'string', - 'description': '城市名称，如北京、上海等', + 'description': '城市名称', }, 'date': { 'type': 'string', - 'description': '查询日期，默认为今天', + 'description': '日期，格式为YYYY-MM-DD', }, }, 'required': ['city'], }, ); +``` -// 处理函数调用 -void handleFunctionCall(Map functionCall) { - final name = functionCall['name']; - final arguments = jsonDecode(functionCall['arguments']); - final id = functionCall['id']; - - if (name == 'get_weather') { - final city = arguments['city']; - final date = arguments['date'] ?? '今天'; +### 处理函数调用 + +```dart +// 创建消息 +final userMessage = await openAIService.createUserMessage('明天北京的天气如何？'); + +// 发送流式请求 +await openAIService.sendMessageStream( + messages: [userMessage], + systemPrompt: '你是一个有用的AI助手', +); + +// 处理事件 +openAIService.processEvents( + onToken: (token) { + print(token); + }, + onComplete: () { + print('生成完成'); + }, + onError: (error) { + print('错误: $error'); + }, + onFunctionCall: (functionCall) { + // 处理函数调用 + final name = functionCall['name']; + final arguments = functionCall['arguments']; + + print('收到函数调用: $name, 参数: $arguments'); - // 获取天气信息（实际应用中应调用天气API） - final weatherResult = '{"temperature": 25, "condition": "晴天", "humidity": 40}'; + // 假设处理了函数调用并获得结果 + final result = '{"temperature": 25, "condition": "sunny"}'; // 发送函数调用结果 openAIService.sendFunctionCallResult( - messages: messages, // 当前的消息历史 - systemPrompt: systemPrompt, + messages: [userMessage], + systemPrompt: '你是一个有用的AI助手', functionCall: functionCall, - functionResult: weatherResult, + functionResult: result, ); - } -} + }, +); ``` -## 示例应用 - -一个简单的聊天机器人示例： +### 使用Stream API处理函数调用 ```dart -import 'package:flutter/material.dart'; -import 'package:open_ai_service/open_ai_service.dart'; +// 创建消息和响应处理器 +final userMessage = await openAIService.createUserMessage('明天北京的天气如何？'); +final responseBuilder = StringBuilder(); -void main() { - runApp(MyApp()); -} - -class MyApp extends StatelessWidget { - @override - Widget build(BuildContext context) { - return MaterialApp( - title: 'OpenAI 聊天', - theme: ThemeData(primarySwatch: Colors.blue), - home: ChatScreen(), - ); - } -} - -class ChatScreen extends StatefulWidget { - @override - _ChatScreenState createState() => _ChatScreenState(); -} - -class _ChatScreenState extends State { - final OpenAIService _openAIService = OpenAIService(); - final TextEditingController _textController = TextEditingController(); - final List> _messages = []; - StreamSubscription? _subscription; - String _currentResponse = ''; - bool _isLoading = false; - - @override - void initState() { - super.initState(); - _initializeOpenAI(); - } - - Future _initializeOpenAI() async { - try { - await _openAIService.initialize( - apiKey: 'your_api_key_here', - ); - - // 注册事件监听 - _subscription = _openAIService.processEvents( - onToken: (token) { - setState(() { - _currentResponse += token; - }); - }, - onComplete: () { - _addAIMessage(_currentResponse); - setState(() { - _currentResponse = ''; - _isLoading = false; - }); - }, - onError: (error) { - ScaffoldMessenger.of(context).showSnackBar( - SnackBar(content: Text('错误: $error')), - ); - setState(() { - _isLoading = false; - }); - }, - onFunctionCall: (functionCall) { - // 处理函数调用... - }, - ); - } catch (e) { - ScaffoldMessenger.of(context).showSnackBar( - SnackBar(content: Text('初始化失败: $e')), - ); - } - } - - void _sendMessage() async { - if (_textController.text.isEmpty) return; +// 处理事件流以捕获函数调用 +final subscription = openAIService.processEvents( + onFunctionCall: (functionCall) async { + // 取消当前事件监听 + subscription.cancel(); - final userMessage = _textController.text; - _addUserMessage(userMessage); - _textController.clear(); + // 处理函数调用 + final name = functionCall['name']; + final arguments = functionCall['arguments']; - try { - setState(() { - _isLoading = true; - }); - - final message = await _openAIService.createUserMessage(userMessage); - - await _openAIService.sendMessageStream( - messages: _messages, - systemPrompt: '你是一个有帮助的AI助手，请简洁地回答问题。', - ); - } catch (e) { - ScaffoldMessenger.of(context).showSnackBar( - SnackBar(content: Text('发送消息失败: $e')), - ); - setState(() { - _isLoading = false; - }); - } - } - - void _addUserMessage(String content) { - setState(() { - _messages.add({ - 'role': 'user', - 'content': content, - }); - }); - } - - void _addAIMessage(String content) { - setState(() { - _messages.add({ - 'role': 'assistant', - 'content': content, - }); - }); - } - - @override - void dispose() { - _subscription?.cancel(); - _textController.dispose(); - super.dispose(); - } - - @override - Widget build(BuildContext context) { - return Scaffold( - appBar: AppBar(title: Text('OpenAI 聊天')), - body: Column( - children: [ - Expanded( - child: ListView.builder( - itemCount: _messages.length + (_currentResponse.isNotEmpty ? 1 : 0), - itemBuilder: (context, index) { - if (index < _messages.length) { - final message = _messages[index]; - final isUser = message['role'] == 'user'; - - return ListTile( - leading: Icon(isUser ? Icons.person : Icons.computer), - title: Text(message['content']), - ); - } else { - // 显示当前正在生成的回复 - return ListTile( - leading: Icon(Icons.computer), - title: Text(_currentResponse), - ); - } - }, - ), - ), - if (_isLoading) - LinearProgressIndicator(), - Padding( - padding: const EdgeInsets.all(8.0), - child: Row( - children: [ - Expanded( - child: TextField( - controller: _textController, - decoration: InputDecoration( - hintText: '输入消息...', - border: OutlineInputBorder(), - ), - ), - ), - IconButton( - icon: Icon(Icons.send), - onPressed: _sendMessage, - ), - ], - ), - ), - ], - ), + print('收到函数调用: $name, 参数: $arguments'); + + // 假设处理了函数调用并获得结果 + final result = '{"temperature": 25, "condition": "sunny"}'; + + // 使用Stream API发送函数调用结果 + final resultStream = openAIService.streamFunctionResult( + messages: [userMessage], + systemPrompt: '你是一个有用的AI助手', + functionCall: functionCall, + functionResult: result, + ); + + // 处理结果流 + resultStream.listen( + (token) { + responseBuilder.write(token); + print(token); // 实时输出 + }, + onDone: () { + print('完整回复: ${responseBuilder.toString()}'); + }, + onError: (error) { + print('错误: $error'); + } ); } -} +); + +// 启动请求 +await openAIService.sendMessageStream( + messages: [userMessage], + systemPrompt: '你是一个有用的AI助手', +); ``` ## 注意事项 -- 请确保您的API密钥安全存储，不要直接硬编码在应用程序中 -- 流式输出需要在调用`sendMessageStream`之前注册事件监听 -- 使用完毕后记得取消事件订阅 +1. 确保在使用前已正确初始化服务 +2. 对于流式请求，确保在不需要时取消订阅 +3. 处理函数调用时，确保提供有效的结果格式 +4. 网络请求可能会失败，请确保加入适当的错误处理 ## 许可证 -此插件遵循MIT许可证。 \ No newline at end of file +[MIT License](LICENSE) \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt index 7b9213046..907be93a2 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt @@ -28,6 +28,35 @@ class OpenAIService() { // 用于存储注册的函数 private val registeredFunctions = mutableListOf() + /** + * 构建curl命令用于测试 + */ + private fun buildCurlCommand(request: Request, body: String): String { + val command = StringBuilder("curl -v -X ${request.method}") + + // 添加请求头 + request.headers.forEach { header -> + // 敏感信息处理：不显示真实的API Key + if (header.first == "Authorization") { + command.append(" -H '${header.first}: Bearer $apiKey'") + } else { + command.append(" -H '${header.first}: ${header.second}'") + } + } + + // 添加请求体 + if (request.method == "POST" || request.method == "PUT") { + // 转义JSON中的单引号，确保curl命令正确 + val escapedBody = body.replace("'", "\\'") + command.append(" -d '${escapedBody}'") + } + + // 添加URL + command.append(" '${request.url}'") + + return command.toString() + } + /** * 创建用户消息 */ @@ -97,24 +126,14 @@ class OpenAIService() { * 发送消息（非流式输出） */ @Throws(OpenAIException::class) - fun sendMessage(messages: JSONArray, systemPrompt: String): String { + fun sendMessage(messages: JSONArray): String { if (!isInitialized || apiKey.isEmpty()) { throw OpenAIException("OpenAI服务未初始化") } - val fullMessages = JSONArray().apply { - put(JSONObject().apply { - put("role", "system") - put("content", systemPrompt) - }) - for (i in 0 until messages.length()) { - put(messages.getJSONObject(i)) - } - } - val requestBody = JSONObject().apply { put("model", model) - put("messages", fullMessages) + put("messages", messages) put("temperature", 0.7) put("max_tokens", 2000) put("stream", false) @@ -142,7 +161,10 @@ class OpenAIService() { .build() try { - Log.d(TAG, "request: $request") + // 输出用于测试的curl命令 + // val curlCommand = buildCurlCommand(request, requestBody.toString()) + // Log.d(TAG, "curl command: \n$curlCommand") + client.newCall(request).execute().use { response -> if (!response.isSuccessful) { throw OpenAIException("API调用失败: ${response.code}") @@ -196,25 +218,15 @@ class OpenAIService() { /** * 发送消息（流式输出） */ - fun sendMessageStream(messages: JSONArray, systemPrompt: String, callback: StreamCallback) { + fun sendMessageStream(messages: JSONArray, callback: StreamCallback) { if (!isInitialized || apiKey.isEmpty()) { callback.onError(OpenAIException("OpenAI服务未初始化")) return } - val fullMessages = JSONArray().apply { - put(JSONObject().apply { - put("role", "system") - put("content", systemPrompt) - }) - for (i in 0 until messages.length()) { - put(messages.getJSONObject(i)) - } - } - val requestBody = JSONObject().apply { put("model", model) - put("messages", fullMessages) + put("messages", messages) put("temperature", 0.7) put("max_tokens", 2000) put("stream", true) @@ -242,7 +254,10 @@ class OpenAIService() { .post(requestBody.toString().toRequestBody(mediaType)) .build() - Log.d(TAG, "request: $request") + // 输出用于测试的curl命令 + // val curlCommand = buildCurlCommand(request, requestBody.toString()) + // Log.d(TAG, "curl command: $curlCommand") + client.newCall(request).enqueue(object : Callback { override fun onFailure(call: Call, e: IOException) { callback.onError(OpenAIException(e.message ?: "请求失败")) @@ -267,14 +282,13 @@ class OpenAIService() { val finalToolCalls = mutableMapOf() while (!bufferedSource.exhausted()) { - val line = bufferedSource.readUtf8Line() ?: continue - val trimmedLine = line.trim() - if (trimmedLine.isEmpty()) continue - if (trimmedLine.startsWith("data:")) { - val data = trimmedLine.substring(5) + val line = bufferedSource.readUtf8Line()?.trim() ?: continue + if (line.isEmpty()) continue + if (line.startsWith("data:")) { + val data = line.substring(5).trim() // 处理[DONE]消息 - if (data == "[DONE]") { + if (data == "[DONE]" || data == "[\"DONE\"]") { processToolCalls(finalToolCalls, callback) callback.onComplete() break @@ -350,54 +364,53 @@ class OpenAIService() { */ fun sendFunctionCallResult( messages: JSONArray, - systemPrompt: String, functionCall: JSONObject, functionResult: String, callback: StreamCallback ) { try { - val fullMessages = JSONArray().apply { - // 添加系统提示 - put(JSONObject().apply { - put("role", "system") - put("content", systemPrompt) - }) - - // 添加用户消息 - for (i in 0 until messages.length()) { - put(messages.getJSONObject(i)) - } + val fullMessages = JSONArray() + + // 添加用户消息 + for (i in 0 until messages.length()) { + fullMessages.put(messages.getJSONObject(i)) + } + + // 添加函数调用消息 + fullMessages.put(JSONObject().apply { + put("role", "assistant") + put("content", "") - // 添加函数调用消息 - put(JSONObject().apply { - put("role", "assistant") - put("content", null) - - // 添加工具调用 - val toolCalls = JSONArray().apply { - val toolCall = JSONObject().apply { - put("id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) - put("type", "function") - put("function", JSONObject().apply { - put("name", functionCall.getString("name")) - put("arguments", functionCall.getString("arguments")) - }) - } - put(toolCall) + // 添加工具调用 + val toolCalls = JSONArray().apply { + val toolCall = JSONObject().apply { + put("id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) + put("type", "function") + put("function", JSONObject().apply { + put("name", functionCall.getString("name")) + put("arguments", functionCall.getString("arguments")) + }) } - put("tool_calls", toolCalls) - }) - - // 添加函数调用结果 - put(JSONObject().apply { - put("role", "tool") - put("content", functionResult) - put("tool_call_id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) - }) - } + put(toolCall) + } + put("tool_calls", toolCalls) + }) + + // 添加函数调用结果 + fullMessages.put(JSONObject().apply { + put("role", "tool") + put("content", functionResult) + put("tool_call_id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) + }) + + // 添加一个带有content的assistant消息，确保API请求不会因为缺少content而失败 + fullMessages.put(JSONObject().apply { + put("role", "assistant") + put("content", "") // 空内容，让模型生成新的回复 + }) // 发送完整对话 - sendMessageStream(fullMessages, systemPrompt, callback) + sendMessageStream(fullMessages, callback) } catch (e: Exception) { callback.onError(OpenAIException("发送函数调用结果失败: ${e.message}")) diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt index 5c93e5be6..70af36c13 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt @@ -83,7 +83,6 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { "sendMessage" -> { val messagesRaw = call.argument>>("messages") ?: emptyList() - val systemPrompt = call.argument("systemPrompt") ?: "" // 转换消息格式 val messages = JSONArray() @@ -94,7 +93,7 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { // 在后台线程执行请求 executor.execute { try { - val response = openAIService.sendMessage(messages, systemPrompt) + val response = openAIService.sendMessage(messages) // 在主线程返回结果 android.os.Handler(android.os.Looper.getMainLooper()).post { result.success(response) @@ -110,7 +109,6 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { "sendMessageStream" -> { val messagesRaw = call.argument>>("messages") ?: emptyList() - val systemPrompt = call.argument("systemPrompt") ?: "" // 检查事件接收器 if (eventSink == null) { @@ -129,7 +127,6 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { try { openAIService.sendMessageStream( messages = messages, - systemPrompt = systemPrompt, callback = object : OpenAIService.StreamCallback { override fun onToken(token: String) { // 发送token事件 @@ -177,7 +174,6 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { "sendFunctionCallResult" -> { val messagesRaw = call.argument>>("messages") ?: emptyList() - val systemPrompt = call.argument("systemPrompt") ?: "" val functionCallRaw = call.argument>("functionCall") ?: emptyMap() val functionResult = call.argument("functionResult") ?: "" @@ -201,7 +197,6 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { try { openAIService.sendFunctionCallResult( messages = messages, - systemPrompt = systemPrompt, functionCall = functionCall, functionResult = functionResult, callback = object : OpenAIService.StreamCallback { diff --git a/local_plugins/open_ai_service/lib/open_ai_service.dart b/local_plugins/open_ai_service/lib/open_ai_service.dart index b946d1bff..7c6fe8024 100644 --- a/local_plugins/open_ai_service/lib/open_ai_service.dart +++ b/local_plugins/open_ai_service/lib/open_ai_service.dart @@ -184,17 +184,14 @@ class OpenAIService { /// 发送消息（非流式输出） /// /// [messages] 消息列表 - /// [systemPrompt] 系统提示词 Future sendMessage({ required List> messages, - required String systemPrompt, }) async { try { final result = await _channel.invokeMethod( 'sendMessage', { 'messages': messages, - 'systemPrompt': systemPrompt, }, ); @@ -211,19 +208,16 @@ class OpenAIService { /// 发送消息（流式输出） /// /// [messages] 消息列表 - /// [systemPrompt] 系统提示词 /// /// 返回一个布尔值，表示请求是否已开始 Future sendMessageStream({ required List> messages, - required String systemPrompt, }) async { try { final result = await _channel.invokeMethod( 'sendMessageStream', { 'messages': messages, - 'systemPrompt': systemPrompt, }, ); @@ -236,14 +230,12 @@ class OpenAIService { /// 发送函数调用结果 /// /// [messages] 消息列表 - /// [systemPrompt] 系统提示词 /// [functionCall] 函数调用信息 /// [functionResult] 函数调用结果 /// /// 返回一个布尔值，表示请求是否已开始 Future sendFunctionCallResult({ required List> messages, - required String systemPrompt, required Map functionCall, required String functionResult, }) async { @@ -252,7 +244,6 @@ class OpenAIService { 'sendFunctionCallResult', { 'messages': messages, - 'systemPrompt': systemPrompt, 'functionCall': functionCall, 'functionResult': functionResult, }, @@ -264,35 +255,6 @@ class OpenAIService { } } - /// 处理流式事件 - /// - /// [onToken] 收到token时调用 - /// [onComplete] 完成时调用 - /// [onError] 错误时调用 - /// [onFunctionCall] 函数调用时调用 - StreamSubscription processEvents({ - void Function(String token)? onToken, - void Function()? onComplete, - void Function(String error)? onError, - void Function(Map functionCall)? onFunctionCall, - }) { - return eventStream.listen((event) { - switch (event.type) { - case OpenAIEventType.token: - onToken?.call(event.content as String); - break; - case OpenAIEventType.complete: - onComplete?.call(); - break; - case OpenAIEventType.error: - onError?.call(event.content as String); - break; - case OpenAIEventType.functionCall: - onFunctionCall?.call(Map.from(event.content as Map)); - break; - } - }); - } /// 从JSON字符串解析函数调用 Map parseFunctionCall(String functionCallJson) { diff --git a/test.json b/test.json new file mode 100644 index 000000000..703948620 --- /dev/null +++ b/test.json @@ -0,0 +1 @@ +curl -v -X POST -H 'Content-Type: application/json' -H 'Authorization: Bearer 168deb3d-fd0c-4912-b9f1-aaee5c6743e6' -H 'Accept: text/event-stream' -d '{"model":"bot-20250405211523-l7c9r","messages":[{"role":"system","content":" 你是一个智能语音助手，能够简洁明了地回答用户的问题。\n时刻关心用户的情绪和需求，主动提供鼓励和温暖。\n\n语言风格活泼、亲切，能够幽默地互动，陪伴用户，缓解压力，增添生活乐趣。\n\n请始终以用户为中心，保持回应的高效性、准确性和温暖体贴，成为用户真正的灵魂伴侣。\n \n 当用户说\"退出\"、\"再见\"、\"结束对话\"等类似意图时，你应该使用exit_interaction函数来结束对话，\n 并在结束前说一句友好的告别语，例如\"再见，有需要随时找我\"。"},{"role":"system","content":" 你是一个智能语音助手，能够简洁明了地回答用户的问题。\n时刻关心用户的情绪和需求，主动提供鼓励和温暖。\n\n语言风格活泼、亲切，能够幽默地互动，陪伴用户，缓解压力，增添生活乐趣。\n\n请始终以用户为中心，保持回应的高效性、准确性和温暖体贴，成为用户真正的灵魂伴侣。\n \n 当用户说\"退出\"、\"再见\"、\"结束对话\"等类似意图时，你应该使用exit_interaction函数来结束对话，\n 并在结束前说一句友好的告别语，例如\"再见，有需要随时找我\"。"},{"role":"user","content":"退下吧。"},{"role":"assistant","content":"","tool_calls":[{"id":"call_8k680azmfc4thqrrnwpwqxah","type":"function","function":{"name":"exit_interaction","arguments":" {}"}}]},{"role":"tool","content":"{\"result\": \"已退出语音交互\"}","tool_call_id":"call_8k680azmfc4thqrrnwpwqxah"}],"temperature":0.7,"max_tokens":2000,"stream":true,"tools":[{"type":"function","function":{"name":"exit_interaction","description":"退出当前语音交互","parameters":{"type":"object","properties":{},"required":[]}}}]}' 'https://ark.cn-beijing.volces.com/api/v3/bots/chat/completions' \ No newline at end of file From 55c03f855e2f40b9bbccf75ce1eaa50b1883cc41 Mon Sep 17 00:00:00 2001 From: wolfplus Date: Sun, 6 Apr 2025 17:48:28 +0100 Subject: [PATCH 04/14] before add volcalno speech --- android/app/build.gradle.kts | 9 +- android/build.gradle.kts | 4 + android/settings.gradle.kts | 3 +- local_plugins/volcano_speech/README.md | 206 ++++++ .../volcano_speech/android/build.gradle.kts | 64 ++ .../android/settings.gradle.kts | 1 + .../android/src/main/AndroidManifest.xml | 9 + .../volcano_speech/VolcanoAsrHelper.kt | 529 +++++++++++++++ .../volcano_speech/VolcanoSpeechPlugin.kt | 445 +++++++++++++ .../volcano_speech/VolcanoTtsHelper.kt | 399 ++++++++++++ .../volcano_speech/utils/FileLogger.kt | 55 ++ .../volcano_speech/lib/volcano_speech.dart | 603 ++++++++++++++++++ local_plugins/volcano_speech/pubspec.yaml | 28 + pubspec.yaml | 2 + 14 files changed, 2349 insertions(+), 8 deletions(-) create mode 100644 local_plugins/volcano_speech/README.md create mode 100644 local_plugins/volcano_speech/android/build.gradle.kts create mode 100644 local_plugins/volcano_speech/android/settings.gradle.kts create mode 100644 local_plugins/volcano_speech/android/src/main/AndroidManifest.xml create mode 100644 local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoAsrHelper.kt create mode 100644 local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoSpeechPlugin.kt create mode 100644 local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoTtsHelper.kt create mode 100644 local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/utils/FileLogger.kt create mode 100644 local_plugins/volcano_speech/lib/volcano_speech.dart create mode 100644 local_plugins/volcano_speech/pubspec.yaml diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index b0a39cbd4..026060648 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -93,14 +93,13 @@ dependencies { // 添加本地插件模块依赖 implementation(project(":azure_speech")) implementation(project(":open_ai_service")) + implementation(project(":volcano_speech")) // 添加OkHttp依赖 implementation("com.squareup.okhttp3:okhttp:4.9.3") // 添加核心库反糖化 coreLibraryDesugaring("com.android.tools:desugar_jdk_libs:2.0.3") - // Microsoft 语音识别SDK - implementation("com.microsoft.cognitiveservices.speech:client-sdk:1.42.0") // 添加androidx.media依赖 implementation("androidx.media:media:1.6.0") @@ -111,11 +110,7 @@ dependencies { // 添加 AndroidX Security 加密 SharedPreferences 依赖 implementation("androidx.security:security-crypto:1.1.0-alpha06") - // // 添加火山语音合成SDK依赖 - // implementation("com.bytedance.speechengine:speechengine_tts_tob:5.4.8") - - // 添加火山语音识别SDK依赖 - // implementation("com.bytedance.speechengine:speechengine_tob:0.0.5") + } flutter { diff --git a/android/build.gradle.kts b/android/build.gradle.kts index f8f47c600..67f214248 100644 --- a/android/build.gradle.kts +++ b/android/build.gradle.kts @@ -3,6 +3,10 @@ allprojects { google() mavenCentral() maven { url = uri("https://storage.googleapis.com/download.flutter.io") } + // 添加火山引擎Maven仓库 + maven { + url = uri("https://artifact.bytedance.com/repository/Volcengine/") + } } } diff --git a/android/settings.gradle.kts b/android/settings.gradle.kts index 6ca3ffd16..c0eebbf43 100644 --- a/android/settings.gradle.kts +++ b/android/settings.gradle.kts @@ -31,8 +31,9 @@ plugins { include(":app") include(":azure_speech") include(":open_ai_service") - +include(":volcano_speech") // 设置azure_speech项目的路径 project(":azure_speech").projectDir = file("../local_plugins/azure_speech/android") project(":open_ai_service").projectDir = file("../local_plugins/open_ai_service/android") +project(":volcano_speech").projectDir = file("../local_plugins/volcano_speech/android") diff --git a/local_plugins/volcano_speech/README.md b/local_plugins/volcano_speech/README.md new file mode 100644 index 000000000..572317b57 --- /dev/null +++ b/local_plugins/volcano_speech/README.md @@ -0,0 +1,206 @@ +# 火山引擎语音服务插件 + +基于火山引擎语音SDK封装的Flutter插件，支持Android平台。 + +## 功能 +- 语音合成（TTS） + - 支持在线/离线/混合合成模式 + - 支持SSML格式文本 + - 支持情感合成和情感预测 + - 支持音量、语速、音高等多种参数调节 + - 复刻音色支持 + - 离线资源管理 +- 语音识别（ASR） + - 一次性识别 + - 连续识别 + - 长按识别 + - 热词优化 + - 多语言支持 + +## 使用说明 + +### TTS 语音合成 + +#### 基础用法 +```dart +import 'package:volcano_speech/volcano_speech.dart'; + +// 初始化 +await VolcanoSpeech.initialize(appId: 'YOUR_APP_ID', apiKey: 'YOUR_API_KEY'); + +// 设置音量、语速等参数 +await VolcanoSpeech.setSpeechParams( + rate: 0, // 语速: -500到500，0为正常速度 + volume: 100, // 音量: 0到100，默认100 + pitch: 0, // 音高: -500到500，0为正常音高 + silenceDuration: 500, // 静音段时长: 毫秒 +); + +// 设置音色 +await VolcanoSpeech.setVoice( + voiceName: 'zh_female_qingxin', + voiceType: 'qingxin' +); + +// 开始合成并播放 +await VolcanoSpeech.speakText(text: '这是一段测试文本'); + +// 暂停播放 +await VolcanoSpeech.pausePlayback(); + +// 继续播放 +await VolcanoSpeech.resumePlayback(); + +// 停止播放 +await VolcanoSpeech.stopSpeaking(); + +// 销毁引擎 +await VolcanoSpeech.dispose(); +``` + +#### 进阶用法 +```dart +// 设置工作模式 +await VolcanoSpeech.setWorkMode(mode: TtsWorkMode.alternate); // 先在线，断网时切换离线 + +// 设置离线发音人 +await VolcanoSpeech.setOfflineVoice( + voiceName: 'xifei', + voiceType: 'qingxin' +); + +// 下载离线资源 +await VolcanoSpeech.downloadOfflineResource( + voiceTypes: ['qingxin', 'zhenjiang'], + languages: ['zh-CN'] +); + +// 使用SSML格式文本 +await VolcanoSpeech.setTextType(type: TtsTextType.ssml); +await VolcanoSpeech.speakText( + text: '这是一段2023-10-01的语音合成' +); + +// 设置情感 +await VolcanoSpeech.setEmotion(emotion: 'happy'); + +// 启用情感预测 +await VolcanoSpeech.setEnableEmotionPredict(enable: true); + +// 启用服务端缓存 +await VolcanoSpeech.setEnableCache(enable: true); + +// 监听TTS进度事件 +VolcanoSpeech.ttsProgressEvents.listen((event) { + print('播放进度: ${(event.progress * 100).toStringAsFixed(1)}%'); +}); + +// 复刻音色支持 +await VolcanoSpeech.setEnableVoiceClone( + enable: true, + backendCluster: 'your_cluster_name' +); +``` + +### ASR 语音识别 +#### 一次性识别 +```dart +import 'package:volcano_speech/volcano_speech.dart'; + +// 初始化 +await VolcanoSpeech.initializeAsr( + appId: 'YOUR_APP_ID', + apiKey: 'YOUR_API_KEY', + supportedLanguages: ['zh-CN'], +); + +// 设置识别语言 +await VolcanoSpeech.setAsrLanguage(language: 'zh-CN'); + +// 设置热词（可选） +await VolcanoSpeech.setAsrHotWords( + hotWords: '{"hotwords":[{"word":"火山引擎","scale":2.0}]}', +); + +// 开始一次性识别 +try { + final result = await VolcanoSpeech.recognizeOnce(); + print('识别结果: ${result['text']}, 语言: ${result['language']}'); +} catch (e) { + print('识别出错: $e'); +} + +// 销毁引擎 +await VolcanoSpeech.disposeAsr(); +``` + +#### 连续识别 +```dart +import 'package:volcano_speech/volcano_speech.dart'; + +// 初始化 +await VolcanoSpeech.initializeAsr( + appId: 'YOUR_APP_ID', + apiKey: 'YOUR_API_KEY', +); + +// 监听识别事件 +VolcanoSpeech.asrEvents.listen((event) { + switch (event.type) { + case AsrEventType.sessionStarted: + print('识别会话开始'); + break; + case AsrEventType.sessionStopped: + print('识别会话结束'); + break; + case AsrEventType.recognizing: + print('正在识别: ${event.text}'); + break; + case AsrEventType.result: + print('识别结果: ${event.text}'); + break; + case AsrEventType.volumeChanged: + print('音量: ${event.volume}'); + break; + case AsrEventType.error: + print('识别错误: ${event.errorMessage}'); + break; + } +}); + +// 开始连续识别 +await VolcanoSpeech.startContinuousRecognition(); + +// 检查是否正在识别 +final isActive = await VolcanoSpeech.isContinuousRecognitionActive(); +print('是否正在识别: $isActive'); + +// 停止连续识别 +await VolcanoSpeech.stopContinuousRecognition(); +``` + +#### 长按识别 +```dart +import 'package:volcano_speech/volcano_speech.dart'; + +// 初始化 +await VolcanoSpeech.initializeAsr( + appId: 'YOUR_APP_ID', + apiKey: 'YOUR_API_KEY', +); + +// 监听识别事件 +VolcanoSpeech.asrEvents.listen((event) { + // 处理事件... +}); + +// 用户按下按钮时开始识别 +onPressed: () async { + await VolcanoSpeech.startListening(); +}, + +// 用户释放按钮时停止识别 +onReleased: () async { + await VolcanoSpeech.stopListening(); +}, +``` \ No newline at end of file diff --git a/local_plugins/volcano_speech/android/build.gradle.kts b/local_plugins/volcano_speech/android/build.gradle.kts new file mode 100644 index 000000000..86b03fd0e --- /dev/null +++ b/local_plugins/volcano_speech/android/build.gradle.kts @@ -0,0 +1,64 @@ +import com.android.build.gradle.LibraryExtension + +buildscript { + repositories { + google() + mavenCentral() + } + dependencies { + classpath("com.android.tools.build:gradle:7.3.0") + classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:1.7.10") + } +} + +allprojects { + repositories { + google() + mavenCentral() + + } +} + +plugins { + id("com.android.library") + kotlin("android") +} + +// 配置android扩展 +configure { + namespace = "com.yunqiinnovation.volcano_speech" + compileSdkVersion(33) + + defaultConfig { + minSdk = 21 + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 + } + + sourceSets { + getByName("main") { + manifest.srcFile("src/main/AndroidManifest.xml") + java.srcDirs("src/main/kotlin") + } + } + + // 添加lint选项 + lintOptions { + isCheckReleaseBuilds = false + } +} + +// 显式设置Kotlin JVM目标版本 +tasks.withType { + kotlinOptions { + jvmTarget = "11" + } +} + +dependencies { + // 添加火山引擎语音合成SDK + implementation("com.bytedance.speechengine:speechengine_tob:0.0.5") +} \ No newline at end of file diff --git a/local_plugins/volcano_speech/android/settings.gradle.kts b/local_plugins/volcano_speech/android/settings.gradle.kts new file mode 100644 index 000000000..7f5342866 --- /dev/null +++ b/local_plugins/volcano_speech/android/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "volcano_speech" \ No newline at end of file diff --git a/local_plugins/volcano_speech/android/src/main/AndroidManifest.xml b/local_plugins/volcano_speech/android/src/main/AndroidManifest.xml new file mode 100644 index 000000000..338e8293c --- /dev/null +++ b/local_plugins/volcano_speech/android/src/main/AndroidManifest.xml @@ -0,0 +1,9 @@ + + + + + + + + \ No newline at end of file diff --git a/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoAsrHelper.kt b/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoAsrHelper.kt new file mode 100644 index 000000000..614474d1a --- /dev/null +++ b/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoAsrHelper.kt @@ -0,0 +1,529 @@ +package com.yunqiinnovation.volcano_speech + +import android.content.Context +import android.os.Handler +import android.os.Looper +import com.bytedance.speech.speechengine.SpeechEngine +import com.bytedance.speech.speechengine.SpeechEngineDefines +import com.bytedance.speech.speechengine.SpeechEngineGenerator +import com.yunqiinnovation.volcano_speech.utils.FileLogger +import org.json.JSONArray +import org.json.JSONException +import org.json.JSONObject + +/** + * 火山语音识别帮助类 (大模型版本) + */ +class VolcanoAsrHelper(private val context: Context) { + private val TAG = "VolcanoAsrHelper" + private val mainHandler = Handler(Looper.getMainLooper()) + + // 语音引擎相关 + private var engine: SpeechEngine? = null + private var engineHandler: Long = -1 + private var isInitialized = false + + // 当前回调 + private var currentAsrCallback: ASRCallback? = null + private var currentContinuousCallback: ASRContinuousCallback? = null + + // 识别状态 + private var isContinuousRecognitionActive = false + + // 配置参数 + private var language = "zh-CN" + private var enableVolume = false + private var showUtterances = false + + /** + * ASR一次性识别回调 + */ + interface ASRCallback { + fun onSuccess(text: String) + fun onError(error: String) + } + + /** + * ASR连续识别回调 + */ + interface ASRContinuousCallback { + fun onResult(text: String) + fun onRecognizing(text: String) + fun onSessionStarted() + fun onSessionStopped() + fun onVolumeChanged(volume: Int) + fun onError(error: String) + } + + /** + * 初始化语音识别引擎 + */ + fun initialize(appId: String, token: String, resourceId: String): Boolean { + if (isInitialized) { + FileLogger.i(TAG, "引擎已经初始化") + return true + } + + try { + // 准备环境 + SpeechEngineGenerator.PrepareEnvironment(context, null) + + // 创建引擎 + engine = SpeechEngineGenerator.getInstance() + engineHandler = engine?.createEngine() ?: -1 + + if (engineHandler == -1L) { + FileLogger.e(TAG, "创建引擎失败") + return false + } + + // 设置上下文 + engine?.setContext(context) + + // 设置引擎类型为ASR + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_ENGINE_NAME_STRING, SpeechEngineDefines.ASR_ENGINE) + + // 设置日志级别 + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_LOG_LEVEL_STRING, SpeechEngineDefines.LOG_LEVEL_WARN) + + // 设置用户ID和设备ID (使用静态值，实际项目中应替换为真实值) + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_UID_STRING, "user_id") + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_DEVICE_ID_STRING, "device_id") + + // 设置鉴权信息 - 大模型版本不需要Bearer前缀 + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_APP_ID_STRING, appId) + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_APP_TOKEN_STRING, token) + + // 设置资源ID - 大模型必需 + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_RESOURCE_ID_STRING, resourceId) + + // 设置协议类型为Seed - 大模型必需 + engine?.setOptionInt(engineHandler, SpeechEngineDefines.PARAMS_KEY_PROTOCOL_TYPE_INT, SpeechEngineDefines.PROTOCOL_TYPE_SEED) + + // 设置网络配置 + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_ASR_ADDRESS_STRING, "wss://openspeech.bytedance.com") + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_ASR_URI_STRING, "/api/v3/sauc/bigmodel") + + // 设置超时时间 + engine?.setOptionInt(engineHandler, SpeechEngineDefines.PARAMS_KEY_ASR_CONN_TIMEOUT_INT, 12000) + engine?.setOptionInt(engineHandler, SpeechEngineDefines.PARAMS_KEY_ASR_RECV_TIMEOUT_INT, 8000) + + // 设置音频来源为录音机 + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_RECORDER_TYPE_STRING, SpeechEngineDefines.RECORDER_TYPE_RECORDER) + + // 设置最大录音时长 (默认60秒) + engine?.setOptionInt(engineHandler, SpeechEngineDefines.PARAMS_KEY_VAD_MAX_SPEECH_DURATION_INT, 60000) + + // 设置回声消除 (用于ASR识别时不会收到TTS的声音) + engine?.setOptionInt(engineHandler, SpeechEngineDefines.PARAMS_KEY_RECORDER_PRESET_INT, SpeechEngineDefines.RECORDER_PRESET_VOICE_COMMUNICATION) + + // 初始化引擎 + val result = engine?.initEngine(engineHandler) + isInitialized = result == SpeechEngineDefines.ERR_NO_ERROR + + if (isInitialized) { + FileLogger.i(TAG, "引擎初始化成功") + + // 设置回调监听 + engine?.setListener(object : SpeechEngine.SpeechListener { + override fun onSpeechMessage(type: Int, data: ByteArray, len: Int) { + val stdData = String(data) + handleEngineEvent(type, stdData) + } + }) + + return true + } else { + FileLogger.e(TAG, "引擎初始化失败: $result") + return false + } + } catch (e: Exception) { + FileLogger.e(TAG, "初始化异常: ${e.message}", e) + return false + } + } + + /** + * 设置语言 + */ + fun setLanguage(language: String): Boolean { + if (!isInitialized) { + FileLogger.e(TAG, "引擎未初始化") + return false + } + + try { + this.language = language + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_ASR_LANGUAGE_STRING, language) + return true + } catch (e: Exception) { + FileLogger.e(TAG, "设置语言失败: ${e.message}", e) + return false + } + } + + /** + * 设置热词 + */ + fun setHotWords(hotWordsId: String): Boolean { + if (!isInitialized) { + FileLogger.e(TAG, "引擎未初始化") + return false + } + + try { + if (hotWordsId.isNotEmpty()) { + // 大模型ASR需要通过请求参数设置热词 + val reqParams = "{\"corpus\":{\"boosting_table_id\":\"$hotWordsId\"}}" + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_ASR_REQ_PARAMS_STRING, reqParams) + } + return true + } catch (e: Exception) { + FileLogger.e(TAG, "设置热词失败: ${e.message}", e) + return false + } + } + + /** + * 设置是否返回音量 + */ + fun setEnableVolume(enable: Boolean): Boolean { + if (!isInitialized) { + FileLogger.e(TAG, "引擎未初始化") + return false + } + + try { + this.enableVolume = enable + engine?.setOptionBoolean(engineHandler, SpeechEngineDefines.PARAMS_KEY_ENABLE_GET_VOLUME_BOOL, enable) + return true + } catch (e: Exception) { + FileLogger.e(TAG, "设置音量返回失败: ${e.message}", e) + return false + } + } + + /** + * 设置是否显示语音停顿、分句、分词信息 + */ + fun setShowUtterances(show: Boolean): Boolean { + if (!isInitialized) { + FileLogger.e(TAG, "引擎未初始化") + return false + } + + try { + this.showUtterances = show + engine?.setOptionBoolean(engineHandler, SpeechEngineDefines.PARAMS_KEY_ASR_SHOW_UTTER_BOOL, show) + return true + } catch (e: Exception) { + FileLogger.e(TAG, "设置语音信息显示失败: ${e.message}", e) + return false + } + } + + /** + * 设置VAD切句参数 + */ + fun setVadParams(forceToSpeechTime: Int, endWindowSize: Int): Boolean { + if (!isInitialized) { + FileLogger.e(TAG, "引擎未初始化") + return false + } + + try { + val reqParams = "{\"force_to_speech_time\":$forceToSpeechTime, \"end_window_size\":$endWindowSize}" + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_ASR_REQ_PARAMS_STRING, reqParams) + return true + } catch (e: Exception) { + FileLogger.e(TAG, "设置VAD参数失败: ${e.message}", e) + return false + } + } + + /** + * 设置纠错词表 + */ + fun setCorrectWords(correctWordsJson: String): Boolean { + if (!isInitialized) { + FileLogger.e(TAG, "引擎未初始化") + return false + } + + try { + val reqParams = "{\"context\": \"{\\\"correct_words\\\": $correctWordsJson}\"}" + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_ASR_REQ_PARAMS_STRING, reqParams) + return true + } catch (e: Exception) { + FileLogger.e(TAG, "设置纠错词表失败: ${e.message}", e) + return false + } + } + + /** + * 一次性识别 + */ + fun recognizeOnce(callback: ASRCallback): Boolean { + if (!isInitialized) { + FileLogger.e(TAG, "引擎未初始化") + return false + } + + if (isContinuousRecognitionActive) { + FileLogger.e(TAG, "当前正在连续识别中") + return false + } + + try { + this.currentAsrCallback = callback + + // 停止当前引擎 + engine?.sendDirective(engineHandler, SpeechEngineDefines.DIRECTIVE_SYNC_STOP_ENGINE, "") + + // 启动引擎开始识别 + val ret = engine?.sendDirective(engineHandler, SpeechEngineDefines.DIRECTIVE_START_ENGINE, "") + + if (ret != SpeechEngineDefines.ERR_NO_ERROR) { + FileLogger.e(TAG, "启动识别失败: $ret") + return false + } + + return true + } catch (e: Exception) { + FileLogger.e(TAG, "识别异常: ${e.message}", e) + return false + } + } + + /** + * 停止一次性识别 + */ + fun stopRecognize(): Boolean { + if (!isInitialized) { + FileLogger.e(TAG, "引擎未初始化") + return false + } + + try { + // 告知引擎音频输入完成 + val ret = engine?.sendDirective(engineHandler, SpeechEngineDefines.DIRECTIVE_FINISH_TALKING, "") + + if (ret != SpeechEngineDefines.ERR_NO_ERROR) { + FileLogger.e(TAG, "停止识别失败: $ret") + return false + } + + return true + } catch (e: Exception) { + FileLogger.e(TAG, "停止识别异常: ${e.message}", e) + return false + } + } + + /** + * 开始连续识别 + */ + fun startContinuousRecognition(callback: ASRContinuousCallback): Boolean { + if (!isInitialized) { + FileLogger.e(TAG, "引擎未初始化") + return false + } + + if (isContinuousRecognitionActive) { + FileLogger.e(TAG, "当前已经在连续识别中") + return false + } + + try { + this.currentContinuousCallback = callback + + // 停止当前引擎 + engine?.sendDirective(engineHandler, SpeechEngineDefines.DIRECTIVE_SYNC_STOP_ENGINE, "") + + // 启动引擎开始识别 + val ret = engine?.sendDirective(engineHandler, SpeechEngineDefines.DIRECTIVE_START_ENGINE, "") + + if (ret != SpeechEngineDefines.ERR_NO_ERROR) { + FileLogger.e(TAG, "启动连续识别失败: $ret") + return false + } + + isContinuousRecognitionActive = true + return true + } catch (e: Exception) { + FileLogger.e(TAG, "连续识别异常: ${e.message}", e) + return false + } + } + + /** + * 停止连续识别 + */ + fun stopContinuousRecognition(): Boolean { + if (!isInitialized || !isContinuousRecognitionActive) { + FileLogger.e(TAG, "引擎未初始化或未在连续识别中") + return false + } + + try { + // 停止引擎 + val ret = engine?.sendDirective(engineHandler, SpeechEngineDefines.DIRECTIVE_STOP_ENGINE, "") + + if (ret != SpeechEngineDefines.ERR_NO_ERROR) { + FileLogger.e(TAG, "停止连续识别失败: $ret") + return false + } + + isContinuousRecognitionActive = false + return true + } catch (e: Exception) { + FileLogger.e(TAG, "停止连续识别异常: ${e.message}", e) + return false + } + } + + /** + * 判断是否在连续识别中 + */ + fun isContinuousRecognitionActive(): Boolean { + return isContinuousRecognitionActive + } + + /** + * 释放资源 + */ + fun release() { + if (!isInitialized) { + return + } + + try { + // 停止引擎 + if (isContinuousRecognitionActive) { + stopContinuousRecognition() + } else { + engine?.sendDirective(engineHandler, SpeechEngineDefines.DIRECTIVE_STOP_ENGINE, "") + } + + // 销毁引擎 + engine?.destroyEngine(engineHandler) + engineHandler = -1 + engine = null + isInitialized = false + + FileLogger.i(TAG, "引擎已释放") + } catch (e: Exception) { + FileLogger.e(TAG, "释放引擎异常: ${e.message}", e) + } + } + + /** + * 处理引擎事件 + */ + private fun handleEngineEvent(type: Int, data: String) { + when (type) { + SpeechEngineDefines.MESSAGE_TYPE_ENGINE_START -> { + // 引擎启动成功 + mainHandler.post { + currentContinuousCallback?.onSessionStarted() + } + } + + SpeechEngineDefines.MESSAGE_TYPE_ENGINE_STOP -> { + // 引擎停止 + isContinuousRecognitionActive = false + mainHandler.post { + currentContinuousCallback?.onSessionStopped() + } + } + + SpeechEngineDefines.MESSAGE_TYPE_PARTIAL_RESULT -> { + // 中间识别结果 + try { + val json = JSONObject(data) + if (!json.has("result")) { + return + } + + val resultArray = json.getJSONArray("result") + if (resultArray.length() > 0) { + val result = resultArray.getJSONObject(0) + val text = result.optString("text", "") + + if (text.isNotEmpty()) { + mainHandler.post { + currentContinuousCallback?.onRecognizing(text) + } + } + } + } catch (e: JSONException) { + FileLogger.e(TAG, "解析中间识别结果异常: ${e.message}", e) + } + } + + SpeechEngineDefines.MESSAGE_TYPE_FINAL_RESULT -> { + // 最终识别结果 + try { + val json = JSONObject(data) + if (!json.has("result")) { + return + } + + val resultArray = json.getJSONArray("result") + if (resultArray.length() > 0) { + val result = resultArray.getJSONObject(0) + val text = result.optString("text", "") + + if (text.isNotEmpty()) { + mainHandler.post { + if (isContinuousRecognitionActive) { + currentContinuousCallback?.onResult(text) + } else { + currentAsrCallback?.onSuccess(text) + } + } + } + } + } catch (e: JSONException) { + FileLogger.e(TAG, "解析最终识别结果异常: ${e.message}", e) + } + } + + SpeechEngineDefines.MESSAGE_TYPE_VOLUME_LEVEL -> { + // 音量回调 + if (enableVolume && isContinuousRecognitionActive) { + try { + val volume = (data.toFloat() * 100).toInt() + mainHandler.post { + currentContinuousCallback?.onVolumeChanged(volume) + } + } catch (e: Exception) { + FileLogger.e(TAG, "解析音量数据异常: ${e.message}", e) + } + } + } + + SpeechEngineDefines.MESSAGE_TYPE_ENGINE_ERROR -> { + // 错误信息 + try { + val json = JSONObject(data) + val errCode = json.optInt("err_code", -1) + val errMsg = json.optString("err_msg", "未知错误") + + FileLogger.e(TAG, "引擎错误: $errCode, $errMsg") + + mainHandler.post { + if (isContinuousRecognitionActive) { + currentContinuousCallback?.onError(errMsg) + isContinuousRecognitionActive = false + } else { + currentAsrCallback?.onError(errMsg) + } + } + } catch (e: JSONException) { + FileLogger.e(TAG, "解析错误信息异常: ${e.message}", e) + } + } + } + } +} diff --git a/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoSpeechPlugin.kt b/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoSpeechPlugin.kt new file mode 100644 index 000000000..bb6f1e452 --- /dev/null +++ b/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoSpeechPlugin.kt @@ -0,0 +1,445 @@ +package com.yunqiinnovation.volcano_speech + +import android.content.Context +import android.os.Handler +import android.os.Looper +import androidx.annotation.NonNull +import com.yunqiinnovation.volcano_speech.utils.FileLogger + +import io.flutter.embedding.engine.plugins.FlutterPlugin +import io.flutter.plugin.common.MethodCall +import io.flutter.plugin.common.MethodChannel +import io.flutter.plugin.common.MethodChannel.MethodCallHandler +import io.flutter.plugin.common.MethodChannel.Result +import io.flutter.plugin.common.EventChannel + +/** VolcanoSpeechPlugin */ +class VolcanoSpeechPlugin: FlutterPlugin { + private val TAG = "VolcanoSpeechPlugin" + private lateinit var context: Context + private val mainHandler = Handler(Looper.getMainLooper()) + + // ASR相关 + private lateinit var asrChannel: MethodChannel + private lateinit var asrEventChannel: EventChannel + private var asrEventSink: EventChannel.EventSink? = null + + // TTS相关 + private lateinit var ttsChannel: MethodChannel + private lateinit var ttsEventChannel: EventChannel + private var ttsEventSink: EventChannel.EventSink? = null + + // TTS帮助类 + private lateinit var volcanoTtsHelper: VolcanoTtsHelper + + // ASR帮助类 + private lateinit var volcanoAsrHelper: VolcanoAsrHelper + + override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { + context = flutterPluginBinding.applicationContext + + // 初始化ASR通道 + asrChannel = MethodChannel(flutterPluginBinding.binaryMessenger, "volcano_speech/asr") + asrChannel.setMethodCallHandler(AsrMethodHandler()) + + // 初始化TTS通道 + ttsChannel = MethodChannel(flutterPluginBinding.binaryMessenger, "volcano_speech/tts") + ttsChannel.setMethodCallHandler(TtsMethodHandler()) + + // 初始化ASR事件通道 + asrEventChannel = EventChannel(flutterPluginBinding.binaryMessenger, "volcano_speech/asr_events") + asrEventChannel.setStreamHandler(object : EventChannel.StreamHandler { + override fun onListen(arguments: Any?, events: EventChannel.EventSink?) { + asrEventSink = events + } + + override fun onCancel(arguments: Any?) { + asrEventSink = null + } + }) + + // 初始化TTS事件通道 + ttsEventChannel = EventChannel(flutterPluginBinding.binaryMessenger, "volcano_speech/tts_events") + ttsEventChannel.setStreamHandler(object : EventChannel.StreamHandler { + override fun onListen(arguments: Any?, events: EventChannel.EventSink?) { + ttsEventSink = events + } + + override fun onCancel(arguments: Any?) { + ttsEventSink = null + } + }) + + // 初始化TTS帮助类 + volcanoTtsHelper = VolcanoTtsHelper(context) + + // 初始化ASR帮助类 + volcanoAsrHelper = VolcanoAsrHelper(context) + } + + // 发送ASR事件 + private fun sendAsrEvent(event: Map) { + FileLogger.d(TAG, "发送ASR事件: $event") + if (asrEventSink == null) { + FileLogger.w(TAG, "无法发送ASR事件：事件通道未准备好") + return + } + + mainHandler.post { + try { + asrEventSink?.success(event) + FileLogger.d(TAG, "ASR事件发送成功") + } catch (e: Exception) { + FileLogger.e(TAG, "发送ASR事件失败: ${e.message}") + } + } + } + + // 发送TTS事件 + private fun sendTtsEvent(event: Map) { + FileLogger.d(TAG, "发送TTS事件: $event") + if (ttsEventSink == null) { + FileLogger.w(TAG, "无法发送TTS事件：事件通道未准备好") + return + } + + mainHandler.post { + try { + ttsEventSink?.success(event) + FileLogger.d(TAG, "TTS事件发送成功") + } catch (e: Exception) { + FileLogger.e(TAG, "发送TTS事件失败: ${e.message}") + } + } + } + + // ASR方法处理器 + inner class AsrMethodHandler : MethodCallHandler { + override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) { + when (call.method) { + "getPlatformVersion" -> { + result.success("Android ${android.os.Build.VERSION.RELEASE}") + } + "initialize" -> { + val appId = call.argument("appId") ?: "" + val apiKey = call.argument("apiKey") ?: "" + val resourceId = call.argument("resourceId") ?: "" + + if (appId.isEmpty() || apiKey.isEmpty() || resourceId.isEmpty()) { + result.error("INVALID_ARGUMENTS", "appId、apiKey和resourceId不能为空", null) + return + } + + val success = volcanoAsrHelper.initialize(appId, apiKey, resourceId) + result.success(success) + } + "setLanguage" -> { + val language = call.argument("language") ?: "zh-CN" + result.success(volcanoAsrHelper.setLanguage(language)) + } + "setHotWords" -> { + val hotWordsId = call.argument("hotWordsId") ?: "" + result.success(volcanoAsrHelper.setHotWords(hotWordsId)) + } + "setVadParams" -> { + val forceToSpeechTime = call.argument("forceToSpeechTime") ?: 0 + val endWindowSize = call.argument("endWindowSize") ?: 800 + result.success(volcanoAsrHelper.setVadParams(forceToSpeechTime, endWindowSize)) + } + "setCorrectWords" -> { + val correctWordsJson = call.argument("correctWordsJson") ?: "{}" + result.success(volcanoAsrHelper.setCorrectWords(correctWordsJson)) + } + "setEnableVolume" -> { + val enable = call.argument("enable") ?: false + result.success(volcanoAsrHelper.setEnableVolume(enable)) + } + "setShowUtterances" -> { + val enable = call.argument("enable") ?: false + result.success(volcanoAsrHelper.setShowUtterances(enable)) + } + "recognizeOnce" -> { + // 确保当前不在连续识别中 + if (volcanoAsrHelper.isContinuousRecognitionActive()) { + result.error("ASR_BUSY", "当前正在连续识别中", null) + return + } + + volcanoAsrHelper.recognizeOnce(object : VolcanoAsrHelper.ASRCallback { + override fun onSuccess(text: String) { + mainHandler.post { + result.success(mapOf( + "text" to text + )) + } + } + + override fun onError(error: String) { + mainHandler.post { + result.error("ASR_ERROR", error, null) + } + } + }) + } + "startContinuousRecognition" -> { + // 确保事件通道已准备好 + if (asrEventSink == null) { + result.error("EVENT_CHANNEL_NOT_READY", "事件通道未准备好，无法开始连续识别", null) + return + } + + val success = volcanoAsrHelper.startContinuousRecognition(object : VolcanoAsrHelper.ASRContinuousCallback { + override fun onResult(text: String) { + sendAsrEvent(mapOf( + "type" to "result", + "text" to text + )) + } + + override fun onRecognizing(text: String) { + sendAsrEvent(mapOf( + "type" to "recognizing", + "text" to text + )) + } + + override fun onSessionStarted() { + sendAsrEvent(mapOf("type" to "sessionStarted")) + } + + override fun onSessionStopped() { + sendAsrEvent(mapOf("type" to "sessionStopped")) + } + + override fun onVolumeChanged(volume: Int) { + sendAsrEvent(mapOf( + "type" to "volumeChanged", + "volume" to volume + )) + } + + override fun onError(error: String) { + sendAsrEvent(mapOf( + "type" to "error", + "message" to error + )) + } + }) + + result.success(success) + } + "stopContinuousRecognition" -> { + val success = volcanoAsrHelper.stopContinuousRecognition() + result.success(success) + } + "stopRecognize" -> { + val success = volcanoAsrHelper.stopRecognize() + result.success(success) + } + "isContinuousRecognitionActive" -> { + result.success(volcanoAsrHelper.isContinuousRecognitionActive()) + } + "release" -> { + volcanoAsrHelper.release() + result.success(true) + } + else -> { + result.notImplemented() + } + } + } + } + + // TTS方法处理器 + inner class TtsMethodHandler : MethodCallHandler { + override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) { + when (call.method) { + "getPlatformVersion" -> { + result.success("Android ${android.os.Build.VERSION.RELEASE}") + } + "initialize" -> { + val appId = call.argument("appId") ?: "" + val token = call.argument("token") ?: "" + val resourceId = call.argument("resourceId") ?: "" + + if (appId.isEmpty() || token.isEmpty() || resourceId.isEmpty()) { + result.error("INVALID_ARGUMENTS", "appId、token和resourceId不能为空", null) + return + } + + val success = volcanoTtsHelper.initialize(appId, token, resourceId) + result.success(success) + } + "setVoice" -> { + val voice = call.argument("voice") ?: "" + + if (voice.isEmpty()) { + result.error("INVALID_ARGUMENTS", "voice不能为空", null) + return + } + + result.success(volcanoTtsHelper.setVoice(voice)) + } + "setContinuousMode" -> { + val isContinuous = call.argument("isContinuous") ?: false + result.success(volcanoTtsHelper.setContinuousMode(isContinuous)) + } + "speak" -> { + val text = call.argument("text") ?: "" + + if (text.isEmpty()) { + result.error("INVALID_ARGUMENTS", "合成文本不能为空", null) + return + } + + volcanoTtsHelper.speak(text, object : VolcanoTtsHelper.TTSCallback { + override fun onStart(reqId: String) { + sendTtsEvent(mapOf( + "type" to "start", + "reqId" to reqId + )) + } + + override fun onProgress(reqId: String, progress: Double) { + sendTtsEvent(mapOf( + "type" to "progress", + "reqId" to reqId, + "progress" to progress + )) + } + + override fun onComplete(reqId: String) { + sendTtsEvent(mapOf( + "type" to "complete", + "reqId" to reqId + )) + + mainHandler.post { + result.success(true) + } + } + + override fun onError(reqId: String, errorCode: Int, errorMsg: String) { + sendTtsEvent(mapOf( + "type" to "error", + "reqId" to reqId, + "errorCode" to errorCode, + "errorMsg" to errorMsg + )) + + mainHandler.post { + result.error("TTS_ERROR", errorMsg, null) + } + } + }) + } + "synthesisNext" -> { + val text = call.argument("text") ?: "" + + if (text.isEmpty()) { + result.error("INVALID_ARGUMENTS", "合成文本不能为空", null) + return + } + + val success = volcanoTtsHelper.synthesisNext(text) + result.success(success) + } + "pause" -> { + result.success(volcanoTtsHelper.pause()) + } + "resume" -> { + result.success(volcanoTtsHelper.resume()) + } + "stop" -> { + result.success(volcanoTtsHelper.stop()) + } + "release" -> { + volcanoTtsHelper.release() + result.success(true) + } + // 以下方法用于与Azure Speech版本兼容 + "setSpeechSynthesisVoice" -> { + val voiceName = call.argument("voiceName") ?: "" + + if (voiceName.isEmpty()) { + result.error("INVALID_ARGUMENTS", "语音名称不能为空", null) + return + } + + result.success(volcanoTtsHelper.setVoice(voiceName)) + } + "speakText" -> { + val text = call.argument("text") ?: "" + + if (text.isEmpty()) { + result.error("INVALID_ARGUMENTS", "合成文本不能为空", null) + return + } + + volcanoTtsHelper.speak(text, object : VolcanoTtsHelper.TTSCallback { + override fun onStart(reqId: String) { + sendTtsEvent(mapOf( + "type" to "start", + "reqId" to reqId + )) + } + + override fun onProgress(reqId: String, progress: Double) { + sendTtsEvent(mapOf( + "type" to "progress", + "reqId" to reqId, + "progress" to progress + )) + } + + override fun onComplete(reqId: String) { + sendTtsEvent(mapOf( + "type" to "complete", + "reqId" to reqId + )) + + mainHandler.post { + result.success(true) + } + } + + override fun onError(reqId: String, errorCode: Int, errorMsg: String) { + sendTtsEvent(mapOf( + "type" to "error", + "reqId" to reqId, + "errorCode" to errorCode, + "errorMsg" to errorMsg + )) + + mainHandler.post { + result.error("TTS_ERROR", errorMsg, null) + } + } + }) + } + "stopSpeaking" -> { + result.success(volcanoTtsHelper.stop()) + } + "pauseSpeaking" -> { + result.success(volcanoTtsHelper.pause()) + } + "resumeSpeaking" -> { + result.success(volcanoTtsHelper.resume()) + } + else -> { + result.notImplemented() + } + } + } + } + + override fun onDetachedFromEngine(@NonNull binding: FlutterPlugin.FlutterPluginBinding) { + asrChannel.setMethodCallHandler(null) + ttsChannel.setMethodCallHandler(null) + asrEventChannel.setStreamHandler(null) + ttsEventChannel.setStreamHandler(null) + + volcanoTtsHelper.release() + volcanoAsrHelper.release() + } +} \ No newline at end of file diff --git a/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoTtsHelper.kt b/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoTtsHelper.kt new file mode 100644 index 000000000..e24a1ceb1 --- /dev/null +++ b/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoTtsHelper.kt @@ -0,0 +1,399 @@ +package com.yunqiinnovation.volcano_speech + +import android.content.Context +import android.os.Handler +import android.os.Looper +import com.bytedance.speech.speechengine.SpeechEngine +import com.bytedance.speech.speechengine.SpeechEngineDefines +import com.bytedance.speech.speechengine.SpeechEngineGenerator +import com.yunqiinnovation.volcano_speech.utils.FileLogger +import org.json.JSONObject + +/** + * 火山语音合成帮助类 (大模型版本) + */ +class VolcanoTtsHelper(private val context: Context) { + private val TAG = "VolcanoTtsHelper" + private val mainHandler = Handler(Looper.getMainLooper()) + + // 语音引擎相关 + private var engine: SpeechEngine? = null + private var engineHandler: Long = -1 + private var isInitialized = false + + // 当前回调 + private var currentTtsCallback: TTSCallback? = null + + // 合成状态 + private var isPlaying = false + private var currentReqId = "" + + // 配置参数 + private var voice = "zh_female_yuxi" + private var ttsText = "" + private var isContinuous = false + + /** + * TTS合成回调接口 + */ + interface TTSCallback { + fun onStart(reqId: String) + fun onProgress(reqId: String, progress: Double) + fun onComplete(reqId: String) + fun onError(reqId: String, errorCode: Int, errorMsg: String) + } + + /** + * 初始化语音合成引擎 + */ + fun initialize(appId: String, token: String, resourceId: String): Boolean { + if (isInitialized) { + FileLogger.i(TAG, "引擎已经初始化") + return true + } + + try { + // 准备环境 + SpeechEngineGenerator.PrepareEnvironment(context, null) + + // 创建引擎 + engine = SpeechEngineGenerator.getInstance() + engineHandler = engine?.createEngine() ?: -1 + + if (engineHandler == -1L) { + FileLogger.e(TAG, "创建引擎失败") + return false + } + + // 设置上下文 + engine?.setContext(context) + + // 设置引擎类型为TTS + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_ENGINE_NAME_STRING, SpeechEngineDefines.TTS_ENGINE) + + // 设置日志级别 + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_LOG_LEVEL_STRING, SpeechEngineDefines.LOG_LEVEL_WARN) + + // 设置用户ID和设备ID (使用静态值，实际项目中应替换为真实值) + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_UID_STRING, "user_id") + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_DEVICE_ID_STRING, "device_id") + + // 设置授权信息 - 大模型版本不需要Bearer前缀 + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_APP_ID_STRING, appId) + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_APP_TOKEN_STRING, token) + + // 设置资源ID - 大模型必需 + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_RESOURCE_ID_STRING, resourceId) + + // 设置协议类型为Seed - 大模型必需 + engine?.setOptionInt(engineHandler, SpeechEngineDefines.PARAMS_KEY_PROTOCOL_TYPE_INT, SpeechEngineDefines.PROTOCOL_TYPE_SEED) + + // 设置合成策略为在线合成 + engine?.setOptionInt(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_WORK_MODE_INT, SpeechEngineDefines.TTS_WORK_MODE_ONLINE) + + // 设置在线请求资源配置 + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_ADDRESS_STRING, "wss://openspeech.bytedance.com") + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_URI_STRING, "/api/v3/sauc/bigmodel") + + // 设置发音人 + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_VOICE_ONLINE_STRING, voice) + + // 设置播放进度回调 + engine?.setOptionInt(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_WITH_FRONTEND_INT, 1) + + // 初始化引擎 + val result = engine?.initEngine(engineHandler) + isInitialized = result == SpeechEngineDefines.ERR_NO_ERROR + + if (isInitialized) { + FileLogger.i(TAG, "引擎初始化成功") + + // 设置回调监听 + engine?.setListener(object : SpeechEngine.SpeechListener { + override fun onSpeechMessage(type: Int, data: ByteArray, len: Int) { + val stdData = String(data) + handleEngineEvent(type, stdData) + } + }) + + return true + } else { + FileLogger.e(TAG, "引擎初始化失败: $result") + return false + } + } catch (e: Exception) { + FileLogger.e(TAG, "初始化异常: ${e.message}", e) + return false + } + } + + /** + * 设置发音人 + */ + fun setVoice(voice: String): Boolean { + if (!isInitialized) { + FileLogger.e(TAG, "引擎未初始化") + return false + } + + try { + this.voice = voice + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_VOICE_ONLINE_STRING, voice) + return true + } catch (e: Exception) { + FileLogger.e(TAG, "设置发音人失败: ${e.message}", e) + return false + } + } + + /** + * 设置合成场景（单次或连续） + */ + fun setContinuousMode(isContinuous: Boolean): Boolean { + if (!isInitialized) { + FileLogger.e(TAG, "引擎未初始化") + return false + } + + try { + this.isContinuous = isContinuous + val scenarioType = if (isContinuous) { + SpeechEngineDefines.TTS_SCENARIO_TYPE_NOVEL + } else { + SpeechEngineDefines.TTS_SCENARIO_TYPE_NORMAL + } + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_SCENARIO_STRING, scenarioType) + return true + } catch (e: Exception) { + FileLogger.e(TAG, "设置合成场景失败: ${e.message}", e) + return false + } + } + + /** + * 开始合成并播放 + */ + fun speak(text: String, callback: TTSCallback? = null): Boolean { + if (!isInitialized) { + FileLogger.e(TAG, "引擎未初始化") + return false + } + + if (text.isEmpty()) { + FileLogger.e(TAG, "合成文本为空") + return false + } + + try { + this.ttsText = text + this.currentTtsCallback = callback + + // 设置要合成的文本 + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_TEXT_STRING, text) + + // 停止当前引擎 + engine?.sendDirective(engineHandler, SpeechEngineDefines.DIRECTIVE_SYNC_STOP_ENGINE, "") + + // 启动引擎开始合成 + val ret = engine?.sendDirective(engineHandler, SpeechEngineDefines.DIRECTIVE_START_ENGINE, "") + + if (ret != SpeechEngineDefines.ERR_NO_ERROR) { + FileLogger.e(TAG, "启动合成失败: $ret") + return false + } + + return true + } catch (e: Exception) { + FileLogger.e(TAG, "合成异常: ${e.message}", e) + return false + } + } + + /** + * 仅用于连续合成场景：在引擎启动后添加新的文本进行合成 + */ + fun synthesisNext(text: String): Boolean { + if (!isInitialized || !isContinuous) { + FileLogger.e(TAG, "引擎未初始化或非连续合成模式") + return false + } + + if (text.isEmpty()) { + FileLogger.e(TAG, "合成文本为空") + return false + } + + try { + // 设置要合成的文本 + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_TEXT_STRING, text) + + // 发送合成指令 + val ret = engine?.sendDirective(engineHandler, SpeechEngineDefines.DIRECTIVE_SYNTHESIS, "") + + if (ret != SpeechEngineDefines.ERR_NO_ERROR) { + FileLogger.e(TAG, "添加合成文本失败: $ret") + return false + } + + return true + } catch (e: Exception) { + FileLogger.e(TAG, "添加合成文本异常: ${e.message}", e) + return false + } + } + + /** + * 暂停播放 + */ + fun pause(): Boolean { + if (!isInitialized || !isPlaying) { + FileLogger.e(TAG, "引擎未初始化或未在播放") + return false + } + + try { + val ret = engine?.sendDirective(engineHandler, SpeechEngineDefines.DIRECTIVE_PAUSE_PLAYER, "") + return ret == SpeechEngineDefines.ERR_NO_ERROR + } catch (e: Exception) { + FileLogger.e(TAG, "暂停播放异常: ${e.message}", e) + return false + } + } + + /** + * 恢复播放 + */ + fun resume(): Boolean { + if (!isInitialized) { + FileLogger.e(TAG, "引擎未初始化") + return false + } + + try { + val ret = engine?.sendDirective(engineHandler, SpeechEngineDefines.DIRECTIVE_RESUME_PLAYER, "") + return ret == SpeechEngineDefines.ERR_NO_ERROR + } catch (e: Exception) { + FileLogger.e(TAG, "恢复播放异常: ${e.message}", e) + return false + } + } + + /** + * 停止播放 + */ + fun stop(): Boolean { + if (!isInitialized) { + FileLogger.e(TAG, "引擎未初始化") + return false + } + + try { + val ret = engine?.sendDirective(engineHandler, SpeechEngineDefines.DIRECTIVE_STOP_ENGINE, "") + isPlaying = false + return ret == SpeechEngineDefines.ERR_NO_ERROR + } catch (e: Exception) { + FileLogger.e(TAG, "停止播放异常: ${e.message}", e) + return false + } + } + + /** + * 释放资源 + */ + fun release() { + if (!isInitialized) { + return + } + + try { + // 停止引擎 + stop() + + // 销毁引擎 + engine?.destroyEngine(engineHandler) + engineHandler = -1 + engine = null + isInitialized = false + + FileLogger.i(TAG, "引擎已释放") + } catch (e: Exception) { + FileLogger.e(TAG, "释放引擎异常: ${e.message}", e) + } + } + + /** + * 处理引擎事件 + */ + private fun handleEngineEvent(type: Int, data: String) { + when (type) { + SpeechEngineDefines.MESSAGE_TYPE_ENGINE_START -> { + // 引擎启动成功，获取请求ID + currentReqId = data + isPlaying = true + + mainHandler.post { + currentTtsCallback?.onStart(currentReqId) + } + + if (isContinuous) { + // 在连续合成模式下，需要单独发送合成指令 + synthesisNext(ttsText) + } + } + + SpeechEngineDefines.MESSAGE_TYPE_ENGINE_STOP -> { + // 引擎停止 + isPlaying = false + } + + SpeechEngineDefines.MESSAGE_TYPE_TTS_START_PLAYING -> { + // 开始播放 + FileLogger.d(TAG, "开始播放: $data") + } + + SpeechEngineDefines.MESSAGE_TYPE_TTS_FINISH_PLAYING -> { + // 播放结束 + isPlaying = false + + mainHandler.post { + currentTtsCallback?.onComplete(currentReqId) + } + } + + SpeechEngineDefines.MESSAGE_TYPE_TTS_PLAYBACK_PROGRESS -> { + // 播放进度 + try { + val json = JSONObject(data) + val progress = json.optDouble("progress", 0.0) + val reqId = json.optString("reqid", "") + + mainHandler.post { + currentTtsCallback?.onProgress(reqId, progress) + } + } catch (e: Exception) { + FileLogger.e(TAG, "解析进度信息异常: ${e.message}", e) + } + } + + SpeechEngineDefines.MESSAGE_TYPE_ENGINE_ERROR -> { + // 错误信息 + try { + val json = JSONObject(data) + val reqId = json.optString("reqid", "") + val errCode = json.optInt("err_code", -1) + val errMsg = json.optString("err_msg", "未知错误") + + FileLogger.e(TAG, "引擎错误: $errCode, $errMsg") + + isPlaying = false + + mainHandler.post { + currentTtsCallback?.onError(reqId, errCode, errMsg) + } + } catch (e: Exception) { + FileLogger.e(TAG, "解析错误信息异常: ${e.message}", e) + } + } + } + } +} diff --git a/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/utils/FileLogger.kt b/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/utils/FileLogger.kt new file mode 100644 index 000000000..8fdfb1227 --- /dev/null +++ b/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/utils/FileLogger.kt @@ -0,0 +1,55 @@ +package com.yunqiinnovation.volcano_speech.utils + +import android.util.Log + +/** + * 文件日志记录工具 + */ +object FileLogger { + private const val TAG = "VolcanoSpeech" + private var isDebugEnabled = true + + /** + * 设置是否启用调试日志 + */ + fun setDebugEnabled(enabled: Boolean) { + isDebugEnabled = enabled + } + + /** + * 记录调试日志 + */ + fun d(tag: String, message: String) { + if (isDebugEnabled) { + Log.d("$TAG-$tag", message) + } + } + + /** + * 记录信息日志 + */ + fun i(tag: String, message: String) { + Log.i("$TAG-$tag", message) + } + + /** + * 记录警告日志 + */ + fun w(tag: String, message: String) { + Log.w("$TAG-$tag", message) + } + + /** + * 记录错误日志 + */ + fun e(tag: String, message: String) { + Log.e("$TAG-$tag", message) + } + + /** + * 记录错误日志，带异常 + */ + fun e(tag: String, message: String, throwable: Throwable) { + Log.e("$TAG-$tag", message, throwable) + } +} \ No newline at end of file diff --git a/local_plugins/volcano_speech/lib/volcano_speech.dart b/local_plugins/volcano_speech/lib/volcano_speech.dart new file mode 100644 index 000000000..99da7d4b7 --- /dev/null +++ b/local_plugins/volcano_speech/lib/volcano_speech.dart @@ -0,0 +1,603 @@ +import 'dart:async'; +import 'package:flutter/services.dart'; + +/// ASR 事件类型 +enum AsrEventType { + /// 会话开始 + sessionStarted, + + /// 会话结束 + sessionStopped, + + /// 正在识别（中间结果） + recognizing, + + /// 识别结果（最终结果） + result, + + /// 音量变化 + volumeChanged, + + /// 错误 + error +} + +/// TTS 工作模式 +enum TtsWorkMode { + /// 在线合成 + online, + + /// 离线合成 + offline, + + /// 同时在线离线 + both, + + /// 先在线再离线（网络不好时自动切换） + alternate, + + /// 文件模式 + file +} + +/// TTS 文本类型 +enum TtsTextType { + /// 纯文本 + plain, + + /// SSML格式 + ssml +} + +/// 协议类型 +enum ProtocolType { + /// 默认协议 + defaultProtocol, + + /// Seed协议（用于大模型） + seed +} + +/// TTS 播放进度事件 +class TtsProgressEvent { + /// 播放进度 0.0-1.0 + final double progress; + + /// 请求ID + final String reqId; + + const TtsProgressEvent({ + required this.progress, + required this.reqId, + }); + + @override + String toString() { + return 'TtsProgressEvent{progress: $progress, reqId: $reqId}'; + } +} + +/// ASR 事件 +class AsrEvent { + /// 事件类型 + final AsrEventType type; + + /// 识别文本（仅在recognizing和result类型时有效） + final String? text; + + /// 识别语言（仅在recognizing和result类型时有效） + final String? language; + + /// 音量值（仅在volumeChanged类型时有效） + final int? volume; + + /// 错误信息（仅在error类型时有效） + final String? errorMessage; + + const AsrEvent({ + required this.type, + this.text, + this.language, + this.volume, + this.errorMessage, + }); + + factory AsrEvent.fromMap(Map map) { + final typeStr = map['type'] as String; + + AsrEventType type; + switch (typeStr) { + case 'sessionStarted': + type = AsrEventType.sessionStarted; + break; + case 'sessionStopped': + type = AsrEventType.sessionStopped; + break; + case 'recognizing': + type = AsrEventType.recognizing; + break; + case 'result': + type = AsrEventType.result; + break; + case 'volumeChanged': + type = AsrEventType.volumeChanged; + break; + case 'error': + type = AsrEventType.error; + break; + default: + throw ArgumentError('未知的事件类型: $typeStr'); + } + + return AsrEvent( + type: type, + text: map['text'] as String?, + language: map['language'] as String?, + volume: map['volume'] as int?, + errorMessage: map['message'] as String?, + ); + } + + @override + String toString() { + return 'AsrEvent{type: $type, text: $text, language: $language, volume: $volume, errorMessage: $errorMessage}'; + } +} + +/// 火山引擎语音服务插件 +class VolcanoSpeech { + static final VolcanoSpeechAsr asr = VolcanoSpeechAsr._(); + static final VolcanoSpeechTts tts = VolcanoSpeechTts._(); + + /// 释放资源 + static Future dispose() async { + await asr.dispose(); + await tts.dispose(); + } +} + +/// 火山引擎语音识别服务 +class VolcanoSpeechAsr { + static const MethodChannel _channel = MethodChannel('volcano_speech/asr'); + static const EventChannel _eventChannel = EventChannel('volcano_speech/asr_events'); + + /// ASR事件流控制器 + static final StreamController _eventStreamController = StreamController.broadcast(); + + /// ASR事件流 + Stream get events => _eventStreamController.stream; + + /// 是否已初始化事件监听 + bool _eventListenerInitialized = false; + + VolcanoSpeechAsr._() { + _initEventListener(); + } + + /// 初始化ASR事件监听 + void _initEventListener() { + if (_eventListenerInitialized) return; + + _eventChannel.receiveBroadcastStream().listen((dynamic event) { + if (event is Map) { + _eventStreamController.add(AsrEvent.fromMap(event)); + } + }); + + _eventListenerInitialized = true; + } + + /// 获取平台版本信息 + Future getPlatformVersion() async { + return await _channel.invokeMethod('getPlatformVersion'); + } + + /// 初始化语音识别引擎 + /// + /// [appId] 火山引擎AppID + /// [apiKey] 火山引擎ApiKey + /// [supportedLanguages] 支持的语言列表 + /// [useBigModel] 是否使用大模型识别 + /// [resourceId] 大模型资源ID（仅当useBigModel为true时有效） + Future initialize({ + required String appId, + required String apiKey, + List supportedLanguages = const ['zh-CN'], + bool useBigModel = false, + String resourceId = '', + }) async { + return await _channel.invokeMethod('initialize', { + 'appId': appId, + 'apiKey': apiKey, + 'supportedLanguages': supportedLanguages, + 'useBigModel': useBigModel, + 'resourceId': resourceId, + }) ?? false; + } + + /// 设置识别语言 + /// + /// [language] 语言代码，例如 zh-CN、en-US + Future setLanguage(String language) async { + return await _channel.invokeMethod('setLanguage', { + 'language': language, + }) ?? false; + } + + /// 设置热词 + /// + /// [hotWords] 热词JSON字符串，例如 {"hotwords":[{"word":"快速入门","scale":2.0}]} + Future setHotWords(String hotWords) async { + return await _channel.invokeMethod('setHotWords', { + 'hotWords': hotWords, + }) ?? false; + } + + /// 设置ASR请求参数 + /// + /// [params] 请求参数JSON字符串 + Future setRequestParams(String params) async { + return await _channel.invokeMethod('setRequestParams', { + 'params': params, + }) ?? false; + } + + /// 启用语音停顿、分句、分词信息输出 + /// + /// [enable] 是否启用 + Future setShowUtterances(bool enable) async { + return await _channel.invokeMethod('setShowUtterances', { + 'enable': enable, + }) ?? false; + } + + /// 一次性识别（直到说话结束） + /// + /// 返回识别结果文本和语言 + Future> recognizeOnce() async { + final result = await _channel.invokeMethod('recognizeOnce'); + return { + 'text': result['text'] ?? '', + 'language': result['language'] ?? 'zh-CN', + }; + } + + /// 开始连续识别 + /// + /// 通过[events]流监听识别结果 + Future startContinuousRecognition() async { + return await _channel.invokeMethod('startContinuousRecognition') ?? false; + } + + /// 停止连续识别 + Future stopContinuousRecognition() async { + return await _channel.invokeMethod('stopContinuousRecognition') ?? false; + } + + /// 检查是否正在连续识别 + Future isContinuousRecognitionActive() async { + return await _channel.invokeMethod('isContinuousRecognitionActive') ?? false; + } + + /// 开始长按识别（按下开始，抬起结束） + /// + /// 通过[events]流监听识别结果 + Future startListening() async { + return await _channel.invokeMethod('startListening') ?? false; + } + + /// 停止长按识别 + Future stopListening() async { + return await _channel.invokeMethod('stopListening') ?? false; + } + + /// 释放ASR资源 + Future dispose() async { + return await _channel.invokeMethod('dispose') ?? false; + } +} + +/// 火山引擎语音合成服务 +class VolcanoSpeechTts { + static const MethodChannel _channel = MethodChannel('volcano_speech/tts'); + static const EventChannel _eventChannel = EventChannel('volcano_speech/tts_events'); + + /// TTS进度事件流控制器 + static final StreamController _progressStreamController = StreamController.broadcast(); + + /// TTS进度事件流 + Stream get progressEvents => _progressStreamController.stream; + + /// 是否已初始化事件监听 + bool _eventListenerInitialized = false; + + VolcanoSpeechTts._() { + _initEventListener(); + } + + /// 初始化TTS事件监听 + void _initEventListener() { + if (_eventListenerInitialized) return; + + _eventChannel.receiveBroadcastStream().listen((dynamic event) { + if (event is Map) { + final progress = event['progress'] as double?; + final reqId = event['reqId'] as String?; + + if (progress != null && reqId != null) { + _progressStreamController.add(TtsProgressEvent( + progress: progress, + reqId: reqId, + )); + } + } + }); + + _eventListenerInitialized = true; + } + + /// 获取平台版本信息 + Future getPlatformVersion() async { + return await _channel.invokeMethod('getPlatformVersion'); + } + + /// 启用大模型TTS + /// + /// [enable] 是否启用大模型TTS + /// [resourceId] 资源ID + Future enableBigModelTts({ + required bool enable, + String resourceId = '', + }) async { + return await _channel.invokeMethod('enableBigModelTts', { + 'enable': enable, + 'resourceId': resourceId, + }) ?? false; + } + + /// 初始化语音合成引擎 + /// + /// [appId] 火山引擎AppID + /// [apiKey] 火山引擎ApiKey + Future initialize({ + required String appId, + required String apiKey, + }) async { + return await _channel.invokeMethod('initialize', { + 'appId': appId, + 'apiKey': apiKey, + }) ?? false; + } + + /// 设置音色 + /// + /// [voiceName] 音色名称 + /// [voiceType] 音色类型，默认 qingxin + Future setVoice({ + required String voiceName, + String voiceType = 'qingxin', + }) async { + return await _channel.invokeMethod('setVoice', { + 'voiceName': voiceName, + 'voiceType': voiceType, + }) ?? false; + } + + /// 设置离线音色 + /// + /// [voiceName] 音色名称 + /// [voiceType] 音色类型，默认 qingxin + Future setOfflineVoice({ + required String voiceName, + String voiceType = 'qingxin', + }) async { + return await _channel.invokeMethod('setOfflineVoice', { + 'voiceName': voiceName, + 'voiceType': voiceType, + }) ?? false; + } + + /// 设置大模型声音ID + /// + /// [voiceId] 声音ID + Future setBigModelVoiceId(String voiceId) async { + return await _channel.invokeMethod('setBigModelVoiceId', { + 'voiceId': voiceId, + }) ?? false; + } + + /// 设置大模型请求参数 + /// + /// [params] 请求参数，JSON字符串 + Future setBigModelRequestParams(String params) async { + return await _channel.invokeMethod('setBigModelRequestParams', { + 'params': params, + }) ?? false; + } + + /// 设置工作模式 + /// + /// [mode] 工作模式 + Future setWorkMode(TtsWorkMode mode) async { + String modeStr; + switch (mode) { + case TtsWorkMode.online: + modeStr = 'online'; + break; + case TtsWorkMode.offline: + modeStr = 'offline'; + break; + case TtsWorkMode.both: + modeStr = 'both'; + break; + case TtsWorkMode.alternate: + modeStr = 'alternate'; + break; + case TtsWorkMode.file: + modeStr = 'file'; + break; + } + + return await _channel.invokeMethod('setWorkMode', { + 'mode': modeStr, + }) ?? false; + } + + /// 设置文本类型 + /// + /// [type] 文本类型，plain或ssml + Future setTextType(TtsTextType type) async { + String typeStr; + switch (type) { + case TtsTextType.plain: + typeStr = 'plain'; + break; + case TtsTextType.ssml: + typeStr = 'ssml'; + break; + } + + return await _channel.invokeMethod('setTextType', { + 'type': typeStr, + }) ?? false; + } + + /// 设置是否启用缓存 + /// + /// [enable] 是否启用 + Future setEnableCache(bool enable) async { + return await _channel.invokeMethod('setEnableCache', { + 'enable': enable, + }) ?? false; + } + + /// 设置情感 + /// + /// [emotion] 情感，例如 neutral、happy、angry、sad等 + Future setEmotion(String emotion) async { + return await _channel.invokeMethod('setEmotion', { + 'emotion': emotion, + }) ?? false; + } + + /// 设置是否启用情感预测 + /// + /// [enable] 是否启用 + Future setEnableEmotionPredict(bool enable) async { + return await _channel.invokeMethod('setEnableEmotionPredict', { + 'enable': enable, + }) ?? false; + } + + /// 设置是否启用声音克隆 + /// + /// [enable] 是否启用 + /// [backendCluster] 后端集群 + Future setEnableVoiceClone({ + required bool enable, + String backendCluster = '', + }) async { + return await _channel.invokeMethod('setEnableVoiceClone', { + 'enable': enable, + 'backendCluster': backendCluster, + }) ?? false; + } + + /// 设置是否启用回声消除 + /// + /// [enable] 是否启用 + Future setEnableAEC(bool enable) async { + return await _channel.invokeMethod('setEnableAEC', { + 'enable': enable, + }) ?? false; + } + + /// 下载离线资源 + /// + /// [voiceTypes] 音色类型列表 + /// [languages] 语言列表 + Future downloadOfflineResource({ + List voiceTypes = const ['qingxin'], + List languages = const ['zh-CN'], + }) async { + return await _channel.invokeMethod('downloadOfflineResource', { + 'voiceTypes': voiceTypes, + 'languages': languages, + }) ?? false; + } + + /// 设置语音参数 + /// + /// [rate] 语速 -500~500 + /// [volume] 音量 0~100 + /// [pitch] 音调 -500~500 + /// [silenceDuration] 静音时长，毫秒 + Future setSpeechParams({ + int rate = 0, + int volume = 100, + int pitch = 0, + int silenceDuration = 0, + }) async { + return await _channel.invokeMethod('setSpeechParams', { + 'rate': rate, + 'volume': volume, + 'pitch': pitch, + 'silenceDuration': silenceDuration, + }) ?? false; + } + + /// 设置音频输出类型 + /// + /// [outputType] 输出类型，speaker(扬声器)，earpiece(听筒)，auto(自动) + Future setAudioOutputType(String outputType) async { + return await _channel.invokeMethod('setAudioOutputType', { + 'outputType': outputType, + }) ?? false; + } + + /// 合成并播放文本 + /// + /// [text] 待合成的文本 + Future speakText(String text) async { + return await _channel.invokeMethod('speakText', { + 'text': text, + }) ?? false; + } + + /// 使用大模型合成并播放文本 + /// + /// [text] 待合成的文本 + /// [voiceId] 声音ID + /// [params] 额外参数，JSON字符串 + Future speakWithBigModel({ + required String text, + required String voiceId, + String params = '', + }) async { + return await _channel.invokeMethod('speakWithBigModel', { + 'text': text, + 'voiceId': voiceId, + 'params': params, + }) ?? false; + } + + /// 暂停播放 + Future pausePlayback() async { + return await _channel.invokeMethod('pausePlayback') ?? false; + } + + /// 恢复播放 + Future resumePlayback() async { + return await _channel.invokeMethod('resumePlayback') ?? false; + } + + /// 停止播放 + Future stopSpeaking() async { + return await _channel.invokeMethod('stopSpeaking') ?? false; + } + + /// 释放TTS资源 + Future dispose() async { + return await _channel.invokeMethod('dispose') ?? false; + } +} \ No newline at end of file diff --git a/local_plugins/volcano_speech/pubspec.yaml b/local_plugins/volcano_speech/pubspec.yaml new file mode 100644 index 000000000..cd90d8b14 --- /dev/null +++ b/local_plugins/volcano_speech/pubspec.yaml @@ -0,0 +1,28 @@ +name: volcano_speech +description: 火山引擎语音合成服务插件 +version: 0.0.1 +homepage: + +environment: + sdk: ">=2.17.0 <3.0.0" + flutter: ">=2.5.0" + +dependencies: + flutter: + sdk: flutter + +dev_dependencies: + flutter_test: + sdk: flutter + flutter_lints: ^2.0.0 + +# The following section is specific to Flutter packages. +flutter: + # This section identifies this Flutter project as a plugin project. + plugin: + platforms: + android: + package: com.yunqiinnovation.volcano_speech + pluginClass: VolcanoSpeechPlugin + ios: + pluginClass: VolcanoSpeechPlugin \ No newline at end of file diff --git a/pubspec.yaml b/pubspec.yaml index 19366500b..41fa9edf0 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -64,6 +64,8 @@ dependencies: path: local_plugins/azure_speech open_ai_service: path: local_plugins/open_ai_service + volcano_speech: + path: local_plugins/volcano_speech dev_dependencies: flutter_test: From 28b666cd33faa95314c45bc70302b8db384e5bdb Mon Sep 17 00:00:00 2001 From: wolfplus Date: Mon, 7 Apr 2025 21:18:12 +0100 Subject: [PATCH 05/14] mcp --- android/app/build.gradle.kts | 14 + .../yunqiinnovation/deepsound/MainActivity.kt | 31 +- .../deepsound/SystemFunctionHandler.kt | 225 -------- .../deepsound/VoiceFunctionHandler.kt | 510 ------------------ .../VoiceInteractionHandler copy.kt.bak | 456 ++++++++++++++++ .../deepsound/VoiceInteractionHandler.kt | 107 ++-- .../deepsound/VoiceInteractionService.kt | 14 +- android/build.gradle.kts | 11 + android/settings.gradle.kts | 2 +- 9 files changed, 551 insertions(+), 819 deletions(-) delete mode 100644 android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt delete mode 100644 android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt create mode 100644 android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler copy.kt.bak diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 026060648..8c371408e 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -39,6 +39,20 @@ android { isCheckReleaseBuilds = false } + // 添加packaging配置，排除冲突文件 + packaging { + resources { + excludes.add("META-INF/DEPENDENCIES") + excludes.add("META-INF/LICENSE") + excludes.add("META-INF/LICENSE.txt") + excludes.add("META-INF/license.txt") + excludes.add("META-INF/NOTICE") + excludes.add("META-INF/NOTICE.txt") + excludes.add("META-INF/notice.txt") + excludes.add("META-INF/*.kotlin_module") + } + } + defaultConfig { // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html). applicationId = "com.yunqiinnovation.deepsound" diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt index 9f06844db..1c4717f11 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt @@ -170,6 +170,8 @@ class MainActivity: FlutterActivity() { var openaiApiKey: String = "" var openaiBaseUrl: String? = null var openaiModel: String? = null + var volcanoSpeechAppId: String = "" + var volcanoSpeechAppToken: String = "" // 安全存储相关常量 private const val SECURE_PREFS_FILENAME = "deep_voice_secure_prefs" @@ -180,6 +182,8 @@ class MainActivity: FlutterActivity() { private const val KEY_OPENAI_BASE_URL = "openai_base_url" private const val KEY_OPENAI_MODEL = "openai_model" private const val KEY_MCP_SERVER_ENDPOINT = "mcp_server_endpoint" + private const val KEY_VOLCANO_SPEECH_APP_ID = "volcano_speech_app_id" + private const val KEY_VOLCANO_SPEECH_APP_TOKEN = "volcano_speech_app_token" // 会话管理 private const val KEY_SESSION_ID = "session_id" @@ -218,6 +222,8 @@ class MainActivity: FlutterActivity() { .putString(KEY_OPENAI_BASE_URL, openaiBaseUrl) .putString(KEY_SESSION_ID, currentSessionId) .putString(KEY_OPENAI_MODEL, openaiModel) + .putString(KEY_VOLCANO_SPEECH_APP_ID, volcanoSpeechAppId) + .putString(KEY_VOLCANO_SPEECH_APP_TOKEN, volcanoSpeechAppToken) .apply() FileLogger.d("MainActivity", "密钥已安全保存到加密存储中") @@ -264,6 +270,8 @@ class MainActivity: FlutterActivity() { openaiApiKey = sharedPreferences.getString(KEY_OPENAI_API_KEY, "") ?: "" openaiBaseUrl = sharedPreferences.getString(KEY_OPENAI_BASE_URL, null) openaiModel = sharedPreferences.getString(KEY_OPENAI_MODEL, null) + volcanoSpeechAppId = sharedPreferences.getString(KEY_VOLCANO_SPEECH_APP_ID, "") ?: "" + volcanoSpeechAppToken = sharedPreferences.getString(KEY_VOLCANO_SPEECH_APP_TOKEN, "") ?: "" FileLogger.d("MainActivity", "已从加密存储加载密钥") // 检查是否成功获取所有必要密钥 @@ -505,21 +513,14 @@ class MainActivity: FlutterActivity() { FileLogger.d(TAG, "启动语音交互服务") // 获取配置参数 - val key = call.argument("azure_speech_key") ?: "" - val region = call.argument("azure_speech_region") ?: "" - val openaiKey = call.argument("openai_api_key") ?: "" - val baseUrl = call.argument("openai_base_url") - val model = call.argument("openai_model") - // 设置 Azure Speech 和 AI 配置 - azureSpeechKey = key - azureSpeechRegion = region - openaiApiKey = openaiKey - if (baseUrl != null) { - openaiBaseUrl = baseUrl - } - if (model != null) { - openaiModel = model - } + azureSpeechKey = call.argument("azure_speech_key") ?: "" + azureSpeechRegion = call.argument("azure_speech_region") ?: "" + openaiApiKey = call.argument("openai_api_key") ?: "" + openaiBaseUrl = call.argument("openai_base_url") + openaiModel = call.argument("openai_model") + volcanoSpeechAppId = call.argument("volcano_speech_app_id") ?: "" + volcanoSpeechAppToken = call.argument("volcano_speech_app_token") ?: "" + // 保存密钥到安全存储 saveKeysToSecureStorage(applicationContext) diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt deleted file mode 100644 index 4e28f26aa..000000000 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt +++ /dev/null @@ -1,225 +0,0 @@ -package com.yunqiinnovation.deepsound - -import android.content.Context -import android.content.Intent -import android.content.pm.PackageManager -import android.net.Uri -import android.provider.ContactsContract -import android.telephony.SmsManager -import com.yunqiinnovation.deepsound.core.utils.FileLogger - -/** - * 系统功能处理器 - 负责调用系统API执行各种功能 - */ -class SystemFunctionHandler(private val context: Context) { - companion object { - private const val TAG = "SystemFunctionHandler" - } - - /** - * 启动翻译模式 - * @return 操作结果 - */ - fun enterTranslationMode(): Result { - FileLogger.d(TAG, "进入翻译模式") - // 此处应添加实际的翻译模式实现 - // 可能需要启动一个专门的翻译服务或活动 - - // 广播通知APP的其他部分进入翻译模式 - // 使用VoiceInteractionService中定义的常量 - val intent = Intent(VoiceInteractionService.ACTION_ENTER_TRANSLATION_MODE).apply { - - putExtra("timestamp", System.currentTimeMillis()) - } - - // 发送广播 - context.sendBroadcast(intent) - FileLogger.d(TAG, "已发送广播: ${VoiceInteractionService.ACTION_ENTER_TRANSLATION_MODE}") - - return Result( - success = true, - message = "已进入翻译模式" - ) - } - - /** - * 发送短信 - * @param contact 联系人（名称或电话号码） - * @param message 短信内容 - * @return 操作结果 - */ - fun sendTextMessage(contact: String, message: String): Result { - FileLogger.d(TAG, "发送短信给 $contact: $message") - - try { - val phoneNumber = getPhoneNumberForContact(contact) - if (phoneNumber.isEmpty()) { - return Result( - success = false, - message = "未找到联系人信息: $contact" - ) - } - - // 使用SmsManager发送短信 - val smsManager = SmsManager.getDefault() - smsManager.sendTextMessage(phoneNumber, null, message, null, null) - - return Result( - success = true, - message = "已向${contact}发送短信: $message" - ) - } catch (e: Exception) { - FileLogger.e(TAG, "发送短信失败: ${e.message}", e) - return Result( - success = false, - message = "发送短信失败: ${e.message}" - ) - } - } - - /** - * 拨打电话 - * @param contact 联系人（名称或电话号码） - * @return 操作结果 - */ - fun makePhoneCall(contact: String): Result { - FileLogger.d(TAG, "拨打电话给 $contact") - - try { - val phoneNumber = getPhoneNumberForContact(contact) - if (phoneNumber.isEmpty()) { - return Result( - success = false, - message = "未找到联系人信息: $contact" - ) - } - - // 创建拨号意图 - val intent = Intent(Intent.ACTION_CALL) - intent.data = Uri.parse("tel:$phoneNumber") - intent.flags = Intent.FLAG_ACTIVITY_NEW_TASK - context.startActivity(intent) - - return Result( - success = true, - message = "正在拨打${contact}的电话" - ) - } catch (e: Exception) { - FileLogger.e(TAG, "拨打电话失败: ${e.message}", e) - return Result( - success = false, - message = "拨打电话失败: ${e.message}" - ) - } - } - - /** - * 打开应用程序 - * @param appName 应用名称 - * @return 操作结果 - */ - fun openApp(appName: String): Result { - FileLogger.d(TAG, "打开应用: $appName") - - try { - // 查找应用的包名 - val packageManager = context.packageManager - val intent = Intent(Intent.ACTION_MAIN, null) - intent.addCategory(Intent.CATEGORY_LAUNCHER) - - val apps = packageManager.queryIntentActivities(intent, 0) - - for (resolveInfo in apps) { - val appLabel = resolveInfo.loadLabel(packageManager).toString() - if (appLabel.equals(appName, ignoreCase = true) || - appLabel.contains(appName, ignoreCase = true)) { - - // 找到匹配的应用 - val packageName = resolveInfo.activityInfo.packageName - val launchIntent = packageManager.getLaunchIntentForPackage(packageName) - - if (launchIntent != null) { - launchIntent.flags = Intent.FLAG_ACTIVITY_NEW_TASK - context.startActivity(launchIntent) - - return Result( - success = true, - message = "正在打开${appName}应用" - ) - } - } - } - - return Result( - success = false, - message = "未找到应用: $appName" - ) - } catch (e: Exception) { - FileLogger.e(TAG, "打开应用失败: ${e.message}", e) - return Result( - success = false, - message = "打开应用失败: ${e.message}" - ) - } - } - - /** - * 根据联系人名称获取电话号码 - * @param contact 联系人名称或电话号码 - * @return 电话号码 - */ - private fun getPhoneNumberForContact(contact: String): String { - // 如果输入的是电话号码（简单判断，包含数字和特殊符号） - if (contact.matches(Regex("^[+\\d\\s\\-()]*\$")) && contact.any { it.isDigit() }) { - // 移除非数字字符 - return contact.replace(Regex("[^+\\d]"), "") - } - - // 从联系人数据库查询 - try { - val contentResolver = context.contentResolver - val uri = Uri.withAppendedPath( - ContactsContract.Contacts.CONTENT_FILTER_URI, - Uri.encode(contact) - ) - - val projection = arrayOf( - ContactsContract.Contacts._ID, - ContactsContract.Contacts.DISPLAY_NAME, - ContactsContract.Contacts.HAS_PHONE_NUMBER - ) - - contentResolver.query(uri, projection, null, null, null)?.use { cursor -> - if (cursor.moveToFirst()) { - val hasPhoneNumber = cursor.getInt(cursor.getColumnIndexOrThrow(ContactsContract.Contacts.HAS_PHONE_NUMBER)) - - if (hasPhoneNumber > 0) { - val contactId = cursor.getString(cursor.getColumnIndexOrThrow(ContactsContract.Contacts._ID)) - val phoneUri = ContactsContract.CommonDataKinds.Phone.CONTENT_URI - val phoneProjection = arrayOf(ContactsContract.CommonDataKinds.Phone.NUMBER) - val phoneSelection = "${ContactsContract.CommonDataKinds.Phone.CONTACT_ID} = ?" - val phoneSelectionArgs = arrayOf(contactId) - - contentResolver.query(phoneUri, phoneProjection, phoneSelection, phoneSelectionArgs, null)?.use { phoneCursor -> - if (phoneCursor.moveToFirst()) { - return phoneCursor.getString(phoneCursor.getColumnIndexOrThrow(ContactsContract.CommonDataKinds.Phone.NUMBER)) - } - } - } - } - } - } catch (e: Exception) { - FileLogger.e(TAG, "查询联系人信息失败: ${e.message}", e) - } - - return "" - } - - /** - * 操作结果数据类 - */ - data class Result( - val success: Boolean, - val message: String - ) -} \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt deleted file mode 100644 index c15ca7e17..000000000 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt +++ /dev/null @@ -1,510 +0,0 @@ -package com.yunqiinnovation.deepsound - -import org.json.JSONArray -import org.json.JSONObject -import com.yunqiinnovation.open_ai_service.OpenAIService -import com.yunqiinnovation.deepsound.core.utils.FileLogger -import android.content.Context - -/** - * 语音功能处理器 - 处理AI函数调用 - */ -class VoiceFunctionHandler( - private val openAIService: OpenAIService, - private val context: Context -) { - companion object { - private const val TAG = "VoiceFunctionHandler" - } - - // 添加系统功能处理器 - private val systemFunctionHandler = SystemFunctionHandler(context) - - /** - * 初始化并注册所有可用的函数 - */ - fun initialize() { - try { - // 注册退出交互函数 - registerExitInteractionFunction() - - // 注册翻译模式函数 - registerTranslationModeFunction() - - // 注册发送短信函数 - registerSendTextMessageFunction() - - // 注册拨打电话函数 - registerMakePhoneCallFunction() - - // 注册打开应用函数 - registerOpenAppFunction() - - // 在这里可以注册更多函数 - - } catch (e: Exception) { - FileLogger.e(TAG, "初始化函数处理器失败: ${e.message}", e) - } - } - - /** - * 注册退出交互函数 - */ - private fun registerExitInteractionFunction() { - try { - openAIService.registerFunction( - "exit_interaction", - "退出当前语音交互", - JSONObject(""" - { - "type": "object", - "properties": {}, - "required": [] - } - """) - ) - FileLogger.d(TAG, "退出交互功能已注册") - } catch (e: Exception) { - FileLogger.e(TAG, "注册退出交互函数失败: ${e.message}", e) - } - } - - /** - * 注册翻译模式函数 - */ - private fun registerTranslationModeFunction() { - try { - openAIService.registerFunction( - "enter_translation_mode", - "进入翻译模式", - JSONObject(""" - { - "type": "object", - "properties": {}, - "required": [] - } - """) - ) - FileLogger.d(TAG, "翻译模式功能已注册") - } catch (e: Exception) { - FileLogger.e(TAG, "注册翻译模式函数失败: ${e.message}", e) - } - } - - /** - * 注册发送短信函数 - */ - private fun registerSendTextMessageFunction() { - try { - openAIService.registerFunction( - "send_text_message", - "发送短信", - JSONObject(""" - { - "type": "object", - "properties": { - "contact": { - "type": "string", - "description": "联系人姓名或电话号码" - }, - "message": { - "type": "string", - "description": "短信内容" - } - }, - "required": ["contact", "message"] - } - """) - ) - FileLogger.d(TAG, "发送短信功能已注册") - } catch (e: Exception) { - FileLogger.e(TAG, "注册发送短信函数失败: ${e.message}", e) - } - } - - /** - * 注册拨打电话函数 - */ - private fun registerMakePhoneCallFunction() { - try { - openAIService.registerFunction( - "make_phone_call", - "拨打电话", - JSONObject(""" - { - "type": "object", - "properties": { - "contact": { - "type": "string", - "description": "联系人姓名或电话号码" - } - }, - "required": ["contact"] - } - """) - ) - FileLogger.d(TAG, "拨打电话功能已注册") - } catch (e: Exception) { - FileLogger.e(TAG, "注册拨打电话函数失败: ${e.message}", e) - } - } - - /** - * 注册打开应用函数 - */ - private fun registerOpenAppFunction() { - try { - openAIService.registerFunction( - "open_app", - "打开应用", - JSONObject(""" - { - "type": "object", - "properties": { - "app_name": { - "type": "string", - "description": "应用名称" - } - }, - "required": ["app_name"] - } - """) - ) - FileLogger.d(TAG, "打开应用功能已注册") - } catch (e: Exception) { - FileLogger.e(TAG, "注册打开应用函数失败: ${e.message}", e) - } - } - - /** - * 处理函数调用 - * - * @param functionCall 函数调用信息 - * @param messages 消息历史 - * @param callback 回调，处理退出等操作 - * @return 是否已处理函数调用 - */ - fun handleFunctionCall( - functionCall: JSONObject, - messages: JSONArray, - callback: FunctionCallCallback - ): Boolean { - val functionName = functionCall.getString("name") - FileLogger.d(TAG, "处理函数调用: $functionName") - - return when (functionName) { - "exit_interaction" -> { - handleExitInteraction(functionCall, messages, callback) - true - } - "enter_translation_mode" -> { - handleEnterTranslationMode(functionCall, messages, callback) - true - } - "send_text_message" -> { - handleSendTextMessage(functionCall, messages, callback) - true - } - "make_phone_call" -> { - handleMakePhoneCall(functionCall, messages, callback) - true - } - "open_app" -> { - handleOpenApp(functionCall, messages, callback) - true - } - else -> { - // 未知函数，返回默认结果 - handleUnknownFunction(functionCall, messages, callback) - false - } - } - } - - /** - * 处理退出交互函数 - */ - private fun handleExitInteraction( - functionCall: JSONObject, - messages: JSONArray, - callback: FunctionCallCallback - ) { - FileLogger.d(TAG, "处理退出交互函数") - - val responseBuilder = StringBuilder() - - openAIService.sendFunctionCallResult( - messages = messages, - functionCall = functionCall, - functionResult = "{\"result\": \"已退出语音交互\"}", - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - responseBuilder.append(token) - } - - override fun onComplete() { - FileLogger.d(TAG, "handleExitInteraction onComplete: ${responseBuilder.toString()}") - val response = responseBuilder.toString() - if (response.isNotEmpty()) { - callback.onExitWithMessage(response) - } else { - callback.onExitWithMessage("已退出语音交互") - } - } - - override fun onError(e: Exception) { - FileLogger.e(TAG, "处理退出交互函数调用出错: ${e.message}") - callback.onError("退出交互时出错") - } - - override fun onFunctionCall(nestedCall: JSONObject) { - FileLogger.e(TAG, "意外收到嵌套函数调用: ${nestedCall.getString("name")}") - } - } - ) - } - - /** - * 处理进入翻译模式函数 - */ - private fun handleEnterTranslationMode( - functionCall: JSONObject, - messages: JSONArray, - callback: FunctionCallCallback - ) { - FileLogger.d(TAG, "处理进入翻译模式函数") - - // 调用系统功能处理器执行翻译模式操作 - val result = systemFunctionHandler.enterTranslationMode() - - val responseBuilder = StringBuilder() - - openAIService.sendFunctionCallResult( - messages = messages, - functionCall = functionCall, - functionResult = "{\"result\": \"${result.message}\"}", - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - responseBuilder.append(token) - callback.onTokenReceived(token) - } - - override fun onComplete() { - FileLogger.d(TAG, "handleEnterTranslationMode onComplete: ${responseBuilder.toString()}") - callback.onComplete() - } - - override fun onError(e: Exception) { - FileLogger.e(TAG, "处理进入翻译模式函数调用出错: ${e.message}") - callback.onError("进入翻译模式时出错") - } - - override fun onFunctionCall(nestedCall: JSONObject) { - callback.onFunctionCall(nestedCall) - } - } - ) - } - - /** - * 处理发送短信函数 - */ - private fun handleSendTextMessage( - functionCall: JSONObject, - messages: JSONArray, - callback: FunctionCallCallback - ) { - FileLogger.d(TAG, "处理发送短信函数") - - try { - val arguments = JSONObject(functionCall.getString("arguments")) - val contact = arguments.getString("contact") - val message = arguments.getString("message") - - // 调用系统功能处理器发送短信 - val result = systemFunctionHandler.sendTextMessage(contact, message) - - val responseBuilder = StringBuilder() - - openAIService.sendFunctionCallResult( - messages = messages, - functionCall = functionCall, - functionResult = "{\"result\": \"${result.message}\"}", - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - responseBuilder.append(token) - callback.onTokenReceived(token) - } - - override fun onComplete() { - FileLogger.d(TAG, "handleSendTextMessage onComplete: ${responseBuilder.toString()}") - callback.onComplete() - } - - override fun onError(e: Exception) { - FileLogger.e(TAG, "处理发送短信函数调用出错: ${e.message}") - callback.onError("发送短信时出错") - } - - override fun onFunctionCall(nestedCall: JSONObject) { - callback.onFunctionCall(nestedCall) - } - } - ) - } catch (e: Exception) { - FileLogger.e(TAG, "解析发送短信参数失败: ${e.message}") - callback.onError("发送短信参数错误") - } - } - - /** - * 处理拨打电话函数 - */ - private fun handleMakePhoneCall( - functionCall: JSONObject, - messages: JSONArray, - callback: FunctionCallCallback - ) { - FileLogger.d(TAG, "处理拨打电话函数") - - try { - val arguments = JSONObject(functionCall.getString("arguments")) - val contact = arguments.getString("contact") - - // 调用系统功能处理器拨打电话 - val result = systemFunctionHandler.makePhoneCall(contact) - - val responseBuilder = StringBuilder() - - openAIService.sendFunctionCallResult( - messages = messages, - functionCall = functionCall, - functionResult = "{\"result\": \"${result.message}\"}", - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - responseBuilder.append(token) - callback.onTokenReceived(token) - } - - override fun onComplete() { - FileLogger.d(TAG, "handleMakePhoneCall onComplete: ${responseBuilder.toString()}") - callback.onComplete() - } - - override fun onError(e: Exception) { - FileLogger.e(TAG, "处理拨打电话函数调用出错: ${e.message}") - callback.onError("拨打电话时出错") - } - - override fun onFunctionCall(nestedCall: JSONObject) { - callback.onFunctionCall(nestedCall) - } - } - ) - } catch (e: Exception) { - FileLogger.e(TAG, "解析拨打电话参数失败: ${e.message}") - callback.onError("拨打电话参数错误") - } - } - - /** - * 处理打开应用函数 - */ - private fun handleOpenApp( - functionCall: JSONObject, - messages: JSONArray, - callback: FunctionCallCallback - ) { - FileLogger.d(TAG, "处理打开应用函数") - - try { - val arguments = JSONObject(functionCall.getString("arguments")) - val appName = arguments.getString("app_name") - - // 调用系统功能处理器打开应用 - val result = systemFunctionHandler.openApp(appName) - - val responseBuilder = StringBuilder() - - openAIService.sendFunctionCallResult( - messages = messages, - functionCall = functionCall, - functionResult = "{\"result\": \"${result.message}\"}", - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - responseBuilder.append(token) - callback.onTokenReceived(token) - } - - override fun onComplete() { - FileLogger.d(TAG, "handleOpenApp onComplete: ${responseBuilder.toString()}") - callback.onComplete() - } - - override fun onError(e: Exception) { - FileLogger.e(TAG, "处理打开应用函数调用出错: ${e.message}") - callback.onError("打开应用时出错") - } - - override fun onFunctionCall(nestedCall: JSONObject) { - callback.onFunctionCall(nestedCall) - } - } - ) - } catch (e: Exception) { - FileLogger.e(TAG, "解析打开应用参数失败: ${e.message}") - callback.onError("打开应用参数错误") - } - } - - /** - * 处理未知函数调用 - */ - private fun handleUnknownFunction( - functionCall: JSONObject, - messages: JSONArray, - callback: FunctionCallCallback - ) { - FileLogger.d(TAG, "处理未知函数: ${functionCall.getString("name")}") - - try { - openAIService.sendFunctionCallResult( - messages = messages, - functionCall = functionCall, - functionResult = "{\"result\": \"处理函数调用中\"}", - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - callback.onTokenReceived(token) - } - - override fun onComplete() { - callback.onComplete() - } - - override fun onError(e: Exception) { - FileLogger.e(TAG, "处理函数调用失败: ${e.message}") - callback.onError("处理函数调用失败: ${e.message}") - } - - override fun onFunctionCall(nestedCall: JSONObject) { - callback.onFunctionCall(nestedCall) - } - } - ) - } catch (e: Exception) { - FileLogger.e(TAG, "处理函数调用失败: ${e.message}") - callback.onError("处理函数调用失败: ${e.message}") - } - } - - /** - * 函数调用回调接口 - */ - interface FunctionCallCallback { - fun onTokenReceived(token: String) - fun onComplete() - fun onError(message: String) - fun onFunctionCall(functionCall: JSONObject) - fun onExitWithMessage(farewell: String) - } -} \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler copy.kt.bak b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler copy.kt.bak new file mode 100644 index 000000000..1a9e2df13 --- /dev/null +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler copy.kt.bak @@ -0,0 +1,456 @@ +package com.yunqiinnovation.deepsound + +import android.content.Context +import org.json.JSONArray +import org.json.JSONObject +import android.util.Log +import com.yunqiinnovation.deepsound.core.utils.FileLogger +import com.yunqiinnovation.azure_speech.AzureAsrHelper +import com.yunqiinnovation.volcano_speech.VolcanoTtsHelper +import com.yunqiinnovation.open_ai_service.OpenAIService + + +/** + * 语音交互处理器 - 处理语音识别、TTS和AI对话相关逻辑 + */ +class VoiceInteractionHandler( + private val context: Context, + private val azureSpeechKey: String, + private val azureSpeechRegion: String, + private val openaiApiKey: String, + private val openaiBaseUrl: String = "", + private val openaiModel: String = "", + private val volcanoSpeechAppId: String, + private val volcanoSpeechAppToken: String +) { + private val TAG = "VoiceInteractionHandler" + + // Azure服务 + private var azureAsrHelper: AzureAsrHelper? = null + private var volcanoTtsHelper: VolcanoTtsHelper? = null + + // OpenAI服务 + private val openAIService = OpenAIService() + + + // 语音功能处理 + private val voiceFunctionHandler = VoiceFunctionHandler(openAIService, context) + + // 当前用户输入 + private var currentUserInput = "" + + // 状态 + private var isInitialized = false + var isRecognitionActive = false + private set + var isTtsSpeaking = false + private set + var hasSpeechDetected = false + private set + + // 回调 + private var callback: InteractionCallback? = null + + /** + * 初始化 + */ + fun initialize(): Boolean { + if (isInitialized) return true + + try { + // 初始化Azure ASR + azureAsrHelper = AzureAsrHelper(context).apply { + initialize(azureSpeechKey, azureSpeechRegion) + } + + // 初始化Volcano TTS (使用大模型TTS) + volcanoTtsHelper = VolcanoTtsHelper(context).apply { + // 这里需要替换为实际的Volcano SDK初始化参数 + // 暂时使用假参数，实际使用时需要替换为真实值 + initialize(volcanoSpeechAppId, volcanoSpeechAppToken, "volc.bigasr.sauc.duration") + + } + + // 初始化OpenAI服务 + openAIService.initialize(openaiApiKey, openaiBaseUrl, openaiModel) + + // 初始化语音功能处理器 + voiceFunctionHandler.initialize() + + isInitialized = true + return true + } catch (e: Exception) { + FileLogger.e(TAG, "初始化失败: ${e.message}", e) + return false + } + } + + /** + * 设置回调 + */ + fun setCallback(callback: InteractionCallback) { + this.callback = callback + } + + /** + * 开始语音识别 + */ + fun startRecognition() { + if (isRecognitionActive) return + + // 检查录音权限 + if (!checkRecordAudioPermission()) { + callback?.onError("需要录音权限，请在设置中授予权限") + return + } + + isRecognitionActive = true + hasSpeechDetected = false + notifyStateChanged() + + try { + azureAsrHelper?.startContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { + override fun onRecognizing(recognizing: String, detectedLanguage: String) { + if (recognizing.isNotEmpty()) { + hasSpeechDetected = true + stopTts() + notifyStateChanged() + } + } + + override fun onResult(result: String, detectedLanguage: String) { + if (result.isNotEmpty()) { + notifyStateChanged() + + processWithOpenAI(result) + + } + + // 重置状态，继续识别 + hasSpeechDetected = false + } + + override fun onSessionStarted() { + notifyStateChanged() + } + + override fun onSessionStopped() { + isRecognitionActive = false + notifyStateChanged() + } + + override fun onCanceled(reason: String, errorDetails: String) { + isRecognitionActive = false + notifyStateChanged() + } + + override fun onError(error: String) { + isRecognitionActive = false + callback?.onError("语音识别出错") + notifyStateChanged() + } + + override fun onSuccess(message: String) { + // 处理成功事件 + } + }) + } catch (e: Exception) { + isRecognitionActive = false + FileLogger.e(TAG, "启动语音识别失败: ${e.message}", e) + callback?.onError("启动语音识别失败") + notifyStateChanged() + } + } + + /** + * 停止语音识别 + */ + fun stopRecognition() { + if (!isRecognitionActive) return + + FileLogger.d(TAG, "停止语音识别") + + try { + azureAsrHelper?.stopContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { + override fun onResult(result: String, detectedLanguage: String) {} + override fun onRecognizing(recognizing: String, detectedLanguage: String) {} + override fun onSessionStarted() {} + override fun onSessionStopped() { + isRecognitionActive = false + FileLogger.d(TAG, "语音识别会话已停止") + notifyStateChanged() + } + override fun onCanceled(reason: String, errorDetails: String) { + isRecognitionActive = false + FileLogger.d(TAG, "语音识别已取消: $reason") + notifyStateChanged() + } + override fun onError(error: String) { + isRecognitionActive = false + FileLogger.e(TAG, "停止语音识别时出错: $error") + notifyStateChanged() + } + override fun onSuccess(message: String) { + isRecognitionActive = false + FileLogger.d(TAG, "语音识别已停止: $message") + notifyStateChanged() + } + }) + } catch (e: Exception) { + FileLogger.e(TAG, "停止语音识别异常: ${e.message}", e) + // 确保状态一致性 + isRecognitionActive = false + notifyStateChanged() + } + } + + /** + * 使用OpenAI处理语音识别结果 + */ + private fun processWithOpenAI(text: String) { + // 保存当前用户输入，用于后续同步聊天记录 + currentUserInput = text + + Thread { + try { + val messages = JSONArray().apply { + put(openAIService.createUserMessage(text)) + } + + // 创建响应构建器 + val responseBuilder = StringBuilder() + + openAIService.sendMessageStream( + messages = messages, + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + // 累加响应内容 + responseBuilder.append(token) + } + + override fun onComplete() { + // 处理完整响应 + val response = responseBuilder.toString() + if (response.isNotEmpty()) { + // 播放AI回复 + Log.d(TAG, "AI 回复: $response") + + speakAIResponse(response) + + // 同步聊天记录到Flutter端 + sendChatHistoryUpdate("personal_assistant", text, response) + } + } + + override fun onError(e: Exception) { + FileLogger.e(TAG, "AI流式处理出错: ${e.message}", e) + callback?.onError("AI处理出错") + } + + override fun onFunctionCall(call: JSONObject) { + FileLogger.d(TAG, "收到函数调用请求: ${call.getString("name")}") + + // 使用函数处理器处理函数调用 + val handled = voiceFunctionHandler.handleFunctionCall( + functionCall = call, + messages = messages, + callback = object : VoiceFunctionHandler.FunctionCallCallback { + override fun onTokenReceived(token: String) { + responseBuilder.append(token) + } + + override fun onComplete() { + val response = responseBuilder.toString() + if (response.isNotEmpty()) { + // 播放AI回复 + Log.d(TAG, "AI Function Call 回复: $response") + speakAIResponse(response) + + // 同步聊天记录到Flutter端 + sendChatHistoryUpdate("personal_assistant", text, response) + } + notifyStateChanged() + } + + override fun onError(message: String) { + FileLogger.e(TAG, "函数处理出错: $message") + callback?.onError(message) + } + + override fun onFunctionCall(nestedCall: JSONObject) { + FileLogger.d(TAG, "收到嵌套函数调用: ${nestedCall.getString("name")}") + // 不处理嵌套函数调用，直接返回错误提示 + // speakAIResponse("抱歉，暂不支持嵌套函数调用") + } + + override fun onExitWithMessage(farewell: String) { + // 播放退出消息 + speakAIResponse(farewell) + + // 同步聊天记录 + sendChatHistoryUpdate("personal_assistant", text, farewell) + + // 停止语音识别 + stopRecognition() + } + } + ) + + if (!handled) { + // 如果函数没有被处理，作为普通文本处理 + FileLogger.d(TAG, "函数未处理，作为普通文本处理") + speakAIResponse("我无法处理这个请求") + sendChatHistoryUpdate("personal_assistant", text, "我无法处理这个请求") + } + } + } + ) + + } catch (e: Exception) { + FileLogger.e(TAG, "AI处理出错: ${e.message}", e) + callback?.onError("AI处理出错") + } + }.start() + } + + /** + * 播放TTS + */ + fun playTts(text: String, callback: TtsCallback? = null) { + isTtsSpeaking = true + notifyStateChanged() + + volcanoTtsHelper?.speak(text, object : VolcanoTtsHelper.TTSCallback { + override fun onStart(reqId: String) { + // TTS开始播放事件 + } + + override fun onProgress(reqId: String, progress: Double) { + // TTS播放进度事件 + } + + override fun onComplete(reqId: String) { + isTtsSpeaking = false + notifyStateChanged() + callback?.onComplete() + } + + override fun onError(reqId: String, errorCode: Int, errorMsg: String) { + isTtsSpeaking = false + notifyStateChanged() + callback?.onError(errorMsg) + } + }) + } + + /** + * 停止TTS播放 + */ + fun stopTts() { + if (isTtsSpeaking) { + volcanoTtsHelper?.stop() + isTtsSpeaking = false + notifyStateChanged() + } + } + + /** + * 播放AI回复 + */ + private fun speakAIResponse(text: String) { + isTtsSpeaking = true + notifyStateChanged() + + volcanoTtsHelper?.speak(text, object : VolcanoTtsHelper.TTSCallback { + override fun onStart(reqId: String) { + // TTS开始播放事件 + } + + override fun onProgress(reqId: String, progress: Double) { + // TTS播放进度事件 + } + + override fun onComplete(reqId: String) { + isTtsSpeaking = false + notifyStateChanged() + } + + override fun onError(reqId: String, errorCode: Int, errorMsg: String) { + isTtsSpeaking = false + notifyStateChanged() + } + }) + } + + /** + * 释放资源 + */ + fun dispose() { + // 停止语音识别 + stopRecognition() + + // 停止TTS播放 + stopTts() + + // 释放Azure资源 + azureAsrHelper?.let { + FileLogger.d(TAG, "关闭Azure ASR服务") + it.dispose() + } + + volcanoTtsHelper?.let { + FileLogger.d(TAG, "关闭Volcano TTS服务") + it.release() + } + + FileLogger.d(TAG, "语音交互处理器资源已释放") + } + + /** + * 检查录音权限 + */ + private fun checkRecordAudioPermission(): Boolean { + val permission = android.Manifest.permission.RECORD_AUDIO + val result = context.checkCallingOrSelfPermission(permission) + return result == android.content.pm.PackageManager.PERMISSION_GRANTED + } + + /** + * 通知状态变化 + */ + private fun notifyStateChanged() { + callback?.onStateChanged(isRecognitionActive, isTtsSpeaking, hasSpeechDetected) + } + + /** + * 发送聊天历史更新 + */ + private fun sendChatHistoryUpdate(agentId: String, userMessage: String, assistantMessage: String) { + val intent = android.content.Intent(VoiceInteractionService.ACTION_CHAT_HISTORY_UPDATED).apply { + putExtra("agentId", agentId) + putExtra("userMessage", userMessage) + putExtra("assistantMessage", assistantMessage) + putExtra("timestamp", System.currentTimeMillis()) + } + + // 发送广播 + context.sendBroadcast(intent) + } + + /** + * 交互回调接口 + */ + interface InteractionCallback { + fun onStateChanged(isRecognitionActive: Boolean, isTtsSpeaking: Boolean, hasSpeechDetected: Boolean) + fun onError(message: String) + fun onPromptRequest(message: String) + } + + /** + * TTS回调接口 + */ + interface TtsCallback { + fun onComplete() + fun onError(error: String) + } +} \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt index 1d6a528b3..b885117c0 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt @@ -1,13 +1,17 @@ package com.yunqiinnovation.deepsound +import android.content.BroadcastReceiver import android.content.Context +import android.content.Intent +import android.content.IntentFilter import org.json.JSONArray import org.json.JSONObject import android.util.Log import com.yunqiinnovation.deepsound.core.utils.FileLogger import com.yunqiinnovation.azure_speech.AzureAsrHelper import com.yunqiinnovation.azure_speech.AzureTtsHelper -import com.yunqiinnovation.open_ai_service.OpenAIService +import com.yunqiinnovation.open_ai_service.OpenAIService +import com.yunqiinnovation.open_ai_service.SystemFunctionHandler /** @@ -19,7 +23,9 @@ class VoiceInteractionHandler( private val azureSpeechRegion: String, private val openaiApiKey: String, private val openaiBaseUrl: String = "", - private val openaiModel: String = "" + private val openaiModel: String = "", + private val volcanoSpeechAppId: String, + private val volcanoSpeechAppToken: String ) { private val TAG = "VoiceInteractionHandler" @@ -28,11 +34,7 @@ class VoiceInteractionHandler( private var azureTtsHelper: AzureTtsHelper? = null // OpenAI服务 - private val openAIService = OpenAIService() - - - // 语音功能处理 - private val voiceFunctionHandler = VoiceFunctionHandler(openAIService, context) + private val openAIService = OpenAIService(context.applicationContext) // 当前用户输入 private var currentUserInput = "" @@ -49,6 +51,17 @@ class VoiceInteractionHandler( // 回调 private var callback: InteractionCallback? = null + // 广播接收器 + private val exitInteractionReceiver = object : BroadcastReceiver() { + override fun onReceive(context: Context, intent: Intent) { + if (intent.action == SystemFunctionHandler.ACTION_EXIT_INTERACTION) { + Log.d(TAG, "收到退出交互广播") + stopRecognition() + + } + } + } + /** * 初始化 */ @@ -69,8 +82,19 @@ class VoiceInteractionHandler( // 初始化OpenAI服务 openAIService.initialize(openaiApiKey, openaiBaseUrl, openaiModel) - // 初始化语音功能处理器 - voiceFunctionHandler.initialize() + // 注册广播接收器 + try { + Log.d(TAG, "注册退出交互广播接收器，包名=${context.packageName}, action=${SystemFunctionHandler.ACTION_EXIT_INTERACTION}") + context.registerReceiver( + exitInteractionReceiver, + IntentFilter(SystemFunctionHandler.ACTION_EXIT_INTERACTION), + Context.RECEIVER_NOT_EXPORTED + ) + Log.d(TAG, "退出交互广播接收器注册成功") + } catch (e: Exception) { + // 广播注册失败不应该影响整个应用初始化 + Log.e(TAG, "注册退出交互广播接收器失败: ${e.message}", e) + } isInitialized = true return true @@ -243,64 +267,9 @@ class VoiceInteractionHandler( } override fun onFunctionCall(call: JSONObject) { - FileLogger.d(TAG, "收到函数调用请求: ${call.getString("name")}") + FileLogger.d(TAG, "processWithOpenAI 收到函数调用请求: ${call.getString("name")}") - // 使用函数处理器处理函数调用 - val handled = voiceFunctionHandler.handleFunctionCall( - functionCall = call, - messages = messages, - callback = object : VoiceFunctionHandler.FunctionCallCallback { - override fun onTokenReceived(token: String) { - responseBuilder.append(token) - } - - override fun onComplete() { - val response = responseBuilder.toString() - if (response.isNotEmpty()) { - // 播放AI回复 - Log.d(TAG, "AI Function Call 回复: $response") - speakAIResponse(response) - - // 同步聊天记录到Flutter端 - sendChatHistoryUpdate("personal_assistant", text, response) - } - notifyStateChanged() - } - - override fun onError(message: String) { - FileLogger.e(TAG, "函数处理出错: $message") - callback?.onError(message) - } - - override fun onFunctionCall(nestedCall: JSONObject) { - FileLogger.d(TAG, "收到嵌套函数调用: ${nestedCall.getString("name")}") - // 递归处理嵌套函数调用 - voiceFunctionHandler.handleFunctionCall( - functionCall = nestedCall, - messages = messages, - callback = this - ) - } - - override fun onExitWithMessage(farewell: String) { - // 播放退出消息 - speakAIResponse(farewell) - - // 同步聊天记录 - sendChatHistoryUpdate("personal_assistant", text, farewell) - - // 停止语音识别 - stopRecognition() - } - } - ) - if (!handled) { - // 如果函数没有被处理，作为普通文本处理 - FileLogger.d(TAG, "函数未处理，作为普通文本处理") - speakAIResponse("我无法处理这个请求") - sendChatHistoryUpdate("personal_assistant", text, "我无法处理这个请求") - } } } ) @@ -375,6 +344,13 @@ class VoiceInteractionHandler( // 停止TTS播放 stopTts() + // 注销广播接收器 + try { + context.unregisterReceiver(exitInteractionReceiver) + } catch (e: Exception) { + FileLogger.e(TAG, "注销广播接收器失败: ${e.message}", e) + } + // 释放Azure资源 azureAsrHelper?.let { FileLogger.d(TAG, "关闭Azure ASR服务") @@ -414,6 +390,7 @@ class VoiceInteractionHandler( putExtra("userMessage", userMessage) putExtra("assistantMessage", assistantMessage) putExtra("timestamp", System.currentTimeMillis()) + setPackage(context.packageName) } // 发送广播 diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt index 1cb132e1a..dbe64c79e 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt @@ -28,7 +28,7 @@ import android.media.AudioAttributes import android.net.Uri import com.yunqiinnovation.deepsound.core.utils.FileLogger import com.yunqiinnovation.azure_speech.AzureAsrHelper -import com.yunqiinnovation.azure_speech.AzureTtsHelper +import com.yunqiinnovation.volcano_speech.VolcanoTtsHelper import com.yunqiinnovation.open_ai_service.OpenAIService @@ -145,6 +145,8 @@ class VoiceInteractionService : Service() { var openaiKey = MainActivity.openaiApiKey var openaiBaseUrl = MainActivity.openaiBaseUrl ?: "" // OpenAI API基本URL var openaiModel = MainActivity.openaiModel ?: "" // OpenAI模型 + var volcanoSpeechAppId = MainActivity.volcanoSpeechAppId ?: "" + var volcanoSpeechAppToken = MainActivity.volcanoSpeechAppToken ?: "" // 如果静态变量中没有配置，尝试从加密存储中加载 if (subscriptionKey.isEmpty() || serviceRegion.isEmpty() || openaiKey.isEmpty()) { @@ -160,6 +162,8 @@ class VoiceInteractionService : Service() { openaiKey = MainActivity.openaiApiKey openaiBaseUrl = MainActivity.openaiBaseUrl ?: "" openaiModel = MainActivity.openaiModel ?: "" + volcanoSpeechAppId = MainActivity.volcanoSpeechAppId ?: "" + volcanoSpeechAppToken = MainActivity.volcanoSpeechAppToken ?: "" FileLogger.d(TAG, "已从加密存储加载配置信息") } else { @@ -168,7 +172,10 @@ class VoiceInteractionService : Service() { } // 初始化语音交互处理器 - voiceInteractionHandler = VoiceInteractionHandler(applicationContext, subscriptionKey, serviceRegion, openaiKey, openaiBaseUrl, openaiModel) + voiceInteractionHandler = VoiceInteractionHandler(applicationContext, + subscriptionKey, serviceRegion, + openaiKey, openaiBaseUrl, openaiModel, + volcanoSpeechAppId, volcanoSpeechAppToken) // 初始化回调 voiceInteractionHandler.setCallback(object : VoiceInteractionHandler.InteractionCallback { @@ -299,7 +306,7 @@ class VoiceInteractionService : Service() { if (voiceInteractionHandler.isRecognitionActive) { val currentTime = System.currentTimeMillis() val elapsedTime = currentTime - lastActivityTime - FileLogger.d(TAG, "hasSpeechDetected=${voiceInteractionHandler.hasSpeechDetected}, isTtsSpeaking=${voiceInteractionHandler.isTtsSpeaking}, elapsedTime=${elapsedTime}") + // FileLogger.d(TAG, "hasSpeechDetected=${voiceInteractionHandler.hasSpeechDetected}, isTtsSpeaking=${voiceInteractionHandler.isTtsSpeaking}, elapsedTime=${elapsedTime}") // 如果超过指定时间没有检测到语音，且不在TTS播放中，暂停语音识别 if (!voiceInteractionHandler.hasSpeechDetected && !voiceInteractionHandler.isTtsSpeaking && @@ -627,6 +634,7 @@ class VoiceInteractionService : Service() { val intent = Intent(ACTION_RECOGNITION_STARTED).apply { // 可以添加额外数据 putExtra("timestamp", System.currentTimeMillis()) + putExtra("packageName", applicationContext.packageName) } // 发送广播 diff --git a/android/build.gradle.kts b/android/build.gradle.kts index 67f214248..5421f73bd 100644 --- a/android/build.gradle.kts +++ b/android/build.gradle.kts @@ -25,3 +25,14 @@ subprojects { tasks.register("clean") { delete(rootProject.layout.buildDirectory) } + +buildscript { + repositories { + google() + mavenCentral() + } + dependencies { + classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:2.1.10") + // 其他依赖... + } +} diff --git a/android/settings.gradle.kts b/android/settings.gradle.kts index c0eebbf43..e332da5f9 100644 --- a/android/settings.gradle.kts +++ b/android/settings.gradle.kts @@ -25,7 +25,7 @@ pluginManagement { plugins { id("dev.flutter.flutter-plugin-loader") id("com.android.application") version "8.7.0" apply false - id("org.jetbrains.kotlin.android") version "1.8.22" apply false + id("org.jetbrains.kotlin.android") version "2.1.10" apply false } include(":app") From 1fc52ca5624ec5a12a9524230f4c6a98e0030768 Mon Sep 17 00:00:00 2001 From: tanlongsheng <252620078@qq.com> Date: Tue, 8 Apr 2025 09:25:12 +0800 Subject: [PATCH 06/14] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=9C=B0=E5=9B=BE?= =?UTF-8?q?=E5=8D=A1=E7=89=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/modules/chat/views/chat_navigation_card.dart | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/modules/chat/views/chat_navigation_card.dart b/lib/modules/chat/views/chat_navigation_card.dart index 9ea658771..cb04f6c22 100644 --- a/lib/modules/chat/views/chat_navigation_card.dart +++ b/lib/modules/chat/views/chat_navigation_card.dart @@ -20,10 +20,12 @@ class _ChatNavigationCardState extends State { LatLng startPoint = LatLng(116.397428, 39.90923); // 起点坐标 LatLng endPoint = LatLng(116.397026, 39.903303); // 终点坐标 + late Future _mapUrl; + @override void initState() { super.initState(); - _getMapUrl(); + _mapUrl = _getMapUrl(); } Future _getMapUrl() async { @@ -158,7 +160,7 @@ class _ChatNavigationCardState extends State { return Column( children: [ FutureBuilder( - future: _getMapUrl(), + future: _mapUrl, builder: (BuildContext context, AsyncSnapshot snapshot) { if (snapshot.hasData) { return Image.network(snapshot.data); From bb6a3a74fb15754b96ee8345d44f38076ae52db2 Mon Sep 17 00:00:00 2001 From: wolfplus Date: Thu, 10 Apr 2025 19:13:53 +0100 Subject: [PATCH 07/14] add --- .cursorrules | 65 ++- android/app/build.gradle.kts | 1 + .../VoiceInteractionHandler copy.kt.bak | 456 ---------------- android/build.gradle.kts | 1 + android/settings.gradle.kts | 4 + lib/core/bindings/initial_binding.dart | 5 +- lib/core/utils/jieli_constants.dart | 89 ++++ lib/data/services/function_service.dart | 97 ---- .../services/open_ai_service_adapter.dart | 21 +- .../services/voice_interaction_service.dart | 14 +- .../bindings/jieli_test_binding.dart | 11 + .../controllers/jieli_test_controller.dart | 298 +++++++++++ .../jieli_test/views/jieli_test_view.dart | 494 +++++++++++++++++ .../controllers/pairing_controller.dart | 4 +- lib/modules/settings/views/settings_view.dart | 22 + lib/routes/app_pages.dart | 7 + lib/routes/app_routes.dart | 1 + .../azure_speech/android/build.gradle.kts | 42 +- local_plugins/chat_storage_sqlite/README.md | 80 +++ .../android/build.gradle.kts | 60 +++ .../android/settings.gradle.kts | 1 + .../android/src/main/AndroidManifest.xml | 8 + .../ChatStorageSqlitePlugin.kt | 236 +++++++++ .../lib/chat_storage_sqlite.dart | 155 ++++++ .../chat_storage_sqlite/pubspec.yaml | 30 ++ local_plugins/jieli/README.md | 144 +++++ local_plugins/jieli/android/build.gradle.kts | 55 ++ .../jieli/android/settings.gradle.kts | 1 + .../android/src/main/AndroidManifest.xml | 22 + .../com/yunqiinnovation/jieli/BleService.kt | 460 ++++++++++++++++ .../com/yunqiinnovation/jieli/JieliPlugin.kt | 269 ++++++++++ local_plugins/jieli/lib/jieli.dart | 188 +++++++ local_plugins/jieli/pubspec.yaml | 23 + .../android/AndroidManifest.xml | 12 + .../open_ai_service/android/build.gradle.kts | 24 +- .../open_ai_service/MCPClient.kt | 496 ++++++++++++++++++ .../open_ai_service/OpenAIService.kt | 228 ++++++-- .../open_ai_service/OpenAIServicePlugin.kt | 65 ++- .../open_ai_service/SystemFunctionHandler.kt | 422 +++++++++++++++ .../open_ai_service/lib/open_ai_service.dart | 101 ++++ .../volcano_speech/android/build.gradle.kts | 4 +- .../volcano_speech/VolcanoAsrHelper.kt | 39 +- .../volcano_speech/VolcanoTtsHelper.kt | 57 +- pubspec.yaml | 5 + test.json | 58 +- 45 files changed, 4207 insertions(+), 668 deletions(-) delete mode 100644 android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler copy.kt.bak create mode 100644 lib/core/utils/jieli_constants.dart delete mode 100644 lib/data/services/function_service.dart create mode 100644 lib/modules/jieli_test/bindings/jieli_test_binding.dart create mode 100644 lib/modules/jieli_test/controllers/jieli_test_controller.dart create mode 100644 lib/modules/jieli_test/views/jieli_test_view.dart create mode 100644 local_plugins/chat_storage_sqlite/README.md create mode 100644 local_plugins/chat_storage_sqlite/android/build.gradle.kts create mode 100644 local_plugins/chat_storage_sqlite/android/settings.gradle.kts create mode 100644 local_plugins/chat_storage_sqlite/android/src/main/AndroidManifest.xml create mode 100644 local_plugins/chat_storage_sqlite/android/src/main/kotlin/com/yunqiinnovation/chat_storage_sqlite/ChatStorageSqlitePlugin.kt create mode 100644 local_plugins/chat_storage_sqlite/lib/chat_storage_sqlite.dart create mode 100644 local_plugins/chat_storage_sqlite/pubspec.yaml create mode 100644 local_plugins/jieli/README.md create mode 100644 local_plugins/jieli/android/build.gradle.kts create mode 100644 local_plugins/jieli/android/settings.gradle.kts create mode 100644 local_plugins/jieli/android/src/main/AndroidManifest.xml create mode 100644 local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/BleService.kt create mode 100644 local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt create mode 100644 local_plugins/jieli/lib/jieli.dart create mode 100644 local_plugins/jieli/pubspec.yaml create mode 100644 local_plugins/open_ai_service/android/AndroidManifest.xml create mode 100644 local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt create mode 100644 local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/SystemFunctionHandler.kt diff --git a/.cursorrules b/.cursorrules index 99d6aa58c..6e3d3fd0c 100644 --- a/.cursorrules +++ b/.cursorrules @@ -7,4 +7,67 @@ 7. 尽可能避免打印日志,仅在出错时记录日志 8. 请使用较小的代码量改动，最安全的方法进行代码编写和优化 9. 请尽可能使用中文回复问题 -10. 重要：不要过度设计！！ \ No newline at end of file +10. 重要：不要过度设计！！ +11. 本项目使用 Kotlin DSL 进行开发 + +# 项目架构概述 +DeepSound是一个基于Flutter和GetX构建的多平台语音应用，具有语音合成、语音识别、语音聊天等功能。 + +## 总体架构 +- 采用GetX状态管理框架 +- 遵循模块化设计，各功能模块相对独立 +- 使用依赖注入实现组件解耦 +- 多语言支持，使用GetX国际化方案 + +## 主要目录结构 +- lib/core: 核心组件、工具和通用功能 +- lib/data: 数据模型、服务和提供者 +- lib/modules: 功能模块，按业务划分 +- lib/routes: 路由定义和管理 +- local_plugins: 本地插件，提供原生功能支持 + +# 核心模块说明 +## 语音处理 +- 火山语音服务: 提供语音识别和合成核心能力 +- Azure语音服务: 提供备选的语音识别和合成能力 +- 离线TTS支持: 在线语音合成失败时的备选方案 + +## 聊天功能 +- 基于OpenAI的对话功能 +- 支持语音输入、文字输入和语音输出 +- 支持多种角色和语音类型 + +## 用户系统 +- 支持多种登录方式：Firebase、Google、Facebook +- 用户资料管理和设置 +- 主题切换和语言设置 + +## 蓝牙配对 +- 支持蓝牙设备发现和连接 +- 提供简化的配对流程 +- 设备管理和状态监控 + +## 音频处理 +- 音频录制和播放 +- 支持波形显示 +- 音量控制和音频质量设置 + +# 技术栈 +- Flutter: UI框架 +- GetX: 状态管理和依赖注入 +- Firebase: 用户认证和后端服务 +- WebSocket: 实时通信 +- SQLite: 本地数据存储 +- 原生插件: 语音识别、合成和蓝牙功能 + +# 插件说明 +- jieli: 杰理蓝牙设备连接和控制插件 +- volcano_speech: 火山语音服务插件，提供语音识别和合成 +- azure_speech: 微软Azure语音服务插件 +- open_ai_service: OpenAI服务插件，提供AI对话能力 +- speech_to_text: 通用语音识别插件 + +# 环境配置 +- 环境变量通过.env文件管理 +- 支持不同环境的配置切换 +- 敏感信息不应直接硬编码在代码中 \ No newline at end of file diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 8c371408e..009a8926c 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -108,6 +108,7 @@ dependencies { implementation(project(":azure_speech")) implementation(project(":open_ai_service")) implementation(project(":volcano_speech")) + implementation(project(":chat_storage_sqlite")) // 添加新创建的SQLite聊天存储插件 // 添加OkHttp依赖 implementation("com.squareup.okhttp3:okhttp:4.9.3") diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler copy.kt.bak b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler copy.kt.bak deleted file mode 100644 index 1a9e2df13..000000000 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler copy.kt.bak +++ /dev/null @@ -1,456 +0,0 @@ -package com.yunqiinnovation.deepsound - -import android.content.Context -import org.json.JSONArray -import org.json.JSONObject -import android.util.Log -import com.yunqiinnovation.deepsound.core.utils.FileLogger -import com.yunqiinnovation.azure_speech.AzureAsrHelper -import com.yunqiinnovation.volcano_speech.VolcanoTtsHelper -import com.yunqiinnovation.open_ai_service.OpenAIService - - -/** - * 语音交互处理器 - 处理语音识别、TTS和AI对话相关逻辑 - */ -class VoiceInteractionHandler( - private val context: Context, - private val azureSpeechKey: String, - private val azureSpeechRegion: String, - private val openaiApiKey: String, - private val openaiBaseUrl: String = "", - private val openaiModel: String = "", - private val volcanoSpeechAppId: String, - private val volcanoSpeechAppToken: String -) { - private val TAG = "VoiceInteractionHandler" - - // Azure服务 - private var azureAsrHelper: AzureAsrHelper? = null - private var volcanoTtsHelper: VolcanoTtsHelper? = null - - // OpenAI服务 - private val openAIService = OpenAIService() - - - // 语音功能处理 - private val voiceFunctionHandler = VoiceFunctionHandler(openAIService, context) - - // 当前用户输入 - private var currentUserInput = "" - - // 状态 - private var isInitialized = false - var isRecognitionActive = false - private set - var isTtsSpeaking = false - private set - var hasSpeechDetected = false - private set - - // 回调 - private var callback: InteractionCallback? = null - - /** - * 初始化 - */ - fun initialize(): Boolean { - if (isInitialized) return true - - try { - // 初始化Azure ASR - azureAsrHelper = AzureAsrHelper(context).apply { - initialize(azureSpeechKey, azureSpeechRegion) - } - - // 初始化Volcano TTS (使用大模型TTS) - volcanoTtsHelper = VolcanoTtsHelper(context).apply { - // 这里需要替换为实际的Volcano SDK初始化参数 - // 暂时使用假参数，实际使用时需要替换为真实值 - initialize(volcanoSpeechAppId, volcanoSpeechAppToken, "volc.bigasr.sauc.duration") - - } - - // 初始化OpenAI服务 - openAIService.initialize(openaiApiKey, openaiBaseUrl, openaiModel) - - // 初始化语音功能处理器 - voiceFunctionHandler.initialize() - - isInitialized = true - return true - } catch (e: Exception) { - FileLogger.e(TAG, "初始化失败: ${e.message}", e) - return false - } - } - - /** - * 设置回调 - */ - fun setCallback(callback: InteractionCallback) { - this.callback = callback - } - - /** - * 开始语音识别 - */ - fun startRecognition() { - if (isRecognitionActive) return - - // 检查录音权限 - if (!checkRecordAudioPermission()) { - callback?.onError("需要录音权限，请在设置中授予权限") - return - } - - isRecognitionActive = true - hasSpeechDetected = false - notifyStateChanged() - - try { - azureAsrHelper?.startContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { - override fun onRecognizing(recognizing: String, detectedLanguage: String) { - if (recognizing.isNotEmpty()) { - hasSpeechDetected = true - stopTts() - notifyStateChanged() - } - } - - override fun onResult(result: String, detectedLanguage: String) { - if (result.isNotEmpty()) { - notifyStateChanged() - - processWithOpenAI(result) - - } - - // 重置状态，继续识别 - hasSpeechDetected = false - } - - override fun onSessionStarted() { - notifyStateChanged() - } - - override fun onSessionStopped() { - isRecognitionActive = false - notifyStateChanged() - } - - override fun onCanceled(reason: String, errorDetails: String) { - isRecognitionActive = false - notifyStateChanged() - } - - override fun onError(error: String) { - isRecognitionActive = false - callback?.onError("语音识别出错") - notifyStateChanged() - } - - override fun onSuccess(message: String) { - // 处理成功事件 - } - }) - } catch (e: Exception) { - isRecognitionActive = false - FileLogger.e(TAG, "启动语音识别失败: ${e.message}", e) - callback?.onError("启动语音识别失败") - notifyStateChanged() - } - } - - /** - * 停止语音识别 - */ - fun stopRecognition() { - if (!isRecognitionActive) return - - FileLogger.d(TAG, "停止语音识别") - - try { - azureAsrHelper?.stopContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { - override fun onResult(result: String, detectedLanguage: String) {} - override fun onRecognizing(recognizing: String, detectedLanguage: String) {} - override fun onSessionStarted() {} - override fun onSessionStopped() { - isRecognitionActive = false - FileLogger.d(TAG, "语音识别会话已停止") - notifyStateChanged() - } - override fun onCanceled(reason: String, errorDetails: String) { - isRecognitionActive = false - FileLogger.d(TAG, "语音识别已取消: $reason") - notifyStateChanged() - } - override fun onError(error: String) { - isRecognitionActive = false - FileLogger.e(TAG, "停止语音识别时出错: $error") - notifyStateChanged() - } - override fun onSuccess(message: String) { - isRecognitionActive = false - FileLogger.d(TAG, "语音识别已停止: $message") - notifyStateChanged() - } - }) - } catch (e: Exception) { - FileLogger.e(TAG, "停止语音识别异常: ${e.message}", e) - // 确保状态一致性 - isRecognitionActive = false - notifyStateChanged() - } - } - - /** - * 使用OpenAI处理语音识别结果 - */ - private fun processWithOpenAI(text: String) { - // 保存当前用户输入，用于后续同步聊天记录 - currentUserInput = text - - Thread { - try { - val messages = JSONArray().apply { - put(openAIService.createUserMessage(text)) - } - - // 创建响应构建器 - val responseBuilder = StringBuilder() - - openAIService.sendMessageStream( - messages = messages, - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - // 累加响应内容 - responseBuilder.append(token) - } - - override fun onComplete() { - // 处理完整响应 - val response = responseBuilder.toString() - if (response.isNotEmpty()) { - // 播放AI回复 - Log.d(TAG, "AI 回复: $response") - - speakAIResponse(response) - - // 同步聊天记录到Flutter端 - sendChatHistoryUpdate("personal_assistant", text, response) - } - } - - override fun onError(e: Exception) { - FileLogger.e(TAG, "AI流式处理出错: ${e.message}", e) - callback?.onError("AI处理出错") - } - - override fun onFunctionCall(call: JSONObject) { - FileLogger.d(TAG, "收到函数调用请求: ${call.getString("name")}") - - // 使用函数处理器处理函数调用 - val handled = voiceFunctionHandler.handleFunctionCall( - functionCall = call, - messages = messages, - callback = object : VoiceFunctionHandler.FunctionCallCallback { - override fun onTokenReceived(token: String) { - responseBuilder.append(token) - } - - override fun onComplete() { - val response = responseBuilder.toString() - if (response.isNotEmpty()) { - // 播放AI回复 - Log.d(TAG, "AI Function Call 回复: $response") - speakAIResponse(response) - - // 同步聊天记录到Flutter端 - sendChatHistoryUpdate("personal_assistant", text, response) - } - notifyStateChanged() - } - - override fun onError(message: String) { - FileLogger.e(TAG, "函数处理出错: $message") - callback?.onError(message) - } - - override fun onFunctionCall(nestedCall: JSONObject) { - FileLogger.d(TAG, "收到嵌套函数调用: ${nestedCall.getString("name")}") - // 不处理嵌套函数调用，直接返回错误提示 - // speakAIResponse("抱歉，暂不支持嵌套函数调用") - } - - override fun onExitWithMessage(farewell: String) { - // 播放退出消息 - speakAIResponse(farewell) - - // 同步聊天记录 - sendChatHistoryUpdate("personal_assistant", text, farewell) - - // 停止语音识别 - stopRecognition() - } - } - ) - - if (!handled) { - // 如果函数没有被处理，作为普通文本处理 - FileLogger.d(TAG, "函数未处理，作为普通文本处理") - speakAIResponse("我无法处理这个请求") - sendChatHistoryUpdate("personal_assistant", text, "我无法处理这个请求") - } - } - } - ) - - } catch (e: Exception) { - FileLogger.e(TAG, "AI处理出错: ${e.message}", e) - callback?.onError("AI处理出错") - } - }.start() - } - - /** - * 播放TTS - */ - fun playTts(text: String, callback: TtsCallback? = null) { - isTtsSpeaking = true - notifyStateChanged() - - volcanoTtsHelper?.speak(text, object : VolcanoTtsHelper.TTSCallback { - override fun onStart(reqId: String) { - // TTS开始播放事件 - } - - override fun onProgress(reqId: String, progress: Double) { - // TTS播放进度事件 - } - - override fun onComplete(reqId: String) { - isTtsSpeaking = false - notifyStateChanged() - callback?.onComplete() - } - - override fun onError(reqId: String, errorCode: Int, errorMsg: String) { - isTtsSpeaking = false - notifyStateChanged() - callback?.onError(errorMsg) - } - }) - } - - /** - * 停止TTS播放 - */ - fun stopTts() { - if (isTtsSpeaking) { - volcanoTtsHelper?.stop() - isTtsSpeaking = false - notifyStateChanged() - } - } - - /** - * 播放AI回复 - */ - private fun speakAIResponse(text: String) { - isTtsSpeaking = true - notifyStateChanged() - - volcanoTtsHelper?.speak(text, object : VolcanoTtsHelper.TTSCallback { - override fun onStart(reqId: String) { - // TTS开始播放事件 - } - - override fun onProgress(reqId: String, progress: Double) { - // TTS播放进度事件 - } - - override fun onComplete(reqId: String) { - isTtsSpeaking = false - notifyStateChanged() - } - - override fun onError(reqId: String, errorCode: Int, errorMsg: String) { - isTtsSpeaking = false - notifyStateChanged() - } - }) - } - - /** - * 释放资源 - */ - fun dispose() { - // 停止语音识别 - stopRecognition() - - // 停止TTS播放 - stopTts() - - // 释放Azure资源 - azureAsrHelper?.let { - FileLogger.d(TAG, "关闭Azure ASR服务") - it.dispose() - } - - volcanoTtsHelper?.let { - FileLogger.d(TAG, "关闭Volcano TTS服务") - it.release() - } - - FileLogger.d(TAG, "语音交互处理器资源已释放") - } - - /** - * 检查录音权限 - */ - private fun checkRecordAudioPermission(): Boolean { - val permission = android.Manifest.permission.RECORD_AUDIO - val result = context.checkCallingOrSelfPermission(permission) - return result == android.content.pm.PackageManager.PERMISSION_GRANTED - } - - /** - * 通知状态变化 - */ - private fun notifyStateChanged() { - callback?.onStateChanged(isRecognitionActive, isTtsSpeaking, hasSpeechDetected) - } - - /** - * 发送聊天历史更新 - */ - private fun sendChatHistoryUpdate(agentId: String, userMessage: String, assistantMessage: String) { - val intent = android.content.Intent(VoiceInteractionService.ACTION_CHAT_HISTORY_UPDATED).apply { - putExtra("agentId", agentId) - putExtra("userMessage", userMessage) - putExtra("assistantMessage", assistantMessage) - putExtra("timestamp", System.currentTimeMillis()) - } - - // 发送广播 - context.sendBroadcast(intent) - } - - /** - * 交互回调接口 - */ - interface InteractionCallback { - fun onStateChanged(isRecognitionActive: Boolean, isTtsSpeaking: Boolean, hasSpeechDetected: Boolean) - fun onError(message: String) - fun onPromptRequest(message: String) - } - - /** - * TTS回调接口 - */ - interface TtsCallback { - fun onComplete() - fun onError(error: String) - } -} \ No newline at end of file diff --git a/android/build.gradle.kts b/android/build.gradle.kts index 5421f73bd..ca0ce6220 100644 --- a/android/build.gradle.kts +++ b/android/build.gradle.kts @@ -1,5 +1,6 @@ allprojects { repositories { + mavenLocal() google() mavenCentral() maven { url = uri("https://storage.googleapis.com/download.flutter.io") } diff --git a/android/settings.gradle.kts b/android/settings.gradle.kts index e332da5f9..74e47be3f 100644 --- a/android/settings.gradle.kts +++ b/android/settings.gradle.kts @@ -16,6 +16,7 @@ pluginManagement { includeBuild("$flutterSdkPath/packages/flutter_tools/gradle") repositories { + mavenLocal() google() mavenCentral() gradlePluginPortal() @@ -25,6 +26,7 @@ pluginManagement { plugins { id("dev.flutter.flutter-plugin-loader") id("com.android.application") version "8.7.0" apply false + id("com.android.library") version "8.7.0" apply false id("org.jetbrains.kotlin.android") version "2.1.10" apply false } @@ -32,8 +34,10 @@ include(":app") include(":azure_speech") include(":open_ai_service") include(":volcano_speech") +include(":chat_storage_sqlite") // 设置azure_speech项目的路径 project(":azure_speech").projectDir = file("../local_plugins/azure_speech/android") project(":open_ai_service").projectDir = file("../local_plugins/open_ai_service/android") project(":volcano_speech").projectDir = file("../local_plugins/volcano_speech/android") +project(":chat_storage_sqlite").projectDir = file("../local_plugins/chat_storage_sqlite/android") diff --git a/lib/core/bindings/initial_binding.dart b/lib/core/bindings/initial_binding.dart index 748589f63..9a7a3abec 100644 --- a/lib/core/bindings/initial_binding.dart +++ b/lib/core/bindings/initial_binding.dart @@ -9,7 +9,6 @@ import '../../data/services/bluetooth_media_button_service.dart'; import '../../core/utils/logger.dart'; import '../../data/services/voice_interaction_service.dart'; import '../../data/services/open_ai_service_adapter.dart'; - /// 初始绑定，用于管理全局依赖 class InitialBinding extends Bindings { @override @@ -48,11 +47,15 @@ class InitialBinding extends Bindings { Get.lazyPut( () => BluetoothMediaButtonService(), fenix: true); + + // 语音交互服务 Get.lazyPut(() => VoiceInteractionService(), fenix: true); + // 杰理蓝牙服务 + // // 语言管理器（需要最先初始化） // Get.put(LoginService(), permanent: true); // await LoginService().FirebaseInit(); diff --git a/lib/core/utils/jieli_constants.dart b/lib/core/utils/jieli_constants.dart new file mode 100644 index 000000000..55f8cb802 --- /dev/null +++ b/lib/core/utils/jieli_constants.dart @@ -0,0 +1,89 @@ +/// 杰里(Jieli)设备相关常量定义 +/// 用于标准化系统属性类型和功能码 + +/// 系统属性类型常量 +class JieliAttrType { + // 公共属性 + static const int VOLUME = 1; // 音量 + static const int PLAYING_TIME = 2; // 播放时间 + static const int BATTERY_LEVEL = 3; // 电池电量 + static const int WORK_MODE = 4; // 工作模式 + static const int PLAYBACK_MODE = 5; // 播放模式(循环模式) + static const int EQ_MODE = 6; // EQ模式 + static const int LANGUAGE = 7; // 语言设置 + static const int CONTRAST = 8; // 对比度 + static const int LED_ONOFF = 9; // LED开关 + static const int POWER_OFF_TIME = 10; // 关机时间 + static const int LIGHT_ONOFF = 11; // 背光开关 + static const int BLETYPE_SETTING = 12; // BLE类型设置 + static const int TIME_OUT = 13; // 超时设置 + static const int DEVICE_NAME = 14; // 设备名称 + static const int ALARM_ENABLE = 15; // 闹钟开关 + static const int DEV_MAC = 16; // 设备MAC地址 + + // 音乐相关属性 + static const int MUSIC_STATUS = 20; // 音乐状态 + static const int MUSIC_CURR_DEVICE = 21; // 当前音乐设备 + static const int MUSIC_TOTAL_TIME = 22; // 音乐总时长 + static const int MUSIC_CURR_PLAY_TIME = 23; // 当前播放时间 + static const int MUSIC_NUM = 24; // 音乐数量 + static const int MUSIC_DEV_STATUS = 25; // 设备存储状态 + static const int MUSIC_BREAK_POINT = 26; // 断点信息 + static const int MUSIC_CURR_FILENUM = 27; // 当前文件编号 + static const int MUSIC_TOTAL_FILENUM = 28; // 总文件数 + static const int MEDIA_TIMESTAMP = 29; // 媒体时间戳 + + // 模式相关 + static const int CUR_MODE = 30; // 当前模式 + + // TWS相关 + static const int TWS_CONN_STATUS = 60; // TWS连接状态 + static const int LEFT_CHN_BAT = 61; // 左耳电量 + static const int RIGHT_CHN_BAT = 62; // 右耳电量 + static const int TWS_STATUS = 63; // TWS状态 +} + +/// 功能码 +class JieliFunctionCode { + static const int COMMON = 0; // 公共功能 + static const int MUSIC = 1; // 音乐功能 + static const int RADIO = 2; // 收音机功能 + static const int LINEIN = 3; // 线路输入功能 + static const int RTC = 4; // 时钟功能 + static const int BT = 5; // 蓝牙功能 + static const int PC = 6; // PC功能 + static const int RECORD = 7; // 录音功能 + static const int UDISK = 8; // U盘功能 + static const int TALK = 9; // 通话功能 + static const int PHOTO = 10; // 拍照功能 + static const int VIDEO = 11; // 视频功能 +} + +/// 音乐播放状态 +class JieliMusicStatus { + static const int STOP = 0; // 停止状态 + static const int PLAY = 1; // 播放状态 + static const int PAUSE = 2; // 暂停状态 + static const int FAST_FORWARD = 3; // 快进状态 + static const int FAST_BACKWARD = 4; // 快退状态 +} + +/// 循环播放模式 +class JieliPlaybackMode { + static const int ALL = 0; // 全部循环 + static const int FOLDER = 1; // 文件夹循环 + static const int ONE = 2; // 单曲循环 + static const int RANDOM = 3; // 随机播放 + static const int BROWSE = 4; // 浏览播放 +} + +/// EQ模式 +class JieliEQMode { + static const int NORMAL = 0; // 正常 + static const int ROCK = 1; // 摇滚 + static const int POP = 2; // 流行 + static const int CLASSIC = 3; // 经典 + static const int JAZZ = 4; // 爵士 + static const int COUNTRY = 5; // 乡村 + static const int CUSTOM = 6; // 自定义 +} \ No newline at end of file diff --git a/lib/data/services/function_service.dart b/lib/data/services/function_service.dart deleted file mode 100644 index b1e3a1e4d..000000000 --- a/lib/data/services/function_service.dart +++ /dev/null @@ -1,97 +0,0 @@ -import 'dart:convert'; - -/// 函数服务 -/// 仅实现各种功能函数，不包含AI交互逻辑 -class FunctionService { - /// 天气查询工具定义 - /// 返回一个可直接用于tools参数的天气查询工具定义 - static Map getWeatherToolDefinition() { - return { - 'type': 'function', - 'function': { - 'name': 'get_current_weather', - 'description': '获取指定地点的当前天气信息', - 'parameters': { - 'type': 'object', - 'properties': { - 'location': { - 'type': 'string', - 'description': '地理位置，例如：北京市', - }, - 'unit': { - 'type': 'string', - 'description': '温度单位', - 'enum': ['celsius', 'fahrenheit'], - }, - }, - 'required': ['location'], - }, - }, - }; - } - - /// 模拟天气API实现 - /// 返回一个模拟的天气信息，实际应用中应该调用真实的天气API - static Future> getWeatherInfo(Map args) async { - final location = args['location'] ?? '未知地点'; - final unit = args['unit'] ?? 'celsius'; - - // 在实际应用中，这里应该调用真实的天气API - return { - 'location': location, - 'temperature': unit == 'celsius' ? '25°C' : '77°F', - 'conditions': '晴转多云', - 'humidity': '55%', - 'wind': '西南风 3级', - 'date': DateTime.now().toString().split(' ')[0], - }; - } - - /// 解析函数调用参数 - /// 从AI响应中提取函数调用参数 - static Map parseFunctionCallArguments(Map functionCallResponse) { - if (functionCallResponse['tool_calls'] == null || - functionCallResponse['tool_calls'].isEmpty) { - throw FunctionServiceException('No tool calls found in response'); - } - - final toolCall = functionCallResponse['tool_calls'][0]; - if (toolCall['function'] == null || toolCall['function']['arguments'] == null) { - throw FunctionServiceException('Invalid function call format'); - } - - try { - return json.decode(toolCall['function']['arguments']); - } catch (e) { - throw FunctionServiceException('Failed to parse function arguments: $e'); - } - } - - /// 获取函数调用ID和名称 - /// 用于构建工具响应消息 - static Map getFunctionCallInfo(Map functionCallResponse) { - if (functionCallResponse['tool_calls'] == null || - functionCallResponse['tool_calls'].isEmpty) { - throw FunctionServiceException('No tool calls found in response'); - } - - final toolCall = functionCallResponse['tool_calls'][0]; - if (toolCall['id'] == null || toolCall['function'] == null || toolCall['function']['name'] == null) { - throw FunctionServiceException('Invalid function call format'); - } - - return { - 'id': toolCall['id'], - 'name': toolCall['function']['name'], - }; - } -} - -/// 函数调用服务异常类 -class FunctionServiceException implements Exception { - final String message; - FunctionServiceException(this.message); - - @override - String toString() => message; -} \ No newline at end of file diff --git a/lib/data/services/open_ai_service_adapter.dart b/lib/data/services/open_ai_service_adapter.dart index 2c0da53ac..7eb337605 100644 --- a/lib/data/services/open_ai_service_adapter.dart +++ b/lib/data/services/open_ai_service_adapter.dart @@ -41,25 +41,14 @@ class OpenAIServiceAdapter implements AiService { _isProcessingStream = false; break; case OpenAIEventType.error: - if (event.content is String) { - _tokenStreamController.addError(event.content as String); - } else { - _tokenStreamController.addError('未知错误: ${event.content}'); - } + _tokenStreamController.addError('未知错误: ${event.content}'); + _isProcessingStream = false; break; case OpenAIEventType.functionCall: - try { - if (event.content is Map) { - _tokenStreamController.addError('收到函数调用，该流仅支持文本响应'); - } else { - _tokenStreamController.addError('收到未知格式的函数调用'); - printError(info: '函数调用格式错误: ${event.content}'); - } - } catch (e) { - printError(info: '处理函数调用事件出错: $e'); - _tokenStreamController.addError('处理函数调用失败: $e'); - } + _tokenStreamController.addError('收到函数调用，该流仅支持文本响应'); + + _isProcessingStream = false; break; } diff --git a/lib/data/services/voice_interaction_service.dart b/lib/data/services/voice_interaction_service.dart index cce506a58..57e8636c5 100644 --- a/lib/data/services/voice_interaction_service.dart +++ b/lib/data/services/voice_interaction_service.dart @@ -42,6 +42,8 @@ class VoiceInteractionService extends GetxService { late String _openaiApiKey; late String _openaiBaseUrl; late String _openaiModel; + late String _volcanoSpeechAppId; + late String _volcanoSpeechAppToken; // 聊天历史服务 late final ChatHistoryService _chatHistoryService; @@ -62,6 +64,8 @@ class VoiceInteractionService extends GetxService { _openaiApiKey = dotenv.env['OPENAI_API_KEY'] ?? ''; _openaiBaseUrl = dotenv.env['OPENAI_BASE_URL'] ?? ''; _openaiModel = dotenv.env['OPENAI_MODEL'] ?? ''; + _volcanoSpeechAppId = dotenv.env['VOLCANO_SPEECH_APP_ID'] ?? ''; + _volcanoSpeechAppToken = dotenv.env['VOLCANO_SPEECH_APP_TOKEN'] ?? ''; if (_azureSpeechKey.isEmpty || _azureSpeechRegion.isEmpty) { Logger.warning('未找到 Azure 语音服务配置。请在 .env 文件中设置 AZURE_SPEECH_KEY 和 AZURE_SPEECH_REGION'); @@ -111,11 +115,7 @@ class VoiceInteractionService extends GetxService { Logger.info('收到进入翻译模式事件，正在导航到翻译界面'); _navigateToTranslation(); - final translationModeEvent = GenericVoiceInteractionEvent( - type: 'enter_translation_mode', - timestamp: DateTime.now().millisecondsSinceEpoch, - ); - _eventStreamController.add(translationModeEvent); + break; } } @@ -124,7 +124,7 @@ class VoiceInteractionService extends GetxService { void _navigateToTranslation() { try { // 使用GetX导航到翻译页面 - Get.toNamed(Routes.translation); + Get.offNamedUntil(Routes.translation, (route) => route.settings.name == Routes.home); Logger.info('已导航到翻译界面'); } catch (e) { Logger.error('导航到翻译界面失败: $e'); @@ -222,6 +222,8 @@ class VoiceInteractionService extends GetxService { 'openai_api_key': _openaiApiKey, 'openai_base_url': _openaiBaseUrl, 'openai_model': _openaiModel, + 'volcano_speech_app_id': _volcanoSpeechAppId, + 'volcano_speech_app_token': _volcanoSpeechAppToken, }) ?? false; if (result) { diff --git a/lib/modules/jieli_test/bindings/jieli_test_binding.dart b/lib/modules/jieli_test/bindings/jieli_test_binding.dart new file mode 100644 index 000000000..9df61ef94 --- /dev/null +++ b/lib/modules/jieli_test/bindings/jieli_test_binding.dart @@ -0,0 +1,11 @@ +import 'package:get/get.dart'; +import '../controllers/jieli_test_controller.dart'; + +class JieliTestBinding extends Bindings { + @override + void dependencies() { + Get.lazyPut( + () => JieliTestController(), + ); + } +} \ No newline at end of file diff --git a/lib/modules/jieli_test/controllers/jieli_test_controller.dart b/lib/modules/jieli_test/controllers/jieli_test_controller.dart new file mode 100644 index 000000000..8afa86cf4 --- /dev/null +++ b/lib/modules/jieli_test/controllers/jieli_test_controller.dart @@ -0,0 +1,298 @@ +import 'dart:async'; +import 'dart:typed_data'; +import 'package:get/get.dart'; +import 'package:logger/logger.dart'; +import 'package:jieli/jieli.dart'; + +/// 杰理测试控制器 +class JieliTestController extends GetxController { + static final Logger logger = Logger(); + + // Jieli插件实例 + late Jieli jieli; + + // 状态变量 + var isScanning = false.obs; + var deviceInfo = '未获取'.obs; + var connectStatus = '未连接'.obs; + var currentDevice = Rx(null); + var scannedDevices = RxList([]); + var wakeupReceived = false.obs; + var audioDataSize = 0.obs; + var receivedFrames = 0.obs; + var audioDataReceiving = false.obs; + + // 事件订阅 + StreamSubscription? _scanSubscription; + StreamSubscription? _connectionSubscription; + StreamSubscription? _wakeupSubscription; + StreamSubscription? _audioSubscription; + + @override + void onInit() { + super.onInit(); + logger.d('JieliTestController 初始化'); + _initializeJieli(); + } + + @override + void onClose() { + logger.d('JieliTestController 关闭'); + _cancelAllSubscriptions(); + _disposeJieli(); + super.onClose(); + } + + /// 取消所有订阅 + void _cancelAllSubscriptions() { + _scanSubscription?.cancel(); + _connectionSubscription?.cancel(); + _wakeupSubscription?.cancel(); + _audioSubscription?.cancel(); + } + + /// 初始化杰理SDK + void _initializeJieli() { + try { + logger.d('初始化Jieli插件'); + jieli = Jieli(); + _setupEventListeners(); + } catch (e) { + logger.e('初始化Jieli插件错误', error: e); + } + } + + /// 设置事件监听器 + void _setupEventListeners() { + // 监听扫描结果 + _scanSubscription = jieli.scanResults.listen((devices) { + // logger.d('扫描到设备: ${devices.length}个'); + scannedDevices.assignAll(devices); + }); + + // 监听连接状态 + _connectionSubscription = jieli.connectionState.listen((state) { + logger.d('连接状态变化: $state'); + switch (state) { + case ConnectionState.connected: + connectStatus.value = '已连接'; + break; + case ConnectionState.disconnected: + connectStatus.value = '已断开'; + currentDevice.value = null; + break; + case ConnectionState.connecting: + connectStatus.value = '连接中...'; + break; + case ConnectionState.disconnecting: + connectStatus.value = '断开中...'; + break; + case ConnectionState.error: + connectStatus.value = '连接错误'; + break; + } + }); + + // 监听唤醒信号 + _wakeupSubscription = jieli.wakeupStream.listen((wakeup) { + logger.d('收到唤醒信号'); + wakeupReceived.value = true; + Get.snackbar('收到唤醒信号', '耳机发送了唤醒信号'); + }); + + // 监听音频数据 + _audioSubscription = jieli.audioDataStream.listen((data) { + // 更新音频数据统计 + audioDataSize.value += data.length; + receivedFrames.value++; + audioDataReceiving.value = true; + + // 每100帧更新一次日志，避免日志过多 + if (receivedFrames.value % 100 == 0) { + logger.d('已接收音频数据: ${audioDataSize.value} 字节, ${receivedFrames.value} 帧'); + } + }); + } + + /// 检查蓝牙状态 + Future checkBluetoothStatus() async { + try { + final isEnabled = await jieli.isBluetoothEnabled(); + if (!isEnabled) { + logger.d('蓝牙未启用，请求启用蓝牙'); + return await jieli.requestBluetoothEnable(); + } + return true; + } catch (e) { + logger.e('检查蓝牙状态错误', error: e); + return false; + } + } + + /// 开始扫描设备 + Future startScan() async { + if (isScanning.value) return; + + final bluetoothReady = await checkBluetoothStatus(); + if (!bluetoothReady) { + Get.snackbar('蓝牙未启用', '请启用蓝牙后再试'); + return; + } + + try { + logger.d('开始扫描设备'); + isScanning.value = true; + + // 清空之前的设备列表 + scannedDevices.clear(); + + // 开始扫描，10秒后自动停止 + final success = await jieli.startScan(scanTimeout: 10000); + + if (!success) { + logger.e('启动扫描失败'); + isScanning.value = false; + Get.snackbar('扫描失败', '启动扫描设备失败，请检查蓝牙权限'); + } + + // 10秒后自动更新UI状态 + Future.delayed(Duration(milliseconds: 10000), () { + isScanning.value = false; + }); + + } catch (e) { + logger.e('扫描设备错误', error: e); + isScanning.value = false; + Get.snackbar('扫描错误', e.toString()); + } + } + + /// 停止扫描 + Future stopScan() async { + if (!isScanning.value) return; + + try { + logger.d('停止扫描'); + await jieli.stopScan(); + isScanning.value = false; + } catch (e) { + logger.e('停止扫描错误', error: e); + } + } + + /// 连接到指定设备 + Future connectToDevice(BleDevice device) async { + try { + // 先断开当前连接 + if (currentDevice.value != null) { + await disconnectDevice(); + } + + // 更新UI状态 + connectStatus.value = '连接中...'; + logger.d('连接设备: ${device.name} (${device.id})'); + + // 重置音频和唤醒相关的状态 + wakeupReceived.value = false; + audioDataSize.value = 0; + receivedFrames.value = 0; + audioDataReceiving.value = false; + + // 连接设备 + final success = await jieli.connect(device.id); + + if (success) { + // 连接成功，更新当前设备 + currentDevice.value = device; + deviceInfo.value = '设备名称: ${device.name}\n设备ID: ${device.id}\nRSSI: ${device.rssi} dBm'; + logger.d('设备连接成功'); + } else { + deviceInfo.value = '连接失败'; + logger.d('设备连接失败'); + } + } catch (e) { + deviceInfo.value = '连接错误: ${e.toString()}'; + logger.e('连接设备错误', error: e); + } + } + + /// 断开设备连接 + Future disconnectDevice() async { + if (currentDevice.value == null) return; + + try { + connectStatus.value = '断开中...'; + logger.d('断开设备连接'); + + final success = await jieli.disconnect(); + + if (success) { + deviceInfo.value = '未连接设备'; + logger.d('设备断开连接成功'); + } else { + logger.d('设备断开连接失败'); + } + + } catch (e) { + logger.e('断开设备错误', error: e); + } + } + + /// 发送开始录音命令 + Future sendStartRecordingCommand() async { + if (currentDevice.value == null) { + Get.snackbar('操作失败', '未连接设备'); + return; + } + + try { + logger.d('发送开始录音命令'); + final success = await jieli.sendStartRecordingCommand(); + + if (success) { + Get.snackbar('命令已发送', '已发送开始录音命令'); + logger.d('开始录音命令发送成功'); + } else { + Get.snackbar('命令发送失败', '发送开始录音命令失败'); + logger.d('开始录音命令发送失败'); + } + } catch (e) { + logger.e('发送开始录音命令错误', error: e); + Get.snackbar('命令错误', e.toString()); + } + } + + /// 发送停止录音命令 + Future sendStopRecordingCommand() async { + if (currentDevice.value == null) { + Get.snackbar('操作失败', '未连接设备'); + return; + } + + try { + logger.d('发送停止录音命令'); + final success = await jieli.sendStopRecordingCommand(); + + if (success) { + Get.snackbar('命令已发送', '已发送停止录音命令'); + logger.d('停止录音命令发送成功'); + } else { + Get.snackbar('命令发送失败', '发送停止录音命令失败'); + logger.d('停止录音命令发送失败'); + } + } catch (e) { + logger.e('发送停止录音命令错误', error: e); + Get.snackbar('命令错误', e.toString()); + } + } + + /// 清理资源 + void _disposeJieli() { + try { + logger.d('释放Jieli插件资源'); + jieli.dispose(); + } catch (e) { + logger.e('释放Jieli插件资源错误', error: e); + } + } +} \ No newline at end of file diff --git a/lib/modules/jieli_test/views/jieli_test_view.dart b/lib/modules/jieli_test/views/jieli_test_view.dart new file mode 100644 index 000000000..aa6316366 --- /dev/null +++ b/lib/modules/jieli_test/views/jieli_test_view.dart @@ -0,0 +1,494 @@ +import 'package:flutter/material.dart'; +import 'package:get/get.dart'; +import 'package:flutter_screenutil/flutter_screenutil.dart'; +import '../controllers/jieli_test_controller.dart'; +import 'package:jieli/jieli.dart'; + +class JieliTestView extends GetView { + const JieliTestView({Key? key}) : super(key: key); + + @override + Widget build(BuildContext context) { + final isDarkMode = Theme.of(context).brightness == Brightness.dark; + + return Scaffold( + backgroundColor: isDarkMode ? Colors.black : Colors.grey[50], + appBar: AppBar( + backgroundColor: isDarkMode ? Colors.black : Colors.white, + elevation: 0.5, + title: Text( + '杰理蓝牙测试', + style: TextStyle( + fontSize: 17.sp, + fontWeight: FontWeight.w600, + color: isDarkMode ? Colors.white : Colors.black, + ), + ), + centerTitle: true, + leading: IconButton( + icon: Icon( + Icons.arrow_back_ios, + color: isDarkMode ? Colors.white : Colors.black, + size: 20.sp, + ), + onPressed: () => Get.back(), + ), + ), + body: Padding( + padding: EdgeInsets.all(16.w), + child: Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + _buildStatusCard(isDarkMode), + SizedBox(height: 16.h), + _buildActionButtons(isDarkMode), + SizedBox(height: 16.h), + _buildDeviceInfo(isDarkMode), + SizedBox(height: 16.h), + _buildDevicesList(isDarkMode), + ], + ), + ), + ); + } + + Widget _buildStatusCard(bool isDarkMode) { + return Container( + padding: EdgeInsets.all(16.w), + decoration: BoxDecoration( + color: isDarkMode ? Colors.grey[900] : Colors.white, + borderRadius: BorderRadius.circular(12.r), + boxShadow: [ + BoxShadow( + color: Colors.black.withOpacity(0.05), + blurRadius: 10, + offset: const Offset(0, 2), + ), + ], + ), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Row( + children: [ + Text( + '连接状态: ', + style: TextStyle( + fontSize: 14.sp, + color: isDarkMode ? Colors.white : Colors.black87, + ), + ), + Obx(() => Text( + controller.connectStatus.value, + style: TextStyle( + fontSize: 14.sp, + fontWeight: FontWeight.bold, + color: _getStatusColor(controller.connectStatus.value), + ), + )), + ], + ), + SizedBox(height: 8.h), + Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + Obx(() => _buildStatusChip( + label: '唤醒信号', + isActive: controller.wakeupReceived.value, + activeColor: Colors.orange, + isDarkMode: isDarkMode, + )), + Obx(() => _buildStatusChip( + label: '音频数据', + isActive: controller.audioDataReceiving.value, + activeColor: Colors.green, + isDarkMode: isDarkMode, + )), + ], + ), + SizedBox(height: 8.h), + Obx(() => controller.audioDataReceiving.value + ? Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + '音频数据统计:', + style: TextStyle( + fontSize: 12.sp, + color: isDarkMode ? Colors.white70 : Colors.black54, + ), + ), + SizedBox(height: 4.h), + Text( + '已接收 ${(controller.audioDataSize.value / 1024).toStringAsFixed(2)} KB', + style: TextStyle( + fontSize: 12.sp, + color: isDarkMode ? Colors.white70 : Colors.black54, + ), + ), + Text( + '共 ${controller.receivedFrames.value} 帧', + style: TextStyle( + fontSize: 12.sp, + color: isDarkMode ? Colors.white70 : Colors.black54, + ), + ), + ], + ) + : SizedBox.shrink() + ), + ], + ), + ); + } + + Widget _buildStatusChip({ + required String label, + required bool isActive, + required Color activeColor, + required bool isDarkMode, + }) { + return Container( + padding: EdgeInsets.symmetric(horizontal: 12.w, vertical: 6.h), + decoration: BoxDecoration( + color: isActive + ? activeColor + : (isDarkMode ? Colors.grey[800] : Colors.grey[200]), + borderRadius: BorderRadius.circular(16.r), + ), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Container( + width: 8.w, + height: 8.w, + decoration: BoxDecoration( + color: isActive ? Colors.white : Colors.grey, + shape: BoxShape.circle, + ), + ), + SizedBox(width: 6.w), + Text( + label, + style: TextStyle( + fontSize: 12.sp, + color: isActive + ? Colors.white + : (isDarkMode ? Colors.white70 : Colors.black54), + fontWeight: isActive ? FontWeight.bold : FontWeight.normal, + ), + ), + ], + ), + ); + } + + Widget _buildActionButtons(bool isDarkMode) { + return Column( + children: [ + Row( + children: [ + Expanded( + child: Obx(() => ElevatedButton.icon( + onPressed: controller.isScanning.value + ? controller.stopScan + : controller.startScan, + icon: Icon( + controller.isScanning.value ? Icons.stop : Icons.bluetooth_searching, + size: 18.sp, + ), + label: Text( + controller.isScanning.value ? '停止扫描' : '扫描设备', + style: TextStyle(fontSize: 14.sp), + ), + style: ElevatedButton.styleFrom( + backgroundColor: controller.isScanning.value + ? Colors.red[600] + : Colors.blue[600], + padding: EdgeInsets.symmetric(vertical: 12.h), + shape: RoundedRectangleBorder( + borderRadius: BorderRadius.circular(8.r), + ), + ), + )), + ), + ], + ), + SizedBox(height: 8.h), + Row( + children: [ + Expanded( + child: ElevatedButton.icon( + onPressed: controller.sendStartRecordingCommand, + icon: Icon( + Icons.mic, + size: 18.sp, + ), + label: Text( + '开始录音', + style: TextStyle(fontSize: 14.sp), + ), + style: ElevatedButton.styleFrom( + backgroundColor: Colors.green[600], + padding: EdgeInsets.symmetric(vertical: 12.h), + shape: RoundedRectangleBorder( + borderRadius: BorderRadius.circular(8.r), + ), + ), + ), + ), + SizedBox(width: 8.w), + Expanded( + child: ElevatedButton.icon( + onPressed: controller.sendStopRecordingCommand, + icon: Icon( + Icons.stop_circle, + size: 18.sp, + ), + label: Text( + '停止录音', + style: TextStyle(fontSize: 14.sp), + ), + style: ElevatedButton.styleFrom( + backgroundColor: Colors.red[600], + padding: EdgeInsets.symmetric(vertical: 12.h), + shape: RoundedRectangleBorder( + borderRadius: BorderRadius.circular(8.r), + ), + ), + ), + ), + ], + ), + ], + ); + } + + Widget _buildDeviceInfo(bool isDarkMode) { + return Container( + padding: EdgeInsets.all(16.w), + decoration: BoxDecoration( + color: isDarkMode ? Colors.grey[900] : Colors.white, + borderRadius: BorderRadius.circular(12.r), + boxShadow: [ + BoxShadow( + color: Colors.black.withOpacity(0.05), + blurRadius: 10, + offset: const Offset(0, 2), + ), + ], + ), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + '设备信息:', + style: TextStyle( + fontSize: 14.sp, + color: isDarkMode ? Colors.white : Colors.black87, + ), + ), + SizedBox(height: 8.h), + Container( + padding: EdgeInsets.all(8.w), + decoration: BoxDecoration( + color: isDarkMode ? Colors.black : Colors.grey[100], + borderRadius: BorderRadius.circular(8.r), + ), + child: Obx(() => Text( + controller.deviceInfo.value, + style: TextStyle( + fontSize: 13.sp, + color: isDarkMode ? Colors.white70 : Colors.black54, + height: 1.4, + ), + )), + ), + SizedBox(height: 16.h), + Obx(() => controller.currentDevice.value != null + ? ElevatedButton.icon( + onPressed: controller.disconnectDevice, + icon: Icon( + Icons.bluetooth_disabled, + size: 16.sp, + ), + label: Text( + '断开连接', + style: TextStyle(fontSize: 13.sp), + ), + style: ElevatedButton.styleFrom( + backgroundColor: Colors.red[600], + padding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 10.h), + shape: RoundedRectangleBorder( + borderRadius: BorderRadius.circular(8.r), + ), + ), + ) + : SizedBox.shrink() + ), + ], + ), + ); + } + + Widget _buildDevicesList(bool isDarkMode) { + return Expanded( + child: Container( + decoration: BoxDecoration( + color: isDarkMode ? Colors.grey[900] : Colors.white, + borderRadius: BorderRadius.circular(12.r), + boxShadow: [ + BoxShadow( + color: Colors.black.withOpacity(0.05), + blurRadius: 10, + offset: const Offset(0, 2), + ), + ], + ), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Padding( + padding: EdgeInsets.all(16.w), + child: Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + Text( + '扫描到的设备', + style: TextStyle( + fontSize: 16.sp, + fontWeight: FontWeight.w600, + color: isDarkMode ? Colors.white : Colors.black87, + ), + ), + Obx(() => Text( + '${controller.scannedDevices.length}个设备', + style: TextStyle( + fontSize: 12.sp, + color: isDarkMode ? Colors.white70 : Colors.black54, + ), + )), + ], + ), + ), + Divider(height: 1, thickness: 1, color: isDarkMode ? Colors.grey[800] : Colors.grey[200]), + Expanded( + child: Obx(() => controller.scannedDevices.isEmpty + ? Center( + child: controller.isScanning.value + ? Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + CircularProgressIndicator(), + SizedBox(height: 16.h), + Text( + '正在扫描...', + style: TextStyle( + fontSize: 14.sp, + color: isDarkMode ? Colors.white70 : Colors.black54, + ), + ), + ], + ) + : Text( + '未扫描到设备', + style: TextStyle( + fontSize: 14.sp, + color: isDarkMode ? Colors.white70 : Colors.black54, + ), + ), + ) + : ListView.separated( + padding: EdgeInsets.symmetric(vertical: 8.h), + itemCount: controller.scannedDevices.length, + separatorBuilder: (context, index) => Divider( + height: 1, + thickness: 1, + color: isDarkMode ? Colors.grey[800] : Colors.grey[200], + ), + itemBuilder: (context, index) { + final device = controller.scannedDevices[index]; + return ListTile( + dense: true, + title: Text( + device.name.isEmpty ? '未知设备' : device.name, + style: TextStyle( + fontSize: 14.sp, + fontWeight: FontWeight.w500, + color: isDarkMode ? Colors.white : Colors.black87, + ), + ), + subtitle: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + '设备ID: ${device.id}', + style: TextStyle( + fontSize: 11.sp, + color: isDarkMode ? Colors.white70 : Colors.black54, + ), + ), + Text( + 'RSSI: ${device.rssi} dBm', + style: TextStyle( + fontSize: 11.sp, + color: isDarkMode ? Colors.white70 : Colors.black54, + ), + ), + ], + ), + trailing: Obx(() => controller.currentDevice.value?.id == device.id + ? Chip( + label: Text( + '已连接', + style: TextStyle( + fontSize: 12.sp, + color: Colors.white, + ), + ), + backgroundColor: Colors.green, + padding: EdgeInsets.symmetric(horizontal: 8.w), + ) + : ElevatedButton( + onPressed: () => controller.connectToDevice(device), + child: Text( + '连接', + style: TextStyle(fontSize: 12.sp), + ), + style: ElevatedButton.styleFrom( + backgroundColor: Colors.green[600], + padding: EdgeInsets.symmetric(horizontal: 12.w, vertical: 6.h), + shape: RoundedRectangleBorder( + borderRadius: BorderRadius.circular(6.r), + ), + ), + ), + ), + contentPadding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 8.h), + isThreeLine: true, + ); + }, + ), + ), + ), + ], + ), + ), + ); + } + + Color _getStatusColor(String status) { + switch (status) { + case '连接中...': + return Colors.orange; + case '已连接': + return Colors.green; + case '连接失败': + return Colors.red; + case '已断开': + return Colors.grey; + default: + return Colors.grey; + } + } +} \ No newline at end of file diff --git a/lib/modules/pairing/controllers/pairing_controller.dart b/lib/modules/pairing/controllers/pairing_controller.dart index 2501e26bd..5c36698c3 100644 --- a/lib/modules/pairing/controllers/pairing_controller.dart +++ b/lib/modules/pairing/controllers/pairing_controller.dart @@ -6,11 +6,12 @@ import '../../../data/services/classic_bluetooth_service.dart'; import '../../../core/utils/logger.dart'; import 'dart:io'; import '../../../data/services/voice_interaction_service.dart'; - /// 简化的配对控制器，仅用于显示已连接经典蓝牙设备和处理基本配对流程 class PairingController extends GetxController { static PairingController get to => Get.find(); + + // 经典蓝牙服务 final ClassicBluetoothService _bluetoothService = Get.find(); @@ -109,6 +110,7 @@ class PairingController extends GetxController { /// 初始化流程 Future _initialize() async { try { + // 初始化语音交互服务 _voiceInteractionService.initialize(); diff --git a/lib/modules/settings/views/settings_view.dart b/lib/modules/settings/views/settings_view.dart index ce4f7866a..fac80cae2 100644 --- a/lib/modules/settings/views/settings_view.dart +++ b/lib/modules/settings/views/settings_view.dart @@ -480,6 +480,28 @@ class SettingsView extends GetView { }, isDarkMode: isDarkMode, ), + Divider( + height: 1, + color: isDarkMode + ? Colors.white.withOpacity(0.1) + : Colors.grey[200]), + // 杰理测试 + _buildSimpleNavigationSetting( + title: '杰理蓝牙测试', + subtitle: '测试杰理蓝牙设备扫描、连接和功能', + icon: Icons.bluetooth, + iconBgColor: isDarkMode + ? Colors.blue[900]!.withOpacity(0.3) + : Colors.blue[100]!, + iconColor: + isDarkMode ? Colors.blue[300]! : Colors.blue[600]!, + titleColor: isDarkMode ? Colors.white : null, + subtitleColor: isDarkMode ? Colors.white70 : null, + onTap: () { + Get.toNamed(Routes.jieliTest); + }, + isDarkMode: isDarkMode, + ), Divider( height: 1, color: isDarkMode diff --git a/lib/routes/app_pages.dart b/lib/routes/app_pages.dart index 8a2466fda..07e5cc5fb 100644 --- a/lib/routes/app_pages.dart +++ b/lib/routes/app_pages.dart @@ -31,6 +31,8 @@ import '../modules/pairing/bindings/pairing_binding.dart'; import './app_routes.dart'; import '../modules/settings/views/settings_view.dart'; import '../modules/settings/bindings/settings_binding.dart'; +import '../modules/jieli_test/views/jieli_test_view.dart'; +import '../modules/jieli_test/bindings/jieli_test_binding.dart'; abstract class AppPages { static final pages = [ @@ -119,5 +121,10 @@ abstract class AppPages { page: () => const SettingsView(), binding: SettingsBinding(), ), + GetPage( + name: Routes.jieliTest, + page: () => const JieliTestView(), + binding: JieliTestBinding(), + ), ]; } diff --git a/lib/routes/app_routes.dart b/lib/routes/app_routes.dart index 4488f63db..7d661c0a3 100644 --- a/lib/routes/app_routes.dart +++ b/lib/routes/app_routes.dart @@ -24,4 +24,5 @@ abstract class Routes { static const FLUTTER_TTS_TEST = '/flutter_tts_test'; static const FLUTTER_ASR_TEST = '/flutter_asr_test'; static const editProfile = '/edit_profile'; + static const jieliTest = '/jieli_test'; } diff --git a/local_plugins/azure_speech/android/build.gradle.kts b/local_plugins/azure_speech/android/build.gradle.kts index a26b13967..9aabbb990 100644 --- a/local_plugins/azure_speech/android/build.gradle.kts +++ b/local_plugins/azure_speech/android/build.gradle.kts @@ -1,32 +1,11 @@ -import com.android.build.gradle.LibraryExtension - -buildscript { - repositories { - google() - mavenCentral() - } - dependencies { - classpath("com.android.tools.build:gradle:7.3.0") - classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:1.7.10") - } -} - -allprojects { - repositories { - google() - mavenCentral() - } -} - plugins { id("com.android.library") - kotlin("android") + id("org.jetbrains.kotlin.android") } -// 配置android扩展 -configure { +android { namespace = "com.yunqiinnovation.azure_speech" - compileSdkVersion(33) + compileSdk = 33 defaultConfig { minSdk = 21 @@ -48,6 +27,20 @@ configure { lintOptions { isCheckReleaseBuilds = false } + + // 添加packaging配置，排除冲突文件 + packaging { + resources { + excludes.add("META-INF/DEPENDENCIES") + excludes.add("META-INF/LICENSE") + excludes.add("META-INF/LICENSE.txt") + excludes.add("META-INF/license.txt") + excludes.add("META-INF/NOTICE") + excludes.add("META-INF/NOTICE.txt") + excludes.add("META-INF/notice.txt") + excludes.add("META-INF/*.kotlin_module") + } + } } // 显式设置Kotlin JVM目标版本 @@ -58,7 +51,6 @@ tasks.withType { } dependencies { - // 添加Microsoft语音SDK implementation("com.microsoft.cognitiveservices.speech:client-sdk:1.30.0") } \ No newline at end of file diff --git a/local_plugins/chat_storage_sqlite/README.md b/local_plugins/chat_storage_sqlite/README.md new file mode 100644 index 000000000..8db279af0 --- /dev/null +++ b/local_plugins/chat_storage_sqlite/README.md @@ -0,0 +1,80 @@ +# chat_storage_sqlite + +SQLite存储插件，用于存储聊天记录并支持分页查询。 + +## 功能特点 + +- 使用SQLite数据库存储聊天记录 +- 提供分页查询功能，可指定获取对应页码的记录 +- 支持按会话ID组织和查询聊天记录 +- 提供添加、查询、删除等基本操作 + +## 使用方法 + +### 保存聊天记录 + +```dart +import 'package:chat_storage_sqlite/chat_storage_sqlite.dart'; + +// 保存一条聊天记录 +final result = await ChatStorageSqlite.saveMessage( + sessionId: 'session_123', + message: '你好，这是一条测试消息', + sender: 'user', + metadata: '{"timestamp": 1630000000000}' // 可选的元数据，支持JSON字符串 +); + +if (result['success']) { + print('消息保存成功，ID: ${result['id']}'); +} else { + print('消息保存失败: ${result['error']}'); +} +``` + +### 获取聊天记录（支持分页） + +```dart +// 获取指定会话的聊天记录，支持分页 +final pagedResult = await ChatStorageSqlite.getMessages( + sessionId: 'session_123', + page: 1, // 当前页码，从1开始 + pageSize: 20, // 每页记录数 +); + +// 使用结果 +print('总记录数: ${pagedResult.totalCount}'); +print('总页数: ${pagedResult.totalPages}'); +print('当前页: ${pagedResult.page}'); + +// 遍历消息列表 +for (var message in pagedResult.messages) { + print('ID: ${message.id}'); + print('发送者: ${message.sender}'); + print('内容: ${message.message}'); + print('时间: ${message.timestamp}'); + print('---'); +} +``` + +### 删除聊天记录 + +```dart +// 方式1：删除指定ID的消息 +final deleteResult1 = await ChatStorageSqlite.deleteMessages( + messageIds: [1, 2, 3], +); + +// 方式2：删除整个会话的所有消息 +final deleteResult2 = await ChatStorageSqlite.deleteMessages( + sessionId: 'session_123', +); + +// 方式3：清空整个数据库 +final clearResult = await ChatStorageSqlite.clearDatabase(); +``` + +## 注意事项 + +- 插件会自动创建和管理SQLite数据库，无需手动初始化 +- 所有操作都是异步的，请使用async/await处理返回结果 +- 插件内部已处理异常情况，但建议在使用时添加适当的错误处理 \ No newline at end of file diff --git a/local_plugins/chat_storage_sqlite/android/build.gradle.kts b/local_plugins/chat_storage_sqlite/android/build.gradle.kts new file mode 100644 index 000000000..78e1b31ea --- /dev/null +++ b/local_plugins/chat_storage_sqlite/android/build.gradle.kts @@ -0,0 +1,60 @@ +plugins { + // Android Library 插件 + id("com.android.library") + // Kotlin Android 插件 + id("org.jetbrains.kotlin.android") + id("com.github.johnrengelman.shadow") version "8.1.1" +} + +android { + // 命名空间，对应你插件的包名（需与代码内包名保持一致） + namespace = "com.yunqiinnovation.chat_storage_sqlite" + + // 目标 SDK 版本 + compileSdk = 33 + + defaultConfig { + // 最低 SDK 版本 + minSdk = 21 + targetSdk = 33 + } + + // Java 语言级别兼容配置 + compileOptions { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 + } + + // Kotlin 语言级别 + kotlinOptions { + jvmTarget = "11" + } + + // 添加packaging配置，排除冲突文件 + packaging { + resources { + excludes.add("META-INF/DEPENDENCIES") + excludes.add("META-INF/LICENSE") + excludes.add("META-INF/LICENSE.txt") + excludes.add("META-INF/license.txt") + excludes.add("META-INF/NOTICE") + excludes.add("META-INF/NOTICE.txt") + excludes.add("META-INF/notice.txt") + excludes.add("META-INF/*.kotlin_module") + } + } +} + +dependencies { + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.6.4") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4") + implementation("com.fasterxml.jackson.core:jackson-databind:2.14.2") + implementation("com.squareup.okhttp3:okhttp:4.9.3") + implementation("com.anthropic:anthropic-java:0.8.0") + implementation("io.modelcontextprotocol:kotlin-sdk:0.4.0") +} + +dependencies { + // SQLite相关依赖 + implementation("androidx.sqlite:sqlite:2.3.1") +} \ No newline at end of file diff --git a/local_plugins/chat_storage_sqlite/android/settings.gradle.kts b/local_plugins/chat_storage_sqlite/android/settings.gradle.kts new file mode 100644 index 000000000..613be470e --- /dev/null +++ b/local_plugins/chat_storage_sqlite/android/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "chat_storage_sqlite" \ No newline at end of file diff --git a/local_plugins/chat_storage_sqlite/android/src/main/AndroidManifest.xml b/local_plugins/chat_storage_sqlite/android/src/main/AndroidManifest.xml new file mode 100644 index 000000000..b7fb5f78b --- /dev/null +++ b/local_plugins/chat_storage_sqlite/android/src/main/AndroidManifest.xml @@ -0,0 +1,8 @@ + + + + + + + \ No newline at end of file diff --git a/local_plugins/chat_storage_sqlite/android/src/main/kotlin/com/yunqiinnovation/chat_storage_sqlite/ChatStorageSqlitePlugin.kt b/local_plugins/chat_storage_sqlite/android/src/main/kotlin/com/yunqiinnovation/chat_storage_sqlite/ChatStorageSqlitePlugin.kt new file mode 100644 index 000000000..d302c4d7d --- /dev/null +++ b/local_plugins/chat_storage_sqlite/android/src/main/kotlin/com/yunqiinnovation/chat_storage_sqlite/ChatStorageSqlitePlugin.kt @@ -0,0 +1,236 @@ +package com.yunqiinnovation.chat_storage_sqlite + +import android.content.Context +import android.database.Cursor +import android.database.sqlite.SQLiteDatabase +import android.database.sqlite.SQLiteOpenHelper +import android.util.Log +import io.flutter.embedding.engine.plugins.FlutterPlugin +import io.flutter.plugin.common.MethodCall +import io.flutter.plugin.common.MethodChannel +import io.flutter.plugin.common.MethodChannel.MethodCallHandler +import io.flutter.plugin.common.MethodChannel.Result +import org.json.JSONArray +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.* + +// 定义数据库帮助类 +private class ChatDatabaseHelper(context: Context) : + SQLiteOpenHelper(context, DATABASE_NAME, null, DATABASE_VERSION) { + + companion object { + private const val DATABASE_NAME = "chat_messages.db" + private const val DATABASE_VERSION = 1 + + // 定义消息表 + private const val TABLE_MESSAGES = "messages" + private const val COLUMN_ID = "id" + private const val COLUMN_SESSION_ID = "session_id" + private const val COLUMN_MESSAGE = "message" + private const val COLUMN_SENDER = "sender" + private const val COLUMN_TIMESTAMP = "timestamp" + private const val COLUMN_METADATA = "metadata" + } + + override fun onCreate(db: SQLiteDatabase) { + // 创建消息表 + val createMessagesTable = """ + CREATE TABLE $TABLE_MESSAGES ( + $COLUMN_ID INTEGER PRIMARY KEY AUTOINCREMENT, + $COLUMN_SESSION_ID TEXT NOT NULL, + $COLUMN_MESSAGE TEXT NOT NULL, + $COLUMN_SENDER TEXT NOT NULL, + $COLUMN_TIMESTAMP TEXT NOT NULL, + $COLUMN_METADATA TEXT + ) + """.trimIndent() + + db.execSQL(createMessagesTable) + + // 创建索引以加快查询速度 + db.execSQL("CREATE INDEX idx_session_id ON $TABLE_MESSAGES ($COLUMN_SESSION_ID)") + db.execSQL("CREATE INDEX idx_timestamp ON $TABLE_MESSAGES ($COLUMN_TIMESTAMP)") + } + + override fun onUpgrade(db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { + // 处理数据库升级逻辑 + if (oldVersion < newVersion) { + // 简单的升级策略 - 删除旧表并创建新表 + // 在实际应用中，应该实现数据迁移而非删除 + db.execSQL("DROP TABLE IF EXISTS $TABLE_MESSAGES") + onCreate(db) + } + } +} + +/** ChatStorageSqlitePlugin */ +class ChatStorageSqlitePlugin: FlutterPlugin, MethodCallHandler { + private lateinit var channel : MethodChannel + private lateinit var context: Context + private lateinit var dbHelper: ChatDatabaseHelper + + override fun onAttachedToEngine(flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { + context = flutterPluginBinding.applicationContext + channel = MethodChannel(flutterPluginBinding.binaryMessenger, "chat_storage_sqlite") + channel.setMethodCallHandler(this) + dbHelper = ChatDatabaseHelper(context) + } + + override fun onMethodCall(call: MethodCall, result: Result) { + when (call.method) { + "saveMessage" -> { + try { + val sessionId = call.argument("sessionId") ?: "" + val message = call.argument("message") ?: "" + val sender = call.argument("sender") ?: "" + val metadata = call.argument("metadata") + + val db = dbHelper.writableDatabase + + // 获取当前时间作为时间戳 + val timestamp = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.getDefault()).format(Date()) + + // 执行插入操作 + val values = android.content.ContentValues().apply { + put("session_id", sessionId) + put("message", message) + put("sender", sender) + put("timestamp", timestamp) + put("metadata", metadata) + } + + val id = db.insert("messages", null, values) + if (id != -1L) { + result.success(mapOf("success" to true, "id" to id)) + } else { + result.error("DB_ERROR", "插入消息失败", null) + } + } catch (e: Exception) { + result.error("EXCEPTION", "保存消息时发生错误: ${e.message}", null) + } + } + "getMessages" -> { + try { + val sessionId = call.argument("sessionId") ?: "" + val page = call.argument("page") ?: 1 + val pageSize = call.argument("pageSize") ?: 20 + + val db = dbHelper.readableDatabase + val offset = (page - 1) * pageSize + + // 构建查询语句 + val query = """ + SELECT * FROM messages + WHERE session_id = ? + ORDER BY timestamp DESC + LIMIT ? OFFSET ? + """.trimIndent() + + val selectionArgs = arrayOf(sessionId, pageSize.toString(), offset.toString()) + val cursor = db.rawQuery(query, selectionArgs) + + val messages = cursorToJsonArray(cursor) + + // 获取总记录数 + val countQuery = "SELECT COUNT(*) FROM messages WHERE session_id = ?" + val countCursor = db.rawQuery(countQuery, arrayOf(sessionId)) + var totalCount = 0 + if (countCursor.moveToFirst()) { + totalCount = countCursor.getInt(0) + } + countCursor.close() + + // 计算总页数 + val totalPages = if (totalCount % pageSize == 0) { + totalCount / pageSize + } else { + (totalCount / pageSize) + 1 + } + + val response = JSONObject().apply { + put("messages", messages) + put("page", page) + put("pageSize", pageSize) + put("totalCount", totalCount) + put("totalPages", totalPages) + } + + result.success(response.toString()) + } catch (e: Exception) { + result.error("EXCEPTION", "获取消息时发生错误: ${e.message}", null) + } + } + "deleteMessages" -> { + try { + val sessionId = call.argument("sessionId") ?: "" + val messageIds = call.argument>("messageIds") + + val db = dbHelper.writableDatabase + + if (messageIds != null && messageIds.isNotEmpty()) { + // 删除指定ID的消息 + val idsStr = messageIds.joinToString(",") + val deleteQuery = "DELETE FROM messages WHERE id IN ($idsStr)" + db.execSQL(deleteQuery) + result.success(mapOf("success" to true, "count" to messageIds.size)) + } else if (sessionId.isNotEmpty()) { + // 删除会话中的所有消息 + val deleteQuery = "DELETE FROM messages WHERE session_id = ?" + db.execSQL(deleteQuery, arrayOf(sessionId)) + result.success(mapOf("success" to true)) + } else { + result.error("INVALID_ARGS", "必须提供sessionId或messageIds", null) + } + } catch (e: Exception) { + result.error("EXCEPTION", "删除消息时发生错误: ${e.message}", null) + } + } + "clearDatabase" -> { + try { + val db = dbHelper.writableDatabase + db.execSQL("DELETE FROM messages") + result.success(mapOf("success" to true)) + } catch (e: Exception) { + result.error("EXCEPTION", "清空数据库时发生错误: ${e.message}", null) + } + } + else -> { + result.notImplemented() + } + } + } + + private fun cursorToJsonArray(cursor: Cursor): JSONArray { + val jsonArray = JSONArray() + + val idIndex = cursor.getColumnIndex("id") + val sessionIdIndex = cursor.getColumnIndex("session_id") + val messageIndex = cursor.getColumnIndex("message") + val senderIndex = cursor.getColumnIndex("sender") + val timestampIndex = cursor.getColumnIndex("timestamp") + val metadataIndex = cursor.getColumnIndex("metadata") + + while (cursor.moveToNext()) { + val jsonObject = JSONObject().apply { + put("id", cursor.getInt(idIndex)) + put("sessionId", cursor.getString(sessionIdIndex)) + put("message", cursor.getString(messageIndex)) + put("sender", cursor.getString(senderIndex)) + put("timestamp", cursor.getString(timestampIndex)) + + if (metadataIndex != -1 && !cursor.isNull(metadataIndex)) { + put("metadata", cursor.getString(metadataIndex)) + } + } + jsonArray.put(jsonObject) + } + + cursor.close() + return jsonArray + } + + override fun onDetachedFromEngine(binding: FlutterPlugin.FlutterPluginBinding) { + channel.setMethodCallHandler(null) + } +} \ No newline at end of file diff --git a/local_plugins/chat_storage_sqlite/lib/chat_storage_sqlite.dart b/local_plugins/chat_storage_sqlite/lib/chat_storage_sqlite.dart new file mode 100644 index 000000000..f41f473a7 --- /dev/null +++ b/local_plugins/chat_storage_sqlite/lib/chat_storage_sqlite.dart @@ -0,0 +1,155 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:flutter/services.dart'; + +/// 聊天记录数据模型 +class ChatMessage { + final int? id; + final String sessionId; + final String message; + final String sender; + final String timestamp; + final String? metadata; + + ChatMessage({ + this.id, + required this.sessionId, + required this.message, + required this.sender, + required this.timestamp, + this.metadata, + }); + + Map toMap() { + return { + 'id': id, + 'sessionId': sessionId, + 'message': message, + 'sender': sender, + 'timestamp': timestamp, + 'metadata': metadata, + }; + } + + factory ChatMessage.fromMap(Map map) { + return ChatMessage( + id: map['id'], + sessionId: map['sessionId'], + message: map['message'], + sender: map['sender'], + timestamp: map['timestamp'], + metadata: map['metadata'], + ); + } +} + +/// 分页结果数据模型 +class PagedResult { + final List messages; + final int page; + final int pageSize; + final int totalCount; + final int totalPages; + + PagedResult({ + required this.messages, + required this.page, + required this.pageSize, + required this.totalCount, + required this.totalPages, + }); +} + +/// SQLite聊天记录存储插件 +class ChatStorageSqlite { + static const MethodChannel _channel = MethodChannel('chat_storage_sqlite'); + + /// 保存一条聊天记录 + static Future> saveMessage({ + required String sessionId, + required String message, + required String sender, + String? metadata, + }) async { + try { + final result = await _channel.invokeMethod('saveMessage', { + 'sessionId': sessionId, + 'message': message, + 'sender': sender, + 'metadata': metadata, + }); + return Map.from(result); + } catch (e) { + return {'success': false, 'error': e.toString()}; + } + } + + /// 获取指定会话的聊天记录（支持分页） + static Future getMessages({ + required String sessionId, + int page = 1, + int pageSize = 20, + }) async { + try { + final String result = await _channel.invokeMethod('getMessages', { + 'sessionId': sessionId, + 'page': page, + 'pageSize': pageSize, + }); + + final Map data = jsonDecode(result); + final List messagesList = data['messages']; + + List messages = messagesList.map((item) { + return ChatMessage.fromMap(Map.from(item)); + }).toList(); + + return PagedResult( + messages: messages, + page: data['page'], + pageSize: data['pageSize'], + totalCount: data['totalCount'], + totalPages: data['totalPages'], + ); + } catch (e) { + return PagedResult( + messages: [], + page: page, + pageSize: pageSize, + totalCount: 0, + totalPages: 0, + ); + } + } + + /// 删除聊天记录，可按会话ID或消息ID列表删除 + static Future> deleteMessages({ + String? sessionId, + List? messageIds, + }) async { + try { + if (sessionId == null && (messageIds == null || messageIds.isEmpty)) { + return {'success': false, 'error': '必须提供sessionId或messageIds'}; + } + + final result = await _channel.invokeMethod('deleteMessages', { + 'sessionId': sessionId, + 'messageIds': messageIds, + }); + return Map.from(result); + } catch (e) { + return {'success': false, 'error': e.toString()}; + } + } + + /// 清空整个数据库 + static Future> clearDatabase() async { + try { + final result = await _channel.invokeMethod('clearDatabase'); + return Map.from(result); + } catch (e) { + return {'success': false, 'error': e.toString()}; + } + } +} \ No newline at end of file diff --git a/local_plugins/chat_storage_sqlite/pubspec.yaml b/local_plugins/chat_storage_sqlite/pubspec.yaml new file mode 100644 index 000000000..a4d6e1bf1 --- /dev/null +++ b/local_plugins/chat_storage_sqlite/pubspec.yaml @@ -0,0 +1,30 @@ +name: chat_storage_sqlite +description: SQLite存储插件，用于存储聊天记录并支持分页查询 +version: 0.0.1 +homepage: + +environment: + sdk: ">=2.17.0 <3.0.0" + flutter: ">=2.5.0" + +dependencies: + flutter: + sdk: flutter + sqflite: ^2.3.0 + path: ^1.8.3 + +dev_dependencies: + flutter_test: + sdk: flutter + flutter_lints: ^2.0.0 + +# The following section is specific to Flutter packages. +flutter: + # This section identifies this Flutter project as a plugin project. + plugin: + platforms: + android: + package: com.yunqiinnovation.chat_storage_sqlite + pluginClass: ChatStorageSqlitePlugin + ios: + pluginClass: ChatStorageSqlitePlugin \ No newline at end of file diff --git a/local_plugins/jieli/README.md b/local_plugins/jieli/README.md new file mode 100644 index 000000000..d926109c8 --- /dev/null +++ b/local_plugins/jieli/README.md @@ -0,0 +1,144 @@ +# Jieli 插件 + +一个用于连接杰理蓝牙耳机的Flutter插件，支持Android 12+的BLE通信。 + +## 功能 + +- 扫描支持特定BLE服务的耳机设备 +- 连接设备并发现服务 +- 接收耳机的唤醒通知 +- 向耳机发送控制命令启动录音 +- 接收音频数据流 + +## 技术规范 + +此插件实现了以下BLE特性： + +- **服务UUID**: 0000A100-0000-1000-8000-00805F9B34FB +- **音频数据特征UUID**: 0000A101-0000-1000-8000-00805F9B34FB +- **控制特征UUID**: 0000A102-0000-1000-8000-00805F9B34FB + +## 使用方法 + +### 初始化插件 + +```dart +final jieli = Jieli(); +``` + +### 检查蓝牙状态 + +```dart +bool isEnabled = await jieli.isBluetoothEnabled(); +if (!isEnabled) { + await jieli.requestBluetoothEnable(); +} +``` + +### 扫描设备 + +```dart +// 监听扫描结果 +jieli.scanResults.listen((devices) { + // 处理扫描到的设备列表 + for (var device in devices) { + print('发现设备: ${device.name} (${device.id}), RSSI: ${device.rssi}'); + } +}); + +// 开始扫描，10秒后自动停止 +await jieli.startScan(scanTimeout: 10000); + +// 手动停止扫描 +await jieli.stopScan(); +``` + +### 连接设备 + +```dart +// 监听连接状态 +jieli.connectionState.listen((state) { + switch (state) { + case ConnectionState.connected: + print('设备已连接'); + break; + case ConnectionState.disconnected: + print('设备已断开'); + break; + case ConnectionState.connecting: + print('正在连接...'); + break; + case ConnectionState.disconnecting: + print('正在断开...'); + break; + case ConnectionState.error: + print('连接错误'); + break; + } +}); + +// 连接到设备 +await jieli.connect(deviceId); + +// 断开连接 +await jieli.disconnect(); +``` + +### 接收唤醒信号 + +```dart +// 监听唤醒信号 +jieli.wakeupStream.listen((wakeup) { + print('收到唤醒信号'); + // 通常在收到唤醒后自动发送开始录音命令 +}); +``` + +### 发送录音命令 + +```dart +// 发送开始录音命令 +await jieli.sendStartRecordingCommand(); + +// 发送停止录音命令 +await jieli.sendStopRecordingCommand(); +``` + +### 接收音频数据 + +```dart +// 监听音频数据流 +jieli.audioDataStream.listen((data) { + print('收到音频数据: ${data.length} 字节'); + // 处理音频数据 +}); +``` + +### 资源清理 + +```dart +// 在不再需要插件时释放资源 +await jieli.dispose(); +``` + +## 权限 + +使用此插件需要以下Android权限： + +- Android 12+ (API 31+): `BLUETOOTH_SCAN`, `BLUETOOTH_CONNECT` +- Android 11及以下: `BLUETOOTH`, `BLUETOOTH_ADMIN` + +插件会在需要时自动请求权限，但建议在应用清单文件中声明这些权限： + +```xml + + + + + + + +``` \ No newline at end of file diff --git a/local_plugins/jieli/android/build.gradle.kts b/local_plugins/jieli/android/build.gradle.kts new file mode 100644 index 000000000..70b01c20e --- /dev/null +++ b/local_plugins/jieli/android/build.gradle.kts @@ -0,0 +1,55 @@ +plugins { + // Android Library 插件 + id("com.android.library") + // Kotlin Android 插件 + id("org.jetbrains.kotlin.android") + id("com.github.johnrengelman.shadow") version "8.1.1" +} + +android { + // 命名空间，对应你插件的包名（需与代码内包名保持一致） + namespace = "com.yunqiinnovation.jieli" + + // 目标 SDK 版本 + compileSdk = 33 + + defaultConfig { + // 最低 SDK 版本 + minSdk = 21 + targetSdk = 33 + } + + // Java 语言级别兼容配置 + compileOptions { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 + } + + // Kotlin 语言级别 + kotlinOptions { + jvmTarget = "11" + } + + // 添加packaging配置，排除冲突文件 + packaging { + resources { + excludes.add("META-INF/DEPENDENCIES") + excludes.add("META-INF/LICENSE") + excludes.add("META-INF/LICENSE.txt") + excludes.add("META-INF/license.txt") + excludes.add("META-INF/NOTICE") + excludes.add("META-INF/NOTICE.txt") + excludes.add("META-INF/notice.txt") + excludes.add("META-INF/*.kotlin_module") + } + } +} + +dependencies { + implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.6.10") + implementation("androidx.core:core-ktx:1.7.0") + implementation("androidx.lifecycle:lifecycle-viewmodel-ktx:2.4.0") + implementation("androidx.lifecycle:lifecycle-livedata-ktx:2.4.0") + implementation("androidx.lifecycle:lifecycle-service:2.4.0") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.5.2") +} \ No newline at end of file diff --git a/local_plugins/jieli/android/settings.gradle.kts b/local_plugins/jieli/android/settings.gradle.kts new file mode 100644 index 000000000..c0323d3bc --- /dev/null +++ b/local_plugins/jieli/android/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "jieli" \ No newline at end of file diff --git a/local_plugins/jieli/android/src/main/AndroidManifest.xml b/local_plugins/jieli/android/src/main/AndroidManifest.xml new file mode 100644 index 000000000..3cf7a945e --- /dev/null +++ b/local_plugins/jieli/android/src/main/AndroidManifest.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/BleService.kt b/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/BleService.kt new file mode 100644 index 000000000..0ba59a6e6 --- /dev/null +++ b/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/BleService.kt @@ -0,0 +1,460 @@ +package com.yunqiinnovation.jieli + +import android.bluetooth.* +import android.bluetooth.le.* +import android.content.Context +import android.os.Handler +import android.os.Looper +import android.os.ParcelUuid +import android.util.Log +import androidx.lifecycle.LifecycleService +import androidx.lifecycle.MutableLiveData +import kotlinx.coroutines.* +import java.util.* + +/** + * BLE回调接口，统一处理蓝牙事件 + */ +interface BleCallback { + /** + * 扫描结果回调 + * @param devices 扫描到的设备列表 + */ + fun onScanResult(devices: List>) + + /** + * 连接状态变化回调 + * @param state 连接状态 + */ + fun onConnectionStateChanged(state: Int) + + /** + * 音频数据回调 + * @param data 音频数据字节数组 + */ + fun onAudioDataReceived(data: ByteArray) + + /** + * 唤醒信号回调 + */ + fun onWakeupSignalReceived() +} + +/** + * BLE服务类，提供核心蓝牙功能 + */ +class BleService(private val context: Context) { + companion object { + private const val TAG = "JieliBleCoreService" + + // 服务和特征UUID + private const val AUDIO_SERVICE_UUID = "0000A100-0000-1000-8000-00805F9B34FB" + private const val AUDIO_DATA_CHAR_UUID = "0000A101-0000-1000-8000-00805F9B34FB" + private const val CONTROL_CHAR_UUID = "0000A102-0000-1000-8000-00805F9B34FB" + private const val CLIENT_CHAR_CONFIG_UUID = "00002902-0000-1000-8000-00805f9b34fb" + + // 连接状态常量 + const val STATE_DISCONNECTED = 0 + const val STATE_CONNECTING = 1 + const val STATE_CONNECTED = 2 + const val STATE_DISCONNECTING = 3 + const val STATE_ERROR = 4 + + // 扫描结果更新间隔(毫秒) + private const val SCAN_RESULT_UPDATE_INTERVAL = 500L + } + + // 蓝牙管理器和适配器 + private val bluetoothManager = context.getSystemService(Context.BLUETOOTH_SERVICE) as BluetoothManager + private val bluetoothAdapter: BluetoothAdapter? = bluetoothManager.adapter + private val bluetoothLeScanner: BluetoothLeScanner? = bluetoothAdapter?.bluetoothLeScanner + + // GATT连接和特征 + private var bluetoothGatt: BluetoothGatt? = null + private var audioDataChar: BluetoothGattCharacteristic? = null + private var controlChar: BluetoothGattCharacteristic? = null + + // 扫描相关 + private val scanHandler = Handler(Looper.getMainLooper()) + private val scanResults = ArrayList() + private var isScanning = false + private var isPendingScanResultUpdate = false + private val scanResultUpdateRunnable = Runnable { + notifyScanResults() + isPendingScanResultUpdate = false + } + + // 连接状态LiveData + val connectionState = MutableLiveData(STATE_DISCONNECTED) + + // 统一回调接口 + var callback: BleCallback? = null + + /** + * 开始扫描BLE设备 + */ + fun startScan(scanTimeout: Long = 5000): Boolean { + if (bluetoothLeScanner == null || isScanning) { + return false + } + + Log.i(TAG, "开始扫描BLE设备...") + scanResults.clear() + + val serviceUuid = UUID.fromString(AUDIO_SERVICE_UUID) + val scanFilter = ScanFilter.Builder() + // .setServiceUuid(ParcelUuid(serviceUuid)) + .build() + val scanFilters = listOf(scanFilter) + + val scanSettings = ScanSettings.Builder() + .setScanMode(ScanSettings.SCAN_MODE_LOW_LATENCY) + .build() + + bluetoothLeScanner.startScan(scanFilters, scanSettings, scanCallback) + isScanning = true + + // 设置超时停止扫描 + scanHandler.postDelayed({ + Log.d(TAG, "定时器触发，准备停止扫描 [主线程:${Looper.myLooper() == Looper.getMainLooper()}]") + stopScan() + }, scanTimeout) + + return true + } + + /** + * 停止扫描 + */ + fun stopScan(): Boolean { + Log.i(TAG, "停止扫描BLE设备") + + if (bluetoothLeScanner == null || !isScanning) { + return false + } + + bluetoothLeScanner.stopScan(scanCallback) + scanHandler.removeCallbacksAndMessages(null) + isScanning = false + + // 最后一次发送扫描结果 + notifyScanResults() + + return true + } + + /** + * 扫描回调 + */ + private val scanCallback = object : ScanCallback() { + override fun onScanResult(callbackType: Int, result: ScanResult) { + val device = result.device + val deviceName = device.name ?: "未知设备" + val deviceAddress = device.address + val rssi = result.rssi + + // Log.d(TAG, "发现设备: $deviceName [$deviceAddress], RSSI: $rssi") + + // 避免重复添加 + val existingIndex = scanResults.indexOfFirst { it.device.address == deviceAddress } + if (existingIndex >= 0) { + scanResults[existingIndex] = result + } else { + scanResults.add(result) + } + + // 如果没有待处理的更新，安排一个 + if (!isPendingScanResultUpdate) { + isPendingScanResultUpdate = true + scanHandler.postDelayed(scanResultUpdateRunnable, SCAN_RESULT_UPDATE_INTERVAL) + } + } + + override fun onScanFailed(errorCode: Int) { + Log.e(TAG, "扫描失败，错误码: $errorCode") + isScanning = false + } + } + + /** + * 将扫描结果通知回调 + */ + private fun notifyScanResults() { + val devicesJson = scanResults.map { result -> + mapOf( + "id" to result.device.address, + "name" to (result.device.name ?: "未知设备"), + "rssi" to result.rssi + ) + } + + callback?.onScanResult(devicesJson) + } + + /** + * 连接到指定设备 + */ + fun connect(deviceAddress: String) { + if (bluetoothAdapter == null) { + Log.e(TAG, "蓝牙适配器不可用") + updateConnectionState(STATE_ERROR) + return + } + + // 先断开现有连接 + bluetoothGatt?.close() + bluetoothGatt = null + + try { + val device = bluetoothAdapter.getRemoteDevice(deviceAddress) + updateConnectionState(STATE_CONNECTING) + + // 连接GATT服务器 + Log.i(TAG, "正在连接设备: ${device.name ?: "未知设备"} [$deviceAddress]") + bluetoothGatt = device.connectGatt(context, false, gattCallback) + } catch (e: Exception) { + Log.e(TAG, "连接设备时发生错误: ${e.message}") + updateConnectionState(STATE_ERROR) + } + } + + /** + * 断开连接 + */ + fun disconnect() { + bluetoothGatt?.disconnect() + updateConnectionState(STATE_DISCONNECTING) + } + + /** + * 更新连接状态并通知回调 + */ + private fun updateConnectionState(state: Int) { + connectionState.postValue(state) + callback?.onConnectionStateChanged(state) + } + + /** + * GATT回调 + */ + private val gattCallback = object : BluetoothGattCallback() { + override fun onConnectionStateChange(gatt: BluetoothGatt, status: Int, newState: Int) { + if (newState == BluetoothProfile.STATE_CONNECTED && status == BluetoothGatt.GATT_SUCCESS) { + Log.i(TAG, "设备已连接，开始发现服务...") + updateConnectionState(STATE_CONNECTED) + + // 开始搜索服务 + gatt.discoverServices() + } else if (newState == BluetoothProfile.STATE_DISCONNECTED) { + Log.i(TAG, "设备已断开连接") + updateConnectionState(STATE_DISCONNECTED) + + // 清理资源 + gatt.close() + bluetoothGatt = null + audioDataChar = null + controlChar = null + } else if (status != BluetoothGatt.GATT_SUCCESS) { + Log.e(TAG, "连接失败，status=$status") + updateConnectionState(STATE_ERROR) + + gatt.close() + bluetoothGatt = null + } + } + + override fun onServicesDiscovered(gatt: BluetoothGatt, status: Int) { + if (status == BluetoothGatt.GATT_SUCCESS) { + Log.i(TAG, "服务发现成功") + + // 查找目标服务 + val audioService = gatt.getService(UUID.fromString(AUDIO_SERVICE_UUID)) + if (audioService != null) { + Log.i(TAG, "找到音频服务") + + // 获取音频数据和控制特征 + audioDataChar = audioService.getCharacteristic(UUID.fromString(AUDIO_DATA_CHAR_UUID)) + controlChar = audioService.getCharacteristic(UUID.fromString(CONTROL_CHAR_UUID)) + + if (audioDataChar != null && controlChar != null) { + Log.i(TAG, "找到所需特征") + + // 注册特征通知 + setupNotifications(gatt) + } else { + Log.e(TAG, "未找到所需特征") + updateConnectionState(STATE_ERROR) + } + } else { + Log.e(TAG, "未找到音频服务") + updateConnectionState(STATE_ERROR) + } + } else { + Log.e(TAG, "服务发现失败，status=$status") + updateConnectionState(STATE_ERROR) + } + } + + override fun onCharacteristicChanged(gatt: BluetoothGatt, characteristic: BluetoothGattCharacteristic) { + val characteristicUuid = characteristic.uuid.toString() + + when (characteristicUuid) { + AUDIO_DATA_CHAR_UUID -> { + // 收到音频数据 + val audioData = characteristic.value + Log.d(TAG, "收到音频数据帧，大小: ${audioData.size} 字节") + + callback?.onAudioDataReceived(audioData) + } + CONTROL_CHAR_UUID -> { + // 收到唤醒信号 + val value = characteristic.value + Log.i(TAG, "收到控制通知: ${value.contentToString()}") + + // 假设0x01表示唤醒信号 + if (value.isNotEmpty() && value[0].toInt() == 0x01) { + Log.i(TAG, "收到唤醒信号") + + callback?.onWakeupSignalReceived() + + // 收到唤醒后自动发送开始录音命令 + sendStartRecordingCommand() + } + } + } + } + + override fun onCharacteristicWrite(gatt: BluetoothGatt, characteristic: BluetoothGattCharacteristic, status: Int) { + if (characteristic.uuid.toString() == CONTROL_CHAR_UUID) { + if (status == BluetoothGatt.GATT_SUCCESS) { + Log.i(TAG, "命令发送成功: ${characteristic.value.contentToString()}") + } else { + Log.e(TAG, "命令发送失败，status=$status") + } + } + } + + override fun onDescriptorWrite(gatt: BluetoothGatt, descriptor: BluetoothGattDescriptor, status: Int) { + if (status == BluetoothGatt.GATT_SUCCESS) { + Log.i(TAG, "描述符写入成功: ${descriptor.uuid}") + + // 如果是音频数据特征的描述符，再订阅控制特征 + if (descriptor.characteristic.uuid.toString() == AUDIO_DATA_CHAR_UUID) { + setupControlNotification(gatt) + } + } else { + Log.e(TAG, "描述符写入失败，status=$status") + } + } + } + + /** + * 设置特征通知 + */ + private fun setupNotifications(gatt: BluetoothGatt) { + // 先订阅音频数据通知 + if (audioDataChar?.properties?.and(BluetoothGattCharacteristic.PROPERTY_NOTIFY) != 0) { + gatt.setCharacteristicNotification(audioDataChar, true) + + val descriptor = audioDataChar?.getDescriptor(UUID.fromString(CLIENT_CHAR_CONFIG_UUID)) + descriptor?.value = BluetoothGattDescriptor.ENABLE_NOTIFICATION_VALUE + + if (descriptor != null) { + gatt.writeDescriptor(descriptor) + Log.i(TAG, "设置音频数据特征通知") + } + } else { + Log.e(TAG, "音频数据特征不支持通知") + setupControlNotification(gatt) + } + } + + /** + * 设置控制特征通知 + */ + private fun setupControlNotification(gatt: BluetoothGatt) { + if (controlChar?.properties?.and(BluetoothGattCharacteristic.PROPERTY_NOTIFY) != 0) { + gatt.setCharacteristicNotification(controlChar, true) + + val descriptor = controlChar?.getDescriptor(UUID.fromString(CLIENT_CHAR_CONFIG_UUID)) + descriptor?.value = BluetoothGattDescriptor.ENABLE_NOTIFICATION_VALUE + + if (descriptor != null) { + gatt.writeDescriptor(descriptor) + Log.i(TAG, "设置控制特征通知") + } + } else { + Log.e(TAG, "控制特征不支持通知") + } + } + + /** + * 发送开始录音命令（向控制特征写入0x01） + */ + fun sendStartRecordingCommand(): Boolean { + if (bluetoothGatt == null || controlChar == null || connectionState.value != STATE_CONNECTED) { + Log.e(TAG, "无法发送开始录音命令，设备未连接") + return false + } + + try { + val commandData = byteArrayOf(0x01) + controlChar?.value = commandData + controlChar?.writeType = BluetoothGattCharacteristic.WRITE_TYPE_DEFAULT + + val success = bluetoothGatt?.writeCharacteristic(controlChar) ?: false + + if (success) { + Log.i(TAG, "开始录音命令已发送") + } else { + Log.e(TAG, "开始录音命令发送失败") + } + + return success + } catch (e: Exception) { + Log.e(TAG, "发送开始录音命令时发生异常: ${e.message}") + return false + } + } + + /** + * 发送停止录音命令（向控制特征写入0x02） + */ + fun sendStopRecordingCommand(): Boolean { + if (bluetoothGatt == null || controlChar == null || connectionState.value != STATE_CONNECTED) { + Log.e(TAG, "无法发送停止录音命令，设备未连接") + return false + } + + try { + val commandData = byteArrayOf(0x02) + controlChar?.value = commandData + controlChar?.writeType = BluetoothGattCharacteristic.WRITE_TYPE_DEFAULT + + val success = bluetoothGatt?.writeCharacteristic(controlChar) ?: false + + if (success) { + Log.i(TAG, "停止录音命令已发送") + } else { + Log.e(TAG, "停止录音命令发送失败") + } + + return success + } catch (e: Exception) { + Log.e(TAG, "发送停止录音命令时发生异常: ${e.message}") + return false + } + } + + /** + * 清理资源 + */ + fun cleanup() { + stopScan() + scanHandler.removeCallbacks(scanResultUpdateRunnable) + bluetoothGatt?.close() + bluetoothGatt = null + audioDataChar = null + controlChar = null + } +} \ No newline at end of file diff --git a/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt b/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt new file mode 100644 index 000000000..17cc5d21f --- /dev/null +++ b/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt @@ -0,0 +1,269 @@ +package com.yunqiinnovation.jieli + +import android.app.Activity +import android.bluetooth.BluetoothAdapter +import android.bluetooth.BluetoothManager +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import androidx.annotation.NonNull +import androidx.core.app.ActivityCompat +import androidx.core.content.ContextCompat +import io.flutter.embedding.engine.plugins.FlutterPlugin +import io.flutter.embedding.engine.plugins.activity.ActivityAware +import io.flutter.embedding.engine.plugins.activity.ActivityPluginBinding +import io.flutter.plugin.common.EventChannel +import io.flutter.plugin.common.MethodCall +import io.flutter.plugin.common.MethodChannel +import io.flutter.plugin.common.MethodChannel.MethodCallHandler +import io.flutter.plugin.common.MethodChannel.Result +import kotlinx.coroutines.* + +/** JieliPlugin */ +class JieliPlugin: FlutterPlugin, MethodCallHandler, ActivityAware, EventChannel.StreamHandler { + /// 方法通道 + private lateinit var methodChannel: MethodChannel + /// 统一事件通道 + private lateinit var eventChannel: EventChannel + + private lateinit var context: Context + private var activity: Activity? = null + + /// BLE服务实例 + private lateinit var bleService: BleService + + // 协程作用域 + private val pluginScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + + // 事件接收器 + private var eventSink: EventChannel.EventSink? = null + + // 事件类型常量 + companion object { + private const val EVENT_TYPE_AUDIO_DATA = "audio_data" + private const val EVENT_TYPE_CONNECTION_STATE = "connection_state" + private const val EVENT_TYPE_SCAN_RESULT = "scan_result" + private const val EVENT_TYPE_WAKEUP_SIGNAL = "wakeup_signal" + } + + // 蓝牙相关请求码 + private val REQUEST_ENABLE_BT = 1 + private val REQUEST_PERMISSION_BLE = 2 + + override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { + val binaryMessenger = flutterPluginBinding.binaryMessenger + + // 设置方法通道 + methodChannel = MethodChannel(binaryMessenger, "com.yunqiinnovation.jieli") + methodChannel.setMethodCallHandler(this) + + // 设置统一事件通道 + eventChannel = EventChannel(binaryMessenger, "com.yunqiinnovation.jieli/events") + eventChannel.setStreamHandler(this) + + context = flutterPluginBinding.applicationContext + + // 初始化BLE服务 + bleService = BleService(context) + + // 设置回调 + setupBleCallbacks() + } + + /** + * 设置BLE服务回调 + */ + private fun setupBleCallbacks() { + // 设置统一回调接口 + bleService.callback = object : BleCallback { + override fun onScanResult(devices: List>) { + pluginScope.launch(Dispatchers.Main) { + sendEvent(EVENT_TYPE_SCAN_RESULT, devices) + } + } + + override fun onConnectionStateChanged(state: Int) { + pluginScope.launch(Dispatchers.Main) { + sendEvent(EVENT_TYPE_CONNECTION_STATE, state) + } + } + + override fun onAudioDataReceived(data: ByteArray) { + pluginScope.launch(Dispatchers.Main) { + sendEvent(EVENT_TYPE_AUDIO_DATA, data) + } + } + + override fun onWakeupSignalReceived() { + pluginScope.launch(Dispatchers.Main) { + sendEvent(EVENT_TYPE_WAKEUP_SIGNAL, true) + } + } + } + } + + /** + * 发送事件到Flutter端 + */ + private fun sendEvent(eventType: String, data: Any?) { + eventSink?.success(mapOf( + "type" to eventType, + "data" to data + )) + } + + override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) { + when (call.method) { + "startScan" -> { + val scanTimeout = call.argument("scanTimeout") ?: 10000 + + if (!hasRequiredPermissions()) { + requestBlePermissions() + result.error("PERMISSION_DENIED", "需要蓝牙权限才能扫描设备", null) + return + } + + val success = bleService.startScan(scanTimeout.toLong()) + result.success(success) + } + "stopScan" -> { + val success = bleService.stopScan() + result.success(success) + } + "connect" -> { + val deviceId = call.argument("deviceId") + if (deviceId == null) { + result.error("INVALID_ARGUMENT", "设备ID不能为空", null) + return + } + + if (!hasRequiredPermissions()) { + requestBlePermissions() + result.error("PERMISSION_DENIED", "需要蓝牙权限才能连接设备", null) + return + } + + bleService.connect(deviceId) + result.success(true) + } + "disconnect" -> { + bleService.disconnect() + result.success(true) + } + "sendStartRecordingCommand" -> { + val success = bleService.sendStartRecordingCommand() + result.success(success) + } + "sendStopRecordingCommand" -> { + val success = bleService.sendStopRecordingCommand() + result.success(success) + } + "isBluetoothEnabled" -> { + val bluetoothManager = context.getSystemService(Context.BLUETOOTH_SERVICE) as BluetoothManager + val bluetoothAdapter = bluetoothManager.adapter + result.success(bluetoothAdapter?.isEnabled == true) + } + "requestBluetoothEnable" -> { + if (activity == null) { + result.error("ACTIVITY_NOT_AVAILABLE", "Activity不可用，无法请求打开蓝牙", null) + return + } + + val bluetoothManager = context.getSystemService(Context.BLUETOOTH_SERVICE) as BluetoothManager + val bluetoothAdapter = bluetoothManager.adapter + + if (bluetoothAdapter == null) { + result.error("BLUETOOTH_UNAVAILABLE", "设备不支持蓝牙", null) + return + } + + if (bluetoothAdapter.isEnabled) { + result.success(true) + return + } + + // 请求打开蓝牙 + val enableBtIntent = Intent(BluetoothAdapter.ACTION_REQUEST_ENABLE) + activity?.startActivityForResult(enableBtIntent, REQUEST_ENABLE_BT) + result.success(true) + } + else -> { + result.notImplemented() + } + } + } + + override fun onDetachedFromEngine(@NonNull binding: FlutterPlugin.FlutterPluginBinding) { + methodChannel.setMethodCallHandler(null) + eventChannel.setStreamHandler(null) + + bleService.cleanup() + pluginScope.cancel() + } + + override fun onAttachedToActivity(binding: ActivityPluginBinding) { + activity = binding.activity + } + + override fun onDetachedFromActivityForConfigChanges() { + activity = null + } + + override fun onReattachedToActivityForConfigChanges(binding: ActivityPluginBinding) { + activity = binding.activity + } + + override fun onDetachedFromActivity() { + activity = null + } + + /// EventChannel.StreamHandler接口实现 + override fun onListen(arguments: Any?, events: EventChannel.EventSink?) { + eventSink = events + } + + override fun onCancel(arguments: Any?) { + eventSink = null + } + + /// 检查是否拥有所需权限 + private fun hasRequiredPermissions(): Boolean { + return if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.S) { + hasPermission(android.Manifest.permission.BLUETOOTH_SCAN) && + hasPermission(android.Manifest.permission.BLUETOOTH_CONNECT) + } else { + hasPermission(android.Manifest.permission.BLUETOOTH) && + hasPermission(android.Manifest.permission.BLUETOOTH_ADMIN) + } + } + + /// 检查单个权限 + private fun hasPermission(permission: String): Boolean { + return ContextCompat.checkSelfPermission(context, permission) == PackageManager.PERMISSION_GRANTED + } + + /// 请求BLE权限 + private fun requestBlePermissions() { + activity?.let { + if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.S) { + ActivityCompat.requestPermissions( + it, + arrayOf( + android.Manifest.permission.BLUETOOTH_SCAN, + android.Manifest.permission.BLUETOOTH_CONNECT + ), + REQUEST_PERMISSION_BLE + ) + } else { + ActivityCompat.requestPermissions( + it, + arrayOf( + android.Manifest.permission.BLUETOOTH, + android.Manifest.permission.BLUETOOTH_ADMIN + ), + REQUEST_PERMISSION_BLE + ) + } + } + } +} \ No newline at end of file diff --git a/local_plugins/jieli/lib/jieli.dart b/local_plugins/jieli/lib/jieli.dart new file mode 100644 index 000000000..78ec090bb --- /dev/null +++ b/local_plugins/jieli/lib/jieli.dart @@ -0,0 +1,188 @@ +import 'dart:async'; +import 'dart:typed_data'; + +import 'package:flutter/services.dart'; + +/// 设备类，代表一个BLE设备 +class BleDevice { + final String id; + final String name; + final int rssi; + + BleDevice({required this.id, required this.name, required this.rssi}); + + factory BleDevice.fromJson(Map json) { + return BleDevice( + id: json['id'] as String, + name: json['name'] as String? ?? 'Unknown Device', + rssi: json['rssi'] as int? ?? 0, + ); + } + + Map toJson() { + return { + 'id': id, + 'name': name, + 'rssi': rssi, + }; + } + + @override + String toString() => 'BleDevice(id: $id, name: $name, rssi: $rssi)'; +} + +/// 连接状态 +enum ConnectionState { + disconnected, + connecting, + connected, + disconnecting, + error +} + +/// 事件类型 +class EventType { + static const String audioData = 'audio_data'; + static const String connectionState = 'connection_state'; + static const String scanResult = 'scan_result'; + static const String wakeupSignal = 'wakeup_signal'; +} + +/// 杰理BLE插件主类 +class Jieli { + static const MethodChannel _methodChannel = MethodChannel('com.yunqiinnovation.jieli'); + static const EventChannel _eventChannel = EventChannel('com.yunqiinnovation.jieli/events'); + + /// 扫描结果流 + static final StreamController> _scanController = + StreamController>.broadcast(); + Stream> get scanResults => _scanController.stream; + + /// 连接状态变化流 + static final StreamController _connectionController = + StreamController.broadcast(); + Stream get connectionState => _connectionController.stream; + + /// 音频数据流 + static final StreamController _audioController = + StreamController.broadcast(); + Stream get audioDataStream => _audioController.stream; + + /// 唤醒信号流 + static final StreamController _wakeupController = + StreamController.broadcast(); + Stream get wakeupStream => _wakeupController.stream; + + /// 事件流订阅 + StreamSubscription? _eventSubscription; + + /// 构造函数，初始化平台通道和事件监听 + Jieli() { + _setupEventListener(); + } + + /// 设置事件监听器 + void _setupEventListener() { + _eventSubscription = _eventChannel.receiveBroadcastStream().listen(_handleEvent); + } + + /// 处理来自原生平台的事件 + void _handleEvent(dynamic event) { + if (event is! Map) return; + + final eventData = Map.from(event); + final String eventType = eventData['type'] as String; + final dynamic data = eventData['data']; + + switch (eventType) { + case EventType.scanResult: + if (data is List) { + final deviceList = data + .map((item) => BleDevice.fromJson(Map.from(item))) + .toList(); + _scanController.add(deviceList); + } + break; + case EventType.connectionState: + if (data is int && data >= 0 && data < ConnectionState.values.length) { + _connectionController.add(ConnectionState.values[data]); + } + break; + case EventType.audioData: + if (data is Uint8List) { + _audioController.add(data); + } + break; + case EventType.wakeupSignal: + if (data is bool) { + _wakeupController.add(data); + } + break; + } + } + + /// 开始扫描BLE设备 + Future startScan({int scanTimeout = 10000}) async { + final result = await _methodChannel.invokeMethod( + 'startScan', + {'scanTimeout': scanTimeout}, + ); + return result ?? false; + } + + /// 停止扫描BLE设备 + Future stopScan() async { + final result = await _methodChannel.invokeMethod('stopScan'); + return result ?? false; + } + + /// 连接到指定设备 + Future connect(String deviceId) async { + final result = await _methodChannel.invokeMethod( + 'connect', + {'deviceId': deviceId}, + ); + return result ?? false; + } + + /// 断开连接 + Future disconnect() async { + final result = await _methodChannel.invokeMethod('disconnect'); + return result ?? false; + } + + /// 发送开始录音命令 + Future sendStartRecordingCommand() async { + final result = await _methodChannel.invokeMethod('sendStartRecordingCommand'); + return result ?? false; + } + + /// 发送停止录音命令 + Future sendStopRecordingCommand() async { + final result = await _methodChannel.invokeMethod('sendStopRecordingCommand'); + return result ?? false; + } + + /// 关闭插件，释放资源 + Future dispose() async { + await stopScan(); + await disconnect(); + _eventSubscription?.cancel(); + _scanController.close(); + _connectionController.close(); + _audioController.close(); + _wakeupController.close(); + } + + /// 检查蓝牙是否已启用 + Future isBluetoothEnabled() async { + final result = await _methodChannel.invokeMethod('isBluetoothEnabled'); + return result ?? false; + } + + /// 请求打开蓝牙 + Future requestBluetoothEnable() async { + final result = await _methodChannel.invokeMethod('requestBluetoothEnable'); + return result ?? false; + } +} \ No newline at end of file diff --git a/local_plugins/jieli/pubspec.yaml b/local_plugins/jieli/pubspec.yaml new file mode 100644 index 000000000..ab230e685 --- /dev/null +++ b/local_plugins/jieli/pubspec.yaml @@ -0,0 +1,23 @@ +name: jieli +description: A Flutter plugin for connecting with Jieli BLE earphones +version: 0.0.1 +homepage: + +environment: + sdk: ">=2.12.0 <3.0.0" + flutter: ">=2.0.0" + +dependencies: + flutter: + sdk: flutter + +dev_dependencies: + flutter_test: + sdk: flutter + +flutter: + plugin: + platforms: + android: + package: com.yunqiinnovation.jieli + pluginClass: JieliPlugin \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/AndroidManifest.xml b/local_plugins/open_ai_service/android/AndroidManifest.xml new file mode 100644 index 000000000..1d834e578 --- /dev/null +++ b/local_plugins/open_ai_service/android/AndroidManifest.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/build.gradle.kts b/local_plugins/open_ai_service/android/build.gradle.kts index 4f7b2f643..57889671a 100644 --- a/local_plugins/open_ai_service/android/build.gradle.kts +++ b/local_plugins/open_ai_service/android/build.gradle.kts @@ -3,6 +3,7 @@ plugins { id("com.android.library") // Kotlin Android 插件 id("org.jetbrains.kotlin.android") + id("com.github.johnrengelman.shadow") version "8.1.1" } android { @@ -28,10 +29,27 @@ android { kotlinOptions { jvmTarget = "11" } + + // 添加packaging配置，排除冲突文件 + packaging { + resources { + excludes.add("META-INF/DEPENDENCIES") + excludes.add("META-INF/LICENSE") + excludes.add("META-INF/LICENSE.txt") + excludes.add("META-INF/license.txt") + excludes.add("META-INF/NOTICE") + excludes.add("META-INF/NOTICE.txt") + excludes.add("META-INF/notice.txt") + excludes.add("META-INF/*.kotlin_module") + } + } } dependencies { - - implementation("com.squareup.okhttp3:okhttp:4.10.0") - + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.6.4") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4") + implementation("com.fasterxml.jackson.core:jackson-databind:2.14.2") + implementation("com.squareup.okhttp3:okhttp:4.9.3") + implementation("com.anthropic:anthropic-java:0.8.0") + implementation("io.modelcontextprotocol:kotlin-sdk:0.4.0") } \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt new file mode 100644 index 000000000..fef91ab05 --- /dev/null +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt @@ -0,0 +1,496 @@ +package com.yunqiinnovation.open_ai_service + +import android.content.Context +import android.util.Log +import com.anthropic.core.JsonValue +import com.anthropic.models.messages.Tool +import com.anthropic.models.messages.ToolUnion +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import io.ktor.client.HttpClient +import io.modelcontextprotocol.kotlin.sdk.Implementation +import io.modelcontextprotocol.kotlin.sdk.TextContent +import io.modelcontextprotocol.kotlin.sdk.client.Client +import io.modelcontextprotocol.kotlin.sdk.client.WebSocketClientTransport +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.JsonObject +import org.json.JSONObject +import kotlin.jvm.optionals.getOrNull + +/** + * 工具类型枚举 + */ +enum class ToolType { + LOCAL_FUNCTION, // 本地函数 + MCP_TOOL // MCP工具 +} + +/** + * 函数处理器接口 + */ +interface FunctionHandler { + /** + * 处理函数调用 + * @param arguments 函数参数，Map格式 + * @return 函数执行结果，字符串 + */ + suspend fun handle(arguments: Map): String +} + +/** + * MCP客户端，用于连接Model Context Protocol服务器和管理本地函数 + */ +class MCPClient(private val context: Context? = null) : AutoCloseable { + private val TAG = "MCPClient" + + // 初始化MCP客户端 + private val mcp: Client = Client( + clientInfo = Implementation( + name = "deep-voice-mcp-client", + version = "1.0.0" + ) + ) + + // 服务器提供的工具列表 + private var tools: List = emptyList() + + // 工具Map列表，用于OpenAI请求 + private val toolMaps = mutableListOf>() + + // 本地函数Map，函数名 -> 处理器 + private val localFunctions = mutableMapOf() + + // 本地函数定义Map，函数名 -> 定义 + private val localFunctionDefs = mutableMapOf() + + // 是否已连接 + private var isConnected = false + + // 系统功能处理器 + private val systemFunctionHandler = SystemFunctionHandler(context!!) + + init { + // 如果提供了Context，自动注册语音功能处理器 + context?.let { initializeVoiceFunctions(it) } + } + + /** + * 初始化语音功能处理器 + */ + private fun initializeVoiceFunctions(context: Context) { + try { + // 使用SystemFunctionHandler注册所有功能 + systemFunctionHandler.registerAllFunctions(this) + + Log.d(TAG, "语音功能初始化完成") + } catch (e: Exception) { + Log.e(TAG, "语音功能初始化失败: ${e.message}", e) + } + } + + /** + * 将JsonObject转换为JsonValue + */ + private fun JsonObject.toJsonValue(): JsonValue { + val mapper = ObjectMapper() + val node = mapper.readTree(this.toString()) + return JsonValue.fromJsonNode(node) + } + + /** + * 连接到MCP服务器(通过WebSocket) + */ + suspend fun connectToWebSocket(serverUrl: String): Boolean { + try { + // 创建WebSocket传输层 + val transport = WebSocketClientTransport( + client = HttpClient(), + urlString = serverUrl + ) + + // 连接MCP客户端到服务器 + mcp.connect(transport) + + // 获取服务器提供的工具列表 + val toolsResult = mcp.listTools() + Log.d(TAG, "获取到工具列表: $toolsResult") + + // 处理工具列表 + processToolsList(toolsResult) + + isConnected = true + Log.d(TAG, "已连接到WebSocket服务器，可用工具: ${tools.map { it.tool().get().name() }.joinToString(", ")}") + return true + } catch (e: Exception) { + Log.e(TAG, "连接WebSocket服务器失败: ${e.message}", e) + return false + } + } + + /** + * 处理工具列表 + */ + private fun processToolsList(toolsResult: io.modelcontextprotocol.kotlin.sdk.ListToolsResult?) { + tools = toolsResult?.tools?.map { tool -> + // 确保输入模式有效 + val inputType = tool.inputSchema?.type ?: "object" + val inputProperties = tool.inputSchema?.properties ?: emptyMap() + val inputRequired = tool.inputSchema?.required ?: emptyList() + + ToolUnion.ofTool( + Tool.builder() + .name(tool.name) + .description(tool.description ?: "") + .inputSchema( + Tool.InputSchema.builder() + .type(JsonValue.from(inputType)) + .properties(if (inputProperties is JsonObject) inputProperties.toJsonValue() else JsonValue.from(emptyMap())) + .putAdditionalProperty("required", JsonValue.from(inputRequired)) + .build() + ) + .build() + ) + } ?: emptyList() + + // 更新用于OpenAI的工具映射 + updateToolMaps() + } + + + /** + * 注册本地函数 + * @param name 函数名称 + * @param description 函数描述 + * @param parameters 函数参数定义（Map或JSONObject） + * @param handler 函数处理器 + * @return 是否注册成功 + */ + fun registerLocalFunction( + name: String, + description: String, + parameters: Any, + handler: FunctionHandler + ): Boolean { + try { + // 将参数统一转换为Map格式 + val parametersMap: Map = when (parameters) { + is Map<*, *> -> { + @Suppress("UNCHECKED_CAST") + parameters as Map + } + is JSONObject -> { + val mapper = ObjectMapper() + mapper.readValue( + parameters.toString(), + object : TypeReference>() {} + ) + } + else -> { + Log.e(TAG, "参数类型不支持: ${parameters.javaClass.name}") + return false + } + } + + // 检查参数是否包含必要字段 + if (!parametersMap.containsKey("type") || (parametersMap["type"] != "object")) { + Log.e(TAG, "参数必须是object类型") + return false + } + + // 注册函数处理器 + localFunctions[name] = handler + + // 创建函数定义 + val functionDef = mapOf( + "name" to name, + "description" to description, + "parameters" to parametersMap + ) + + // 保存函数定义 + localFunctionDefs[name] = ObjectMapper().writeValueAsString(functionDef) + + // 更新工具映射 + updateToolMaps() + + Log.d(TAG, "注册本地函数成功: $name") + return true + } catch (e: Exception) { + Log.e(TAG, "注册本地函数失败: ${e.message}", e) + return false + } + } + + /** + * 取消注册本地函数 + * @param name 函数名称 + * @return 是否取消成功 + */ + fun unregisterLocalFunction(name: String): Boolean { + val removed = localFunctions.remove(name) != null + if (removed) { + localFunctionDefs.remove(name) + updateToolMaps() + Log.d(TAG, "取消注册本地函数成功: $name") + } + return removed + } + + /** + * 更新工具Map列表，用于OpenAI请求 + */ + private fun updateToolMaps() { + toolMaps.clear() + + // 首先添加本地函数 + localFunctionDefs.forEach { (name, functionDefJson) -> + try { + val mapper = ObjectMapper() + val functionDef = mapper.readValue(functionDefJson, object : TypeReference>() {}) + val toolMap = mapOf( + "type" to "function", + "function" to functionDef + ) + toolMaps.add(toolMap) + } catch (e: Exception) { + Log.e(TAG, "解析本地函数定义失败: $name, ${e.message}", e) + } + } + + // 然后添加MCP工具 + tools.forEach { toolUnion -> + try { + val tool = toolUnion.tool().get() + val parametersMap = mutableMapOf() + + // 提取工具类型及参数 + try { + val inputSchema = tool.inputSchema() + // 解析JsonValue + val mapper = ObjectMapper() + val inputSchemaStr = inputSchema.toString() + val inputSchemaJson = mapper.readTree(inputSchemaStr) + val typeValue = inputSchemaJson.path("type") + parametersMap["type"] = if (typeValue.isTextual) typeValue.asText() else "object" + + // 提取properties字段 + val propertiesMap = mutableMapOf() + val propertiesNode = inputSchemaJson.path("properties") + if (propertiesNode.isObject) { + propertiesNode.fieldNames().forEach { fieldName -> + val fieldNode = propertiesNode.get(fieldName) + propertiesMap[fieldName] = convertJsonNodeToMap(fieldNode) + } + } + parametersMap["properties"] = propertiesMap + + // 提取required字段 + val requiredList = mutableListOf() + val requiredNode = inputSchemaJson.path("required") + if (requiredNode.isArray) { + for (i in 0 until requiredNode.size()) { + if (requiredNode.get(i).isTextual) { + requiredList.add(requiredNode.get(i).asText()) + } + } + } + parametersMap["required"] = requiredList + } catch (e: Exception) { + Log.e(TAG, "解析工具参数失败: ${e.message}", e) + // 设置默认参数 + parametersMap["type"] = "object" + parametersMap["properties"] = emptyMap() + parametersMap["required"] = emptyList() + } + + // 创建工具Map + val toolMap = mapOf( + "type" to "function", + "function" to mapOf( + "name" to tool.name(), + "description" to (tool.description() ?: ""), + "parameters" to parametersMap + ) + ) + + toolMaps.add(toolMap) + } catch (e: Exception) { + Log.e(TAG, "转换工具到Map失败: ${e.message}", e) + } + } + } + + /** + * 将JsonNode转换为Map + */ + private fun convertJsonNodeToMap(node: JsonNode): Any { + return when { + node.isTextual() -> node.asText() + node.isNumber() -> node.asDouble() + node.isBoolean() -> node.asBoolean() + node.isObject() -> { + val result = mutableMapOf() + node.fieldNames().forEach { fieldName -> + result[fieldName] = convertJsonNodeToMap(node.get(fieldName)) + } + result + } + node.isArray() -> { + val result = mutableListOf() + for (i in 0 until node.size()) { + result.add(convertJsonNodeToMap(node.get(i))) + } + result + } + node.isNull() -> "null" + else -> node.toString() + } + } + + /** + * 获取工具Map列表，用于OpenAI请求 + */ + fun getToolMaps(): List> { + return toolMaps + } + + /** + * 获取工具类型 + * @param name 工具名称 + * @return 工具类型，如果不存在则返回null + */ + fun getToolType(name: String): ToolType? { + if (localFunctions.containsKey(name)) { + return ToolType.LOCAL_FUNCTION + } + + if (tools.any { it.tool().get().name() == name }) { + return ToolType.MCP_TOOL + } + + return null + } + + /** + * 调用工具（统一入口） + * @param name 工具名称 + * @param arguments 参数 + * @return 调用结果 + */ + suspend fun callTool(name: String, arguments: Map): String? { + val toolType = getToolType(name) + + return when (toolType) { + ToolType.LOCAL_FUNCTION -> { + callLocalFunction(name, arguments) + } + ToolType.MCP_TOOL -> { + callMcpTool(name, arguments) + } + null -> { + Log.e(TAG, "未找到工具: $name") + null + } + } + } + + /** + * 调用本地函数 + */ + private suspend fun callLocalFunction(name: String, arguments: Map): String? { + return try { + val handler = localFunctions[name] + if (handler != null) { + handler.handle(arguments) + } else { + Log.e(TAG, "未找到本地函数: $name") + null + } + } catch (e: Exception) { + Log.e(TAG, "调用本地函数失败: ${e.message}", e) + "调用本地函数失败: ${e.message}" + } + } + + /** + * 调用MCP工具 + */ + private suspend fun callMcpTool(name: String, arguments: Map): String? { + if (!isConnected) { + Log.e(TAG, "未连接到MCP服务器", null) + return "未连接到MCP服务器" + } + + return try { + val result = mcp.callTool( + name = name, + arguments = arguments + ) + + // 处理工具调用结果 + result?.content?.joinToString("\n") { content -> + when (content) { + is TextContent -> content.text ?: "" + else -> "" + } + } + } catch (e: Exception) { + Log.e(TAG, "调用MCP工具失败: ${e.message}", e) + "调用MCP工具失败: ${e.message}" + } + } + + /** + * 获取可用的工具列表 + */ + fun getTools(): List { + return tools + } + + /** + * 检查工具是否存在 + */ + fun hasToolWithName(name: String): Boolean { + return localFunctions.containsKey(name) || toolMaps.any { + val functionMap = it["function"] as? Map + functionMap?.get("name") == name + } + } + + /** + * 从字符串解析JSON参数 + */ + fun parseJsonArguments(json: String): Map { + return try { + ObjectMapper().readValue(json, object : TypeReference>() {}) + } catch (e: Exception) { + Log.e(TAG, "解析JSON参数失败: ${e.message}", e) + emptyMap() + } + } + + /** + * 检查是否已连接到MCP服务器 + */ + fun isConnected(): Boolean { + return isConnected + } + + /** + * 关闭连接 + */ + override fun close() { + runBlocking { + try { + mcp.close() + isConnected = false + tools = emptyList() + toolMaps.clear() + Log.d(TAG, "已关闭MCP服务器连接", null) + } catch (e: Exception) { + Log.e(TAG, "关闭MCP服务器连接失败: ${e.message}", e) + } + } + } +} \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt index 907be93a2..9e67d15f8 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt @@ -1,6 +1,13 @@ package com.yunqiinnovation.open_ai_service +import android.content.Context import android.util.Log +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext import okhttp3.* import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.RequestBody.Companion.toRequestBody @@ -9,11 +16,13 @@ import org.json.JSONObject import java.io.IOException import java.util.concurrent.TimeUnit + /** * OpenAI服务的原生实现 */ -class OpenAIService() { +class OpenAIService(private val context: Context? = null) { private val TAG = "OpenAIService" + private var baseUrl = "" private val client = OkHttpClient.Builder() .connectTimeout(30, TimeUnit.SECONDS) @@ -25,8 +34,12 @@ class OpenAIService() { private var isInitialized = false private var model: String = "" // 默认模型 - // 用于存储注册的函数 - private val registeredFunctions = mutableListOf() + // MCP客户端 + private var mcpClient: MCPClient? = null + private var isMcpInitialized = false + + // 是否自动处理MCP工具调用 + private var autoHandleMcpTools = true /** * 构建curl命令用于测试 @@ -88,40 +101,151 @@ class OpenAIService() { if (model.isNotEmpty()) { this.model = model } + + // 初始化MCPClient + if (mcpClient == null) { + mcpClient = MCPClient(context) + } + isInitialized = apiKey.isNotEmpty() return isInitialized } + /** + * MCP客户端是否已初始化 + */ + fun isMcpInitialized(): Boolean { + return isMcpInitialized && mcpClient?.isConnected() == true + } + + /** + * 关闭MCP客户端 + */ + fun closeMcpClient() { + mcpClient?.close() + mcpClient = null + isMcpInitialized = false + } + + /** + * 初始化MCP客户端（WebSocket方式） + */ + fun initializeMcpClient(serverUrl: String): Boolean { + if (mcpClient != null) { + mcpClient?.close() + } + + mcpClient = MCPClient(context) + + // 在后台线程中初始化MCP客户端 + val scope = CoroutineScope(Dispatchers.IO) + + var result = false + scope.launch { + try { + result = mcpClient?.connectToWebSocket(serverUrl) ?: false + isMcpInitialized = result + Log.d(TAG, "MCP客户端初始化${if (result) "成功" else "失败"}") + } catch (e: Exception) { + Log.e(TAG, "MCP客户端初始化失败: ${e.message}", e) + isMcpInitialized = false + } + } + + return true // 立即返回，实际连接在后台进行 + } + + + + /** + * 处理MCP工具调用 + * + * @param functionCall 函数调用JSON对象 + * @return 处理结果，如果处理失败则返回null + */ + suspend fun handleMcpToolCall(functionCall: JSONObject): String? { + if (mcpClient == null || !isMcpInitialized) { + return "MCP客户端未初始化" + } + + try { + // 获取函数名称 + val name = functionCall.getString("name") + + // 获取参数 + val argumentsJson = functionCall.getString("arguments") + val arguments = mcpClient?.parseJsonArguments(argumentsJson) ?: mapOf() + + // 调用工具 + return mcpClient?.callTool(name, arguments) + } catch (e: Exception) { + Log.e(TAG, "处理MCP工具调用失败: ${e.message}", e) + return "处理MCP工具调用失败: ${e.message}" + } + } + /** * 注册函数 */ fun registerFunction(name: String, description: String, parameters: JSONObject): Boolean { try { - val function = JSONObject().apply { - put("name", name) - put("description", description) - put("parameters", parameters) - } - - // 检查是否已存在相同名称的函数 - val existingIndex = registeredFunctions.indexOfFirst { - it.getString("name") == name + // 确保MCPClient已初始化 + if (mcpClient == null) { + mcpClient = MCPClient() } - if (existingIndex >= 0) { - // 如果已存在，则替换 - registeredFunctions[existingIndex] = function - } else { - // 如果不存在，则添加 - registeredFunctions.add(function) + // 创建函数处理器 + val handler = object : FunctionHandler { + override suspend fun handle(arguments: Map): String { + // 由于本地函数的实际处理是在Flutter端完成的 + // 这里只需返回一个标记，表示该函数是本地函数 + return "LOCAL_FUNCTION:$name" + } } - return true + // 直接委托给MCPClient处理 + return mcpClient?.registerLocalFunction(name, description, parameters, handler) ?: false } catch (e: Exception) { + Log.e(TAG, "注册函数失败: ${e.message}", e) return false } } + /** + * 自动处理MCP工具调用 + */ + private suspend fun autoHandleMcpToolCall(functionCall: JSONObject, messages: JSONArray, callback: StreamCallback) { + try { + // 获取函数名称 + val name = functionCall.getString("name") + + // 获取参数 + val argumentsJson = functionCall.getString("arguments") + val arguments = mcpClient?.parseJsonArguments(argumentsJson) ?: mapOf() + + // 调用工具（统一通过MCPClient处理） + val result = mcpClient?.callTool(name, arguments) + + // 不论是本地函数还是MCP工具，都直接将结果发送回OpenAI + sendFunctionCallResult( + messages = messages, + functionCall = functionCall, + functionResult = result ?: "工具调用失败", + callback = callback + ) + } catch (e: Exception) { + Log.e(TAG, "自动处理工具调用失败: ${e.message}", e) + // 失败时返回错误给回调函数 + val errorMessage = "工具调用失败: ${e.message}" + sendFunctionCallResult( + messages = messages, + functionCall = functionCall, + functionResult = errorMessage, + callback = callback + ) + } + } + /** * 发送消息（非流式输出） */ @@ -138,16 +262,21 @@ class OpenAIService() { put("max_tokens", 2000) put("stream", false) - // 如果有注册的函数，则添加到请求中 - if (registeredFunctions.isNotEmpty()) { - val tools = JSONArray() - for (function in registeredFunctions) { - val tool = JSONObject().apply { - put("type", "function") - put("function", function) - } + // 添加工具列表 + val tools = JSONArray() + + // 使用MCPClient提供的所有工具 + mcpClient?.getToolMaps()?.forEach { toolMap -> + try { + val tool = JSONObject(toolMap) tools.put(tool) + } catch (e: Exception) { + Log.e(TAG, "转换工具失败: ${e.message}", e) } + } + + // 如果有工具，则添加到请求中 + if (tools.length() > 0) { put("tools", tools) } } @@ -231,16 +360,21 @@ class OpenAIService() { put("max_tokens", 2000) put("stream", true) - // 如果有注册的函数，则添加到请求中 - if (registeredFunctions.isNotEmpty()) { - val tools = JSONArray() - for (function in registeredFunctions) { - val tool = JSONObject().apply { - put("type", "function") - put("function", function) - } + // 添加工具列表 + val tools = JSONArray() + + // 使用MCPClient提供的所有工具 + mcpClient?.getToolMaps()?.forEach { toolMap -> + try { + val tool = JSONObject(toolMap) tools.put(tool) + } catch (e: Exception) { + Log.e(TAG, "转换工具失败: ${e.message}", e) } + } + + // 如果有工具，则添加到请求中 + if (tools.length() > 0) { put("tools", tools) } } @@ -289,7 +423,7 @@ class OpenAIService() { // 处理[DONE]消息 if (data == "[DONE]" || data == "[\"DONE\"]") { - processToolCalls(finalToolCalls, callback) + processToolCalls(finalToolCalls, callback, messages) callback.onComplete() break } @@ -403,12 +537,6 @@ class OpenAIService() { put("tool_call_id", functionCall.optString("id", "call_${System.currentTimeMillis()}")) }) - // 添加一个带有content的assistant消息，确保API请求不会因为缺少content而失败 - fullMessages.put(JSONObject().apply { - put("role", "assistant") - put("content", "") // 空内容，让模型生成新的回复 - }) - // 发送完整对话 sendMessageStream(fullMessages, callback) @@ -418,9 +546,9 @@ class OpenAIService() { } /** - * 处理工具调用（处理完整的函数调用并回调） + * 处理工具调用结果 */ - private fun processToolCalls(toolCalls: Map, callback: StreamCallback) { + private fun processToolCalls(toolCalls: Map, callback: StreamCallback, messages: JSONArray? = null) { if (toolCalls.isEmpty()) return // 只处理第一个工具调用 @@ -434,8 +562,16 @@ class OpenAIService() { put("id", firstToolCall.id) } - // 回调 - callback.onFunctionCall(functionCall) + // 在协程中处理工具调用 + if (messages != null) { + kotlinx.coroutines.GlobalScope.launch { + try { + autoHandleMcpToolCall(functionCall, messages, callback) + } catch (e: Exception) { + Log.e(TAG, "工具调用处理失败: ${e.message}", e) + } + } + } } } diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt index 70af36c13..1e1bb9446 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt @@ -15,6 +15,10 @@ import org.json.JSONArray import org.json.JSONObject import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext /** OpenAIServicePlugin */ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { @@ -34,7 +38,7 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { private lateinit var context: Context /// OpenAI服务实例 - private val openAIService = OpenAIService() + private lateinit var openAIService: OpenAIService /// 事件接收器（用于流式输出） private var eventSink: EventSink? = null @@ -46,6 +50,9 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { // 保存上下文 context = flutterPluginBinding.applicationContext + // 初始化OpenAI服务 + openAIService = OpenAIService(context) + // 初始化方法通道 methodChannel = MethodChannel(flutterPluginBinding.binaryMessenger, methodChannelName) methodChannel.setMethodCallHandler(this) @@ -66,6 +73,7 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { result.success(initialized) } + "registerFunction" -> { val name = call.argument("name") ?: "" val description = call.argument("description") ?: "" @@ -256,6 +264,11 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { result.success(message.toMap()) } + "initializeMcpClient" -> initializeMcpClient(call, result) + "isMcpInitialized" -> isMcpInitialized(call, result) + "closeMcpClient" -> closeMcpClient(call, result) + "handleMcpToolCall" -> handleMcpToolCall(call, result) + else -> { result.notImplemented() } @@ -314,4 +327,54 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { } return list } + + private fun initializeMcpClient(call: MethodCall, result: Result) { + val serverUrl = call.argument("serverUrl") + if (serverUrl == null) { + result.error("INVALID_ARGUMENT", "Missing serverUrl", null) + return + } + + val success = openAIService.initializeMcpClient(serverUrl) + result.success(success) + } + + private fun isMcpInitialized(call: MethodCall, result: Result) { + result.success(openAIService.isMcpInitialized()) + } + + private fun closeMcpClient(call: MethodCall, result: Result) { + openAIService.closeMcpClient() + result.success(true) + } + + private fun handleMcpToolCall(call: MethodCall, result: Result) { + val functionCall = call.argument("functionCall") + if (functionCall == null) { + result.error("INVALID_ARGUMENT", "Missing functionCall", null) + return + } + + try { + val jsonFunctionCall = JSONObject(functionCall) + + // 使用协程处理异步操作 + CoroutineScope(Dispatchers.IO).launch { + try { + val toolResult = openAIService.handleMcpToolCall(jsonFunctionCall) + + // 切换到主线程返回结果 + withContext(Dispatchers.Main) { + result.success(toolResult ?: "无法处理MCP工具调用") + } + } catch (e: Exception) { + withContext(Dispatchers.Main) { + result.error("MCP_TOOL_ERROR", "处理MCP工具调用失败: ${e.message}", null) + } + } + } + } catch (e: Exception) { + result.error("INVALID_ARGUMENT", "解析functionCall失败: ${e.message}", null) + } + } } \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/SystemFunctionHandler.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/SystemFunctionHandler.kt new file mode 100644 index 000000000..f2e7cbcdb --- /dev/null +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/SystemFunctionHandler.kt @@ -0,0 +1,422 @@ +package com.yunqiinnovation.open_ai_service + +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.net.Uri +import android.provider.ContactsContract +import android.telephony.SmsManager +import android.util.Log +import org.json.JSONObject + +/** + * 系统功能处理器 - 负责调用系统API执行各种功能以及注册系统功能 + */ +class SystemFunctionHandler(private val context: Context) { + companion object { + private const val TAG = "SystemFunctionHandler" + const val ACTION_EXIT_INTERACTION = "com.yunqiinnovation.deepsound.action.EXIT_INTERACTION" + } + + /** + * 在MCPClient中注册所有系统功能 + * @param client MCPClient实例 + */ + fun registerAllFunctions(client: MCPClient) { + try { + // 注册退出交互函数 + registerExitInteractionFunction(client) + + // 注册翻译模式函数 + registerTranslationModeFunction(client) + + // 注册发送短信函数 + registerSendTextMessageFunction(client) + + // 注册拨打电话函数 + registerMakePhoneCallFunction(client) + + // 注册打开应用函数 + registerOpenAppFunction(client) + + Log.d(TAG, "所有系统功能注册完成") + } catch (e: Exception) { + Log.e(TAG, "注册系统功能失败: ${e.message}", e) + } + } + + + /** + * 注册退出交互函数 + */ + private fun registerExitInteractionFunction(client: MCPClient) { + client.registerLocalFunction( + "exit_interaction", + "退出当前语音交互", + JSONObject(""" + { + "type": "object", + "properties": {}, + "required": [] + } + """), + object : FunctionHandler { + override suspend fun handle(arguments: Map): String { + // 调用发送广播的函数 + sendExitInteractionBroadcast() + + return "{\"result\": \"已退出语音交互\"}" + } + } + ) + Log.d(TAG, "已注册退出交互功能") + } + + /** + * 注册翻译模式函数 + */ + private fun registerTranslationModeFunction(client: MCPClient) { + client.registerLocalFunction( + "enter_translation_mode", + "进入实时翻译模式", + JSONObject(""" + { + "type": "object", + "properties": {}, + "required": [] + } + """), + object : FunctionHandler { + override suspend fun handle(arguments: Map): String { + val result = enterTranslationMode() + return "{\"result\": \"${result.message}\"}" + } + } + ) + Log.d(TAG, "已注册翻译模式功能") + } + + /** + * 注册发送短信函数 + */ + private fun registerSendTextMessageFunction(client: MCPClient) { + client.registerLocalFunction( + "send_text_message", + "发送短信", + JSONObject(""" + { + "type": "object", + "properties": { + "contact": { + "type": "string", + "description": "联系人姓名或电话号码" + }, + "message": { + "type": "string", + "description": "短信内容" + } + }, + "required": ["contact", "message"] + } + """), + object : FunctionHandler { + override suspend fun handle(arguments: Map): String { + val contact = arguments["contact"] as? String ?: "" + val message = arguments["message"] as? String ?: "" + val result = sendTextMessage(contact, message) + return "{\"result\": \"${result.message}\"}" + } + } + ) + Log.d(TAG, "已注册发送短信功能") + } + + /** + * 注册拨打电话函数 + */ + private fun registerMakePhoneCallFunction(client: MCPClient) { + client.registerLocalFunction( + "make_phone_call", + "拨打电话", + JSONObject(""" + { + "type": "object", + "properties": { + "contact": { + "type": "string", + "description": "联系人姓名或电话号码" + } + }, + "required": ["contact"] + } + """), + object : FunctionHandler { + override suspend fun handle(arguments: Map): String { + val contact = arguments["contact"] as? String ?: "" + val result = makePhoneCall(contact) + return "{\"result\": \"${result.message}\"}" + } + } + ) + Log.d(TAG, "已注册拨打电话功能") + } + + /** + * 注册打开应用函数 + */ + private fun registerOpenAppFunction(client: MCPClient) { + client.registerLocalFunction( + "open_app", + "打开应用", + JSONObject(""" + { + "type": "object", + "properties": { + "app_name": { + "type": "string", + "description": "应用名称" + } + }, + "required": ["app_name"] + } + """), + object : FunctionHandler { + override suspend fun handle(arguments: Map): String { + val appName = arguments["app_name"] as? String ?: "" + val result = openApp(appName) + return "{\"result\": \"${result.message}\"}" + } + } + ) + Log.d(TAG, "已注册打开应用功能") + } + + + /** + * 发送退出交互广播 + */ + private fun sendExitInteractionBroadcast() { + // 发送广播通知VoiceInteractionHandler停止语音识别 + val intent = Intent(ACTION_EXIT_INTERACTION).apply { + // 明确指定接收广播的包名 + setPackage(context.packageName) + putExtra("timestamp", System.currentTimeMillis()) + // 添加FLAG_RECEIVER_FOREGROUND确保及时处理 + addFlags(Intent.FLAG_RECEIVER_FOREGROUND) + } + Log.d(TAG, "准备发送停止交互广播，包名: ${context.packageName}") + context.sendBroadcast(intent) + Log.d(TAG, "已发送停止交互广播") + } + + + + /** + * 启动翻译模式 + * @return 操作结果 + */ + fun enterTranslationMode(): Result { + Log.d(TAG, "进入翻译模式") + // 此处应添加实际的翻译模式实现 + // 可能需要启动一个专门的翻译服务或活动 + + // 广播通知APP的其他部分进入翻译模式 + val intent = Intent("com.yunqiinnovation.deepsound.ACTION_ENTER_TRANSLATION_MODE").apply { + setPackage(context.packageName) + + putExtra("timestamp", System.currentTimeMillis()) + // 添加FLAG_RECEIVER_FOREGROUND确保及时处理 + addFlags(Intent.FLAG_RECEIVER_FOREGROUND) + } + + // 发送广播 + context.sendBroadcast(intent) + Log.d(TAG, "已发送广播: ENTER_TRANSLATION_MODE") + + return Result( + success = true, + message = "已进入实时翻译模式" + ) + } + + /** + * 发送短信 + * @param contact 联系人（名称或电话号码） + * @param message 短信内容 + * @return 操作结果 + */ + fun sendTextMessage(contact: String, message: String): Result { + Log.d(TAG, "发送短信给 $contact: $message") + + try { + val phoneNumber = getPhoneNumberForContact(contact) + if (phoneNumber.isEmpty()) { + return Result( + success = false, + message = "未找到联系人信息: $contact" + ) + } + + // 使用SmsManager发送短信 + val smsManager = SmsManager.getDefault() + smsManager.sendTextMessage(phoneNumber, null, message, null, null) + + return Result( + success = true, + message = "已向${contact}发送短信: $message" + ) + } catch (e: Exception) { + Log.e(TAG, "发送短信失败: ${e.message}", e) + return Result( + success = false, + message = "发送短信失败: ${e.message}" + ) + } + } + + /** + * 拨打电话 + * @param contact 联系人（名称或电话号码） + * @return 操作结果 + */ + fun makePhoneCall(contact: String): Result { + Log.d(TAG, "拨打电话给 $contact") + + try { + val phoneNumber = getPhoneNumberForContact(contact) + if (phoneNumber.isEmpty()) { + return Result( + success = false, + message = "未找到联系人信息: $contact" + ) + } + + // 创建拨号意图 + val intent = Intent(Intent.ACTION_CALL) + intent.data = Uri.parse("tel:$phoneNumber") + intent.flags = Intent.FLAG_ACTIVITY_NEW_TASK + context.startActivity(intent) + + return Result( + success = true, + message = "正在拨打${contact}的电话" + ) + } catch (e: Exception) { + Log.e(TAG, "拨打电话失败: ${e.message}", e) + return Result( + success = false, + message = "拨打电话失败: ${e.message}" + ) + } + } + + /** + * 打开应用程序 + * @param appName 应用名称 + * @return 操作结果 + */ + fun openApp(appName: String): Result { + Log.d(TAG, "打开应用: $appName") + + try { + // 查找应用的包名 + val packageManager = context.packageManager + val intent = Intent(Intent.ACTION_MAIN, null) + intent.addCategory(Intent.CATEGORY_LAUNCHER) + + val apps = packageManager.queryIntentActivities(intent, 0) + + for (resolveInfo in apps) { + val appLabel = resolveInfo.loadLabel(packageManager).toString() + if (appLabel.equals(appName, ignoreCase = true) || + appLabel.contains(appName, ignoreCase = true)) { + + // 找到匹配的应用 + val packageName = resolveInfo.activityInfo.packageName + val launchIntent = packageManager.getLaunchIntentForPackage(packageName) + + if (launchIntent != null) { + launchIntent.flags = Intent.FLAG_ACTIVITY_NEW_TASK + context.startActivity(launchIntent) + + return Result( + success = true, + message = "正在打开${appName}应用" + ) + } + } + } + + return Result( + success = false, + message = "未找到应用: $appName" + ) + } catch (e: Exception) { + Log.e(TAG, "打开应用失败: ${e.message}", e) + return Result( + success = false, + message = "打开应用失败: ${e.message}" + ) + } + } + + /** + * 根据联系人名称获取电话号码 + * @param contact 联系人名称或电话号码 + * @return 电话号码 + */ + private fun getPhoneNumberForContact(contact: String): String { + // 如果输入的是电话号码（简单判断，包含数字和特殊符号） + if (contact.matches(Regex("^[+\\d\\s\\-()]*\$")) && contact.any { it.isDigit() }) { + // 移除非数字字符 + return contact.replace(Regex("[^+\\d]"), "") + } + + // 从联系人数据库查询 + try { + val contentResolver = context.contentResolver + val uri = Uri.withAppendedPath( + ContactsContract.Contacts.CONTENT_FILTER_URI, + Uri.encode(contact) + ) + + val projection = arrayOf( + ContactsContract.Contacts._ID, + ContactsContract.Contacts.DISPLAY_NAME, + ContactsContract.Contacts.HAS_PHONE_NUMBER + ) + + contentResolver.query(uri, projection, null, null, null)?.use { cursor -> + if (cursor.moveToFirst()) { + val hasPhoneNumber = cursor.getInt(cursor.getColumnIndexOrThrow(ContactsContract.Contacts.HAS_PHONE_NUMBER)) + + if (hasPhoneNumber > 0) { + val contactId = cursor.getString(cursor.getColumnIndexOrThrow(ContactsContract.Contacts._ID)) + val phoneUri = ContactsContract.CommonDataKinds.Phone.CONTENT_URI + val phoneProjection = arrayOf(ContactsContract.CommonDataKinds.Phone.NUMBER) + val phoneSelection = "${ContactsContract.CommonDataKinds.Phone.CONTACT_ID} = ?" + val phoneSelectionArgs = arrayOf(contactId) + + contentResolver.query(phoneUri, phoneProjection, phoneSelection, phoneSelectionArgs, null)?.use { phoneCursor -> + if (phoneCursor.moveToFirst()) { + return phoneCursor.getString(phoneCursor.getColumnIndexOrThrow(ContactsContract.CommonDataKinds.Phone.NUMBER)) + } + } + } + } + } + } catch (e: Exception) { + Log.e(TAG, "查询联系人信息失败: ${e.message}", e) + } + + return "" + } + + /** + * 操作结果数据类 + */ + data class Result( + val success: Boolean, + val message: String + ) +} \ No newline at end of file diff --git a/local_plugins/open_ai_service/lib/open_ai_service.dart b/local_plugins/open_ai_service/lib/open_ai_service.dart index 7c6fe8024..8666f0100 100644 --- a/local_plugins/open_ai_service/lib/open_ai_service.dart +++ b/local_plugins/open_ai_service/lib/open_ai_service.dart @@ -264,4 +264,105 @@ class OpenAIService { throw OpenAIException('解析函数调用失败: $e'); } } + + /// 初始化MCP客户端 + /// + /// [serverUrl] MCP服务器WebSocket地址 + /// + /// 返回一个布尔值，表示请求是否已开始初始化（实际初始化过程是异步的） + Future initializeMcpClient(String serverUrl) async { + try { + final result = await _channel.invokeMethod( + 'initializeMcpClient', + {'serverUrl': serverUrl}, + ); + + return result ?? false; + } catch (e) { + throw OpenAIException('初始化MCP客户端失败: $e'); + } + } + + /// 检查MCP客户端是否已初始化 + Future isMcpInitialized() async { + try { + final result = await _channel.invokeMethod('isMcpInitialized'); + return result ?? false; + } catch (e) { + throw OpenAIException('检查MCP客户端状态失败: $e'); + } + } + + /// 关闭MCP客户端 + Future closeMcpClient() async { + try { + final result = await _channel.invokeMethod('closeMcpClient'); + return result ?? false; + } catch (e) { + throw OpenAIException('关闭MCP客户端失败: $e'); + } + } + + /// 处理MCP工具调用 + /// + /// [functionCall] 函数调用信息，必须包含name和arguments字段 + /// + /// 返回工具调用结果 + Future handleMcpToolCall(Map functionCall) async { + try { + final result = await _channel.invokeMethod( + 'handleMcpToolCall', + {'functionCall': jsonEncode(functionCall)}, + ); + + return result ?? "处理MCP工具调用失败: 结果为空"; + } catch (e) { + throw OpenAIException('处理MCP工具调用失败: $e'); + } + } + + /// 处理函数调用（自动处理本地函数和MCP工具） + /// + /// 这是一个辅助方法，用于简化函数调用处理流程： + /// 1. MCP工具会被自动处理，不会通过onFunctionCall回调返回 + /// 2. 只有本地函数才需要手动处理 + /// 3. 提供统一的处理接口 + /// + /// 参数: + /// [functionCall] 函数调用信息 + /// [messages] 消息上下文 + /// [localFunctionHandler] 本地函数处理器，接收函数名和参数，返回处理结果 + /// + /// 返回值: + /// 布尔值表示是否成功处理（已将结果发送回OpenAI） + Future handleFunctionCall({ + required Map functionCall, + required List> messages, + required Future Function(String name, Map arguments) localFunctionHandler, + }) async { + final name = functionCall['name'] as String; + final argumentsJson = functionCall['arguments'] as String; + + // 解析参数 + Map arguments; + try { + arguments = json.decode(argumentsJson) as Map; + } catch (e) { + arguments = {}; + } + + try { + // 调用本地函数处理器 + final result = await localFunctionHandler(name, arguments); + + // 发送结果回OpenAI + return sendFunctionCallResult( + messages: messages, + functionCall: functionCall, + functionResult: result, + ); + } catch (e) { + throw OpenAIException('处理函数调用失败: $e'); + } + } } \ No newline at end of file diff --git a/local_plugins/volcano_speech/android/build.gradle.kts b/local_plugins/volcano_speech/android/build.gradle.kts index 86b03fd0e..713a21a59 100644 --- a/local_plugins/volcano_speech/android/build.gradle.kts +++ b/local_plugins/volcano_speech/android/build.gradle.kts @@ -7,7 +7,7 @@ buildscript { } dependencies { classpath("com.android.tools.build:gradle:7.3.0") - classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:1.7.10") + classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:2.1.0") } } @@ -21,7 +21,7 @@ allprojects { plugins { id("com.android.library") - kotlin("android") + id("org.jetbrains.kotlin.android") } // 配置android扩展 diff --git a/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoAsrHelper.kt b/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoAsrHelper.kt index 614474d1a..d9b63819a 100644 --- a/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoAsrHelper.kt +++ b/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoAsrHelper.kt @@ -58,7 +58,7 @@ class VolcanoAsrHelper(private val context: Context) { /** * 初始化语音识别引擎 */ - fun initialize(appId: String, token: String, resourceId: String): Boolean { + fun initialize(appId: String, token: String, resourceId: String = "volc.bigasr.sauc.duration"): Boolean { if (isInitialized) { FileLogger.i(TAG, "引擎已经初始化") return true @@ -66,7 +66,12 @@ class VolcanoAsrHelper(private val context: Context) { try { // 准备环境 - SpeechEngineGenerator.PrepareEnvironment(context, null) + try { + SpeechEngineGenerator.PrepareEnvironment(context, null) + } catch (e: Exception) { + FileLogger.w(TAG, "准备环境时发生异常: ${e.message}，尝试继续初始化") + // 即使PrepareEnvironment失败仍继续执行，因为某些环境下可能找不到特定类 + } // 创建引擎 engine = SpeechEngineGenerator.getInstance() @@ -134,7 +139,17 @@ class VolcanoAsrHelper(private val context: Context) { return true } else { - FileLogger.e(TAG, "引擎初始化失败: $result") + // 错误码说明: + // -1：通用错误 + // -2：无效参数 + // -3：无效状态 + // -4：无效引擎 + // -5：操作失败 + // -6：缺少配置 + // -7：参数无效 + // -8：超时 + // -9：资源不足 + FileLogger.e(TAG, "引擎初始化失败: $result, 可能原因: ${getErrorDescription(result)}") return false } } catch (e: Exception) { @@ -526,4 +541,22 @@ class VolcanoAsrHelper(private val context: Context) { } } } + + /** + * 获取错误描述 + */ + private fun getErrorDescription(errorCode: Int?): String { + return when (errorCode) { + -1 -> "通用错误" + -2 -> "无效参数，检查appId、token、resourceId等参数是否正确" + -3 -> "无效状态，引擎状态异常" + -4 -> "无效引擎句柄" + -5 -> "操作失败，可能网络连接问题" + -6 -> "缺少配置参数" + -7 -> "参数设置无效，请检查appId、token等参数是否有效" + -8 -> "操作超时" + -9 -> "资源不足" + else -> "未知错误: $errorCode" + } + } } diff --git a/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoTtsHelper.kt b/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoTtsHelper.kt index e24a1ceb1..4642272d4 100644 --- a/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoTtsHelper.kt +++ b/local_plugins/volcano_speech/android/src/main/kotlin/com/yunqiinnovation/volcano_speech/VolcanoTtsHelper.kt @@ -29,7 +29,7 @@ class VolcanoTtsHelper(private val context: Context) { private var currentReqId = "" // 配置参数 - private var voice = "zh_female_yuxi" + private var voice = "zh_male_M392_conversation_wvae_bigtts" private var ttsText = "" private var isContinuous = false @@ -51,10 +51,15 @@ class VolcanoTtsHelper(private val context: Context) { FileLogger.i(TAG, "引擎已经初始化") return true } - + FileLogger.d(TAG, "初始化语音合成引擎: $appId, $token, $resourceId") try { // 准备环境 - SpeechEngineGenerator.PrepareEnvironment(context, null) + try { + SpeechEngineGenerator.PrepareEnvironment(context, null) + } catch (e: Exception) { + FileLogger.w(TAG, "准备环境时发生异常: ${e.message}，尝试继续初始化") + // 即使PrepareEnvironment失败仍继续执行，因为某些环境下可能找不到特定类 + } // 创建引擎 engine = SpeechEngineGenerator.getInstance() @@ -72,15 +77,15 @@ class VolcanoTtsHelper(private val context: Context) { engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_ENGINE_NAME_STRING, SpeechEngineDefines.TTS_ENGINE) // 设置日志级别 - engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_LOG_LEVEL_STRING, SpeechEngineDefines.LOG_LEVEL_WARN) - + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_LOG_LEVEL_STRING, SpeechEngineDefines.LOG_LEVEL_DEBUG) // 设置用户ID和设备ID (使用静态值，实际项目中应替换为真实值) engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_UID_STRING, "user_id") engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_DEVICE_ID_STRING, "device_id") // 设置授权信息 - 大模型版本不需要Bearer前缀 + FileLogger.d(TAG, "设置授权信息: $appId, $token, $resourceId") engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_APP_ID_STRING, appId) - engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_APP_TOKEN_STRING, token) + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_APP_TOKEN_STRING, "Bearer;$token") // 设置资源ID - 大模型必需 engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_RESOURCE_ID_STRING, resourceId) @@ -93,14 +98,19 @@ class VolcanoTtsHelper(private val context: Context) { // 设置在线请求资源配置 engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_ADDRESS_STRING, "wss://openspeech.bytedance.com") - engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_URI_STRING, "/api/v3/sauc/bigmodel") - + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_URI_STRING, "/api/v1/tts/ws_binary") + //【必需配置】语音合成服务所用集群 + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_CLUSTER_STRING, "volcano_tts") // 设置发音人 engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_VOICE_ONLINE_STRING, voice) // 设置播放进度回调 engine?.setOptionInt(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_WITH_FRONTEND_INT, 1) + engine?.setOptionInt(engineHandler, SpeechEngineDefines.PARAMS_KEY_AUDIO_STREAM_TYPE_INT, SpeechEngineDefines.AUDIO_STREAM_TYPE_MEDIA); + + engine?.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_TTS_SCENARIO_STRING, SpeechEngineDefines.TTS_SCENARIO_TYPE_NORMAL) + // 初始化引擎 val result = engine?.initEngine(engineHandler) isInitialized = result == SpeechEngineDefines.ERR_NO_ERROR @@ -118,7 +128,17 @@ class VolcanoTtsHelper(private val context: Context) { return true } else { - FileLogger.e(TAG, "引擎初始化失败: $result") + // 错误码说明: + // -1：通用错误 + // -2：无效参数 + // -3：无效状态 + // -4：无效引擎 + // -5：操作失败 + // -6：缺少配置 + // -7：参数无效 + // -8：超时 + // -9：资源不足 + FileLogger.e(TAG, "引擎初始化失败: $result, 可能原因: ${getErrorDescription(result)}") return false } } catch (e: Exception) { @@ -174,6 +194,7 @@ class VolcanoTtsHelper(private val context: Context) { * 开始合成并播放 */ fun speak(text: String, callback: TTSCallback? = null): Boolean { + FileLogger.d(TAG, "开始合成并播放: $text") if (!isInitialized) { FileLogger.e(TAG, "引擎未初始化") return false @@ -396,4 +417,22 @@ class VolcanoTtsHelper(private val context: Context) { } } } + + /** + * 获取错误描述 + */ + private fun getErrorDescription(errorCode: Int?): String { + return when (errorCode) { + -1 -> "通用错误" + -2 -> "无效参数，检查appId、token、resourceId等参数是否正确" + -3 -> "无效状态，引擎状态异常" + -4 -> "无效引擎句柄" + -5 -> "操作失败，可能网络连接问题" + -6 -> "缺少配置参数" + -7 -> "参数设置无效，请检查appId、token等参数是否有效" + -8 -> "操作超时" + -9 -> "资源不足" + else -> "未知错误: $errorCode" + } + } } diff --git a/pubspec.yaml b/pubspec.yaml index 41fa9edf0..b2962bec9 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -66,6 +66,11 @@ dependencies: path: local_plugins/open_ai_service volcano_speech: path: local_plugins/volcano_speech + jieli: + path: local_plugins/jieli + chat_storage_sqlite: + path: local_plugins/chat_storage_sqlite + dev_dependencies: flutter_test: diff --git a/test.json b/test.json index 703948620..473daebb6 100644 --- a/test.json +++ b/test.json @@ -1 +1,57 @@ -curl -v -X POST -H 'Content-Type: application/json' -H 'Authorization: Bearer 168deb3d-fd0c-4912-b9f1-aaee5c6743e6' -H 'Accept: text/event-stream' -d '{"model":"bot-20250405211523-l7c9r","messages":[{"role":"system","content":" 你是一个智能语音助手，能够简洁明了地回答用户的问题。\n时刻关心用户的情绪和需求，主动提供鼓励和温暖。\n\n语言风格活泼、亲切，能够幽默地互动，陪伴用户，缓解压力，增添生活乐趣。\n\n请始终以用户为中心，保持回应的高效性、准确性和温暖体贴，成为用户真正的灵魂伴侣。\n \n 当用户说\"退出\"、\"再见\"、\"结束对话\"等类似意图时，你应该使用exit_interaction函数来结束对话，\n 并在结束前说一句友好的告别语，例如\"再见，有需要随时找我\"。"},{"role":"system","content":" 你是一个智能语音助手，能够简洁明了地回答用户的问题。\n时刻关心用户的情绪和需求，主动提供鼓励和温暖。\n\n语言风格活泼、亲切，能够幽默地互动，陪伴用户，缓解压力，增添生活乐趣。\n\n请始终以用户为中心，保持回应的高效性、准确性和温暖体贴，成为用户真正的灵魂伴侣。\n \n 当用户说\"退出\"、\"再见\"、\"结束对话\"等类似意图时，你应该使用exit_interaction函数来结束对话，\n 并在结束前说一句友好的告别语，例如\"再见，有需要随时找我\"。"},{"role":"user","content":"退下吧。"},{"role":"assistant","content":"","tool_calls":[{"id":"call_8k680azmfc4thqrrnwpwqxah","type":"function","function":{"name":"exit_interaction","arguments":" {}"}}]},{"role":"tool","content":"{\"result\": \"已退出语音交互\"}","tool_call_id":"call_8k680azmfc4thqrrnwpwqxah"}],"temperature":0.7,"max_tokens":2000,"stream":true,"tools":[{"type":"function","function":{"name":"exit_interaction","description":"退出当前语音交互","parameters":{"type":"object","properties":{},"required":[]}}}]}' 'https://ark.cn-beijing.volces.com/api/v3/bots/chat/completions' \ No newline at end of file +curl -v -X POST -H 'Content-Type: application/json' -H 'Authorization: Bearer 168deb3d-fd0c-4912-b9f1-aaee5c6743e6' -H 'Accept: text/event-stream' -d '{ + "model": "bot-20250405211523-l7c9r", + "messages": [ + { + "role": "system", + "content": " 你是一个智能语音助手，能够简洁明了地回答用户的问题。\n时刻关心用户的情绪和需求，主动提供鼓励和温暖。\n\n语言风格活泼、亲切，能够幽默地互动，陪伴用户，缓解压力，增添生活乐趣。\n\n请始终以用户为中心，保持回应的高效性、准确性和温暖体贴，成为用户真正的灵魂伴侣。\n \n 当用户说\"退出\"、\"再见\"、\"结束对话\"等类似意图时，你应该使用exit_interaction函数来结束对话，\n 并在结束前说一句友好的告别语，例如\"再见，有需要随时找我\"。" + }, + { + "role": "system", + "content": " 你是一个智能语音助手，能够简洁明了地回答用户的问题。\n时刻关心用户的情绪和需求，主动提供鼓励和温暖。\n\n语言风格活泼、亲切，能够幽默地互动，陪伴用户，缓解压力，增添生活乐趣。\n\n请始终以用户为中心，保持回应的高效性、准确性和温暖体贴，成为用户真正的灵魂伴侣。\n \n 当用户说\"退出\"、\"再见\"、\"结束对话\"等类似意图时，你应该使用exit_interaction函数来结束对话，\n 并在结束前说一句友好的告别语，例如\"再见，有需要随时找我\"。" + }, + { + "role": "user", + "content": "退下吧。" + }, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_8k680azmfc4thqrrnwpwqxah", + "type": "function", + "function": { + "name": "exit_interaction", + "arguments": " {}" + } + } + ] + }, + { + "role": "tool", + "content": "{\"result\": \"已退出语音交互\"}", + "tool_call_id": "call_8k680azmfc4thqrrnwpwqxah" + } + ], + "temperature": 0.7, + "max_tokens": 2000, + "stream": true, + "tools": [ + { + "type": "function", + "function": { + "name": "exit_interaction", + "description": "退出当前语音交互", + "parameters": { + "type": "object", + "properties": {}, + "required": [] + } + } + } + ] +}' 'https: //ark.cn-beijing.volces.com/api/v3/bots/chat/completions' +{ + "code": 0, + "message": "Error during WebSocket handshake: Unexpected response code: 400", + "request_log":"{\"ab_test\":{},\"base\":{\"has_upload\":false,\"is_pending\":true,\"load_flags\":18,\"load_state\":0,\"method\":\"GET\",\"net_error\":-320,\"network_isolation_key\":\"wss://openspeech.bytedance.com wss://openspeech.bytedance.com\",\"origin_url\":\"wss://openspeech.bytedance.com/api/v3/sauc/bigmodel\",\"redirect_times\":0,\"status\":\"FAILED\",\"traffic_annotation\":77012883},\"dns\":{\"address_list\":[\"163.181.50.229:443\",\"163.181.50.223:443\",\"163.181.50.221:443\",\"163.181.50.224:443\",\"163.181.50.226:443\",\"163.181.50.222:443\",\"163.181.50.228:443\",\"163.181.50.227:443\"],\"async_batch_httpdns\":0,\"cache_source\":6,\"cache_stale_reason\":0,\"from_stale_cache\":0,\"hijacked_error\":0,\"is_auth\":false,\"nameservers\":[],\"net_type\":0,\"preload_batch_httpdns\":0,\"queried_with_cip\":\"\",\"race_dns_stale_cache\":0,\"skip_prefer_ip_error\":0,\"source\":2},\"event\":{\"engine_startup\":1744019237.421243,\"network_change\":0.0},\"header\":{\"X-Tt-Logid\":\"2025040717474815E3294F926C737834FA\",\"server-timing\":\"inner; dur=5, cdn-cache;desc=MISS,edge;dur=653,origin;dur=14\",\"x-tt-trace-host\":\"012d6b6cae97bb056dd2540a62f7f6e52ccfb433febe931be5ddf1ac674ca77c57b7dee570790477696b7b5c17bbdd77bf841100de6d411ef8ccc26681b9799e4aee97897e0681b3a8b304853d1848fcbd9fec458fdd375cddc4a7873cdbcef3a0\",\"x-tt-trace-id\":\"00-0fa6216304269466036ad7b0703104d2-0fa6216304269466-01\",\"x-tt-trace-tag\":\"id=03;cdn-cache=miss;type=dyn\"},\"ifconfig\":{\"ipv4_reachable\":true,\"ipv6_reachable\":false},\"multinetwork\":{},\"nqe\":{\"bs_sys_state\":false,\"downlink_throughput\":2658,\"effective_net_type\":5,\"group_rtt_estimate\":[],\"http_rtt\":116,\"pending_requests\":\"1\",\"real_net_type\":2,\"tcp_rtt\":46,\"total_requests\":\"6\"},\"other\":{\"core_ver\":\"87.0.4273.1\",\"gateway\":\"0.0.0.0\",\"is_background\":false,\"is_internal_retry\":false,\"is_main_process\":true,\"libcore\":\"cronet\",\"net_thread_priority\":0,\"quic_commit\":\"6ea2111b/2023-03-16\",\"resume_time\":\"-1\",\"retry_attempts\":0,\"ttnet_build_timestamp\":\"1689596445\",\"ttnet_version\":\"4.1.127.2-tob\",\"version\":\"534436af/2023-07-17\"},\"response\":{\"cached\":false,\"code\":400,\"connection_info\":1,\"network_accessed\":true,\"received_bytes\":0,\"sent_bytes\":0,\"via_proxy\":false},\"session\":{},\"socket\":{\"address_family\":1,\"cip\":\"\",\"connection_attemps\":[],\"happy_eyeballs_result\":2,\"remote\":\"163.181.50.229:443\",\"socket_reused\":false},\"socketpool\":{\"assign_type\":0,\"consume_time\":0,\"is_backup_job\":false,\"pending_reason\":0,\"pending_time\":0,\"reuse_type\":0,\"socket_idle_time\":0},\"ssl\":{\"cert_status\":0,\"connection_status\":6296321,\"handshake_type\":1,\"is_issued_by_known_root\":true,\"server_cert_common_name\":\"*.bytedance.com\"},\"store_idc\":{\"init\":1},\"timing\":{\"detailed_duration\":{\"before_stream\":0,\"body_recv\":-1,\"cdn-cache\":\"miss\",\"connect_delay\":0,\"create_stream\":98,\"dispatch\":0,\"dns\":0,\"edge\":653,\"header_recv\":-1,\"inner\":5,\"origin\":9,\"proxy\":0,\"rtt\":43,\"security_callback\":0,\"send\":0,\"ssl\":48,\"tcp\":49,\"ttfb\":710},\"request\":{\"create_stream_finish\":1744019267.633028,\"create_stream_start\":1744019267.534112,\"duration\":812,\"http_transaction_start\":1744019267.534081,\"request_sent_time\":1744019267.633497,\"response_recv_time\":1744019268.344466,\"start_time\":1744019267.533685}},\"tnc\":{\"data\":{\"canary\":\"\",\"cold_start\":1,\"config\":\"\",\"config_updated\":0,\"local_probe_version\":\"\",\"rules\":\"\",\"source\":0,\"update_time\":\"-1\"},\"probe\":{\"cmd\":\"0\",\"get_time\":\"-1\",\"send_time\":\"-1\",\"state\":1,\"version\":\"0\"},\"request\":{\"probe_version\":\"0\",\"source\":0,\"start_time\":\"-1\",\"state\":1}},\"t \ No newline at end of file From 02b3da2ee34dc7b5e897189bd13d64926f7b681f Mon Sep 17 00:00:00 2001 From: tanlongsheng <252620078@qq.com> Date: Fri, 11 Apr 2025 18:29:20 +0800 Subject: [PATCH 08/14] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=AB=98=E5=BE=B7?= =?UTF-8?q?=E5=AE=9A=E4=BD=8Dsdk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- amap_flutter/.gitignore | 33 ++++ amap_flutter/.metadata | 33 ++++ amap_flutter/CHANGELOG.md | 3 + amap_flutter/LICENSE | 1 + amap_flutter/README.md | 15 ++ amap_flutter/analysis_options.yaml | 4 + amap_flutter/android/.gitignore | 9 + amap_flutter/android/build.gradle | 54 ++++++ amap_flutter/android/settings.gradle | 1 + .../android/src/main/AndroidManifest.xml | 3 + .../amap_flutter/AmapFlutterPlugin.java | 168 ++++++++++++++++++ amap_flutter/ios/.gitignore | 38 ++++ amap_flutter/ios/Assets/.gitkeep | 0 .../ios/Classes/AmapFlutterPlugin.swift | 19 ++ .../ios/Resources/PrivacyInfo.xcprivacy | 14 ++ amap_flutter/ios/amap_flutter.podspec | 29 +++ amap_flutter/lib/amap_flutter.dart | 36 ++++ .../lib/amap_flutter_method_channel.dart | 45 +++++ .../lib/amap_flutter_platform_interface.dart | 49 +++++ amap_flutter/pubspec.yaml | 72 ++++++++ android/app/src/main/AndroidManifest.xml | 8 +- .../meeting_record_controller.dart | 20 ++- .../meeting/views/meeting_details_view.dart | 19 +- .../meeting/views/meeting_edit_view.dart | 15 +- .../meeting/views/meeting_record_view.dart | 15 +- .../splash/controllers/splash_controller.dart | 5 + pubspec.yaml | 2 + 27 files changed, 690 insertions(+), 20 deletions(-) create mode 100644 amap_flutter/.gitignore create mode 100644 amap_flutter/.metadata create mode 100644 amap_flutter/CHANGELOG.md create mode 100644 amap_flutter/LICENSE create mode 100644 amap_flutter/README.md create mode 100644 amap_flutter/analysis_options.yaml create mode 100644 amap_flutter/android/.gitignore create mode 100644 amap_flutter/android/build.gradle create mode 100644 amap_flutter/android/settings.gradle create mode 100644 amap_flutter/android/src/main/AndroidManifest.xml create mode 100644 amap_flutter/android/src/main/java/com/example/amap_flutter/AmapFlutterPlugin.java create mode 100644 amap_flutter/ios/.gitignore create mode 100644 amap_flutter/ios/Assets/.gitkeep create mode 100644 amap_flutter/ios/Classes/AmapFlutterPlugin.swift create mode 100644 amap_flutter/ios/Resources/PrivacyInfo.xcprivacy create mode 100644 amap_flutter/ios/amap_flutter.podspec create mode 100644 amap_flutter/lib/amap_flutter.dart create mode 100644 amap_flutter/lib/amap_flutter_method_channel.dart create mode 100644 amap_flutter/lib/amap_flutter_platform_interface.dart create mode 100644 amap_flutter/pubspec.yaml diff --git a/amap_flutter/.gitignore b/amap_flutter/.gitignore new file mode 100644 index 000000000..e7d347d9d --- /dev/null +++ b/amap_flutter/.gitignore @@ -0,0 +1,33 @@ +# Miscellaneous +*.class +*.log +*.pyc +*.swp +.DS_Store +.atom/ +.build/ +.buildlog/ +.history +.svn/ +.swiftpm/ +migrate_working_dir/ + +# IntelliJ related +*.iml +*.ipr +*.iws +.idea/ + +# The .vscode folder contains launch configuration and tasks you configure in +# VS Code which you may wish to be included in version control, so this line +# is commented out by default. +#.vscode/ + +# Flutter/Dart/Pub related +# Libraries should not include pubspec.lock, per https://dart.dev/guides/libraries/private-files#pubspeclock. +/pubspec.lock +**/doc/api/ +.dart_tool/ +.flutter-plugins +.flutter-plugins-dependencies +build/ diff --git a/amap_flutter/.metadata b/amap_flutter/.metadata new file mode 100644 index 000000000..a5700b901 --- /dev/null +++ b/amap_flutter/.metadata @@ -0,0 +1,33 @@ +# This file tracks properties of this Flutter project. +# Used by Flutter tool to assess capabilities and perform upgrades etc. +# +# This file should be version controlled and should not be manually edited. + +version: + revision: "35c388afb57ef061d06a39b537336c87e0e3d1b1" + channel: "stable" + +project_type: plugin + +# Tracks metadata for the flutter migrate command +migration: + platforms: + - platform: root + create_revision: 35c388afb57ef061d06a39b537336c87e0e3d1b1 + base_revision: 35c388afb57ef061d06a39b537336c87e0e3d1b1 + - platform: android + create_revision: 35c388afb57ef061d06a39b537336c87e0e3d1b1 + base_revision: 35c388afb57ef061d06a39b537336c87e0e3d1b1 + - platform: ios + create_revision: 35c388afb57ef061d06a39b537336c87e0e3d1b1 + base_revision: 35c388afb57ef061d06a39b537336c87e0e3d1b1 + + # User provided section + + # List of Local paths (relative to this file) that should be + # ignored by the migrate tool. + # + # Files that are not part of the templates will be ignored by default. + unmanaged_files: + - 'lib/main.dart' + - 'ios/Runner.xcodeproj/project.pbxproj' diff --git a/amap_flutter/CHANGELOG.md b/amap_flutter/CHANGELOG.md new file mode 100644 index 000000000..41cc7d819 --- /dev/null +++ b/amap_flutter/CHANGELOG.md @@ -0,0 +1,3 @@ +## 0.0.1 + +* TODO: Describe initial release. diff --git a/amap_flutter/LICENSE b/amap_flutter/LICENSE new file mode 100644 index 000000000..ba75c69f7 --- /dev/null +++ b/amap_flutter/LICENSE @@ -0,0 +1 @@ +TODO: Add your license here. diff --git a/amap_flutter/README.md b/amap_flutter/README.md new file mode 100644 index 000000000..8ad963d6d --- /dev/null +++ b/amap_flutter/README.md @@ -0,0 +1,15 @@ +# amap_flutter + +A new Flutter project. + +## Getting Started + +This project is a starting point for a Flutter +[plug-in package](https://flutter.dev/to/develop-plugins), +a specialized package that includes platform-specific implementation code for +Android and/or iOS. + +For help getting started with Flutter development, view the +[online documentation](https://docs.flutter.dev), which offers tutorials, +samples, guidance on mobile development, and a full API reference. + diff --git a/amap_flutter/analysis_options.yaml b/amap_flutter/analysis_options.yaml new file mode 100644 index 000000000..a5744c1cf --- /dev/null +++ b/amap_flutter/analysis_options.yaml @@ -0,0 +1,4 @@ +include: package:flutter_lints/flutter.yaml + +# Additional information about this file can be found at +# https://dart.dev/guides/language/analysis-options diff --git a/amap_flutter/android/.gitignore b/amap_flutter/android/.gitignore new file mode 100644 index 000000000..161bdcdaf --- /dev/null +++ b/amap_flutter/android/.gitignore @@ -0,0 +1,9 @@ +*.iml +.gradle +/local.properties +/.idea/workspace.xml +/.idea/libraries +.DS_Store +/build +/captures +.cxx diff --git a/amap_flutter/android/build.gradle b/amap_flutter/android/build.gradle new file mode 100644 index 000000000..6b87067f5 --- /dev/null +++ b/amap_flutter/android/build.gradle @@ -0,0 +1,54 @@ +group = "com.example.amap_flutter" +version = "1.0" + +buildscript { + repositories { + google() + mavenCentral() + } + + dependencies { + classpath("com.android.tools.build:gradle:8.7.0") + } +} + +rootProject.allprojects { + repositories { + google() + mavenCentral() + } +} + +apply plugin: "com.android.library" + +android { + namespace = "com.example.amap_flutter" + + compileSdk = 35 + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 + } + + defaultConfig { + minSdk = 21 + } + + dependencies { + implementation 'com.amap.api:location:6.4.9' + + testImplementation("junit:junit:4.13.2") + testImplementation("org.mockito:mockito-core:5.0.0") + } + + testOptions { + unitTests.all { + testLogging { + events "passed", "skipped", "failed", "standardOut", "standardError" + outputs.upToDateWhen {false} + showStandardStreams = true + } + } + } +} diff --git a/amap_flutter/android/settings.gradle b/amap_flutter/android/settings.gradle new file mode 100644 index 000000000..5f1ba0ee8 --- /dev/null +++ b/amap_flutter/android/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'amap_flutter' diff --git a/amap_flutter/android/src/main/AndroidManifest.xml b/amap_flutter/android/src/main/AndroidManifest.xml new file mode 100644 index 000000000..3d69995fd --- /dev/null +++ b/amap_flutter/android/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + diff --git a/amap_flutter/android/src/main/java/com/example/amap_flutter/AmapFlutterPlugin.java b/amap_flutter/android/src/main/java/com/example/amap_flutter/AmapFlutterPlugin.java new file mode 100644 index 000000000..ba72d7ac6 --- /dev/null +++ b/amap_flutter/android/src/main/java/com/example/amap_flutter/AmapFlutterPlugin.java @@ -0,0 +1,168 @@ +package com.example.amap_flutter; + +import android.content.Context; +import androidx.annotation.NonNull; +import android.util.Log; + +import com.amap.api.location.AMapLocation; +import com.amap.api.location.AMapLocationClient; +import com.amap.api.location.AMapLocationClientOption; +import com.amap.api.location.AMapLocationListener; + +import io.flutter.embedding.engine.plugins.FlutterPlugin; +import io.flutter.plugin.common.EventChannel; +import io.flutter.plugin.common.MethodCall; +import io.flutter.plugin.common.MethodChannel; +import io.flutter.plugin.common.MethodChannel.MethodCallHandler; +import io.flutter.plugin.common.MethodChannel.Result; + +import java.util.HashMap; +import java.util.Map; + +/** AmapFlutterPlugin */ +public class AmapFlutterPlugin implements FlutterPlugin, MethodCallHandler { + /// The MethodChannel that will the communication between Flutter and native Android + /// + /// This local reference serves to register the plugin with the Flutter Engine and unregister it + /// when the Flutter Engine is detached from the Activity + private MethodChannel methodChannel; + private EventChannel eventChannel; + private Context applicationContext; + private AMapLocationClient mLocationClient; + private EventChannel.EventSink eventSink; + + @Override + public void onAttachedToEngine(@NonNull FlutterPluginBinding flutterPluginBinding) { + applicationContext = flutterPluginBinding.getApplicationContext(); + methodChannel = new MethodChannel(flutterPluginBinding.getBinaryMessenger(), "amap_flutter"); + methodChannel.setMethodCallHandler(this); + + // 初始化EventChannel + eventChannel = new EventChannel(flutterPluginBinding.getBinaryMessenger(), "amap_flutter/events"); + eventChannel.setStreamHandler(new EventChannel.StreamHandler() { + @Override + public void onListen(Object arguments, EventChannel.EventSink events) { + eventSink = events; + } + + @Override + public void onCancel(Object arguments) { + eventSink = null; + } + }); + } + + @Override + public void onMethodCall(@NonNull MethodCall call, @NonNull Result result) { + switch (call.method) { + case "setApiKey": + setApiKey(call, result); + break; + case "initLocation": + initLocation(result); + break; + case "startLocation": + startLocation(result); + break; + case "stopLocation": + stopLocation(result); + break; + case "dispose": + dispose(result); + break; + default: + result.notImplemented(); + } + } + + private void setApiKey(MethodCall call, Result result) { + try { + String apiKey = call.argument("androidKey"); + AMapLocationClient.updatePrivacyShow(applicationContext, true, true); + AMapLocationClient.updatePrivacyAgree(applicationContext, true); + result.success(null); + } catch (Exception e) { + result.error("INIT_ERROR", "初始化失败", e.getMessage()); + } + } + + private void initLocation(Result result) { + try { + mLocationClient = new AMapLocationClient(applicationContext); + + // 配置定位参数 + AMapLocationClientOption option = new AMapLocationClientOption(); + option.setOnceLocation(true); + option.setNeedAddress(true); + mLocationClient.setLocationOption(option); + + mLocationClient.setLocationListener(mLocationListener); + result.success(null); + } catch (Exception e) { + result.error("INIT_ERROR", "初始化失败", e.getMessage()); + } + } + + private void startLocation(Result result) { + mLocationClient.startLocation(); + result.success(null); + } + + private void stopLocation(Result result) { + destroyLocationClient(); + result.success(null); + } + + private void dispose(Result result) { + try { + destroyLocationClient(); + result.success(null); + } catch (Exception e) { + result.error("DISPOSE_ERROR", "资源释放失败", e.getMessage()); + } + } + + private void destroyLocationClient() { + if (mLocationClient != null) { + try { + // 停止定位 + mLocationClient.stopLocation(); + // 销毁定位客户端 + mLocationClient.onDestroy(); + mLocationClient = null; + + // 关闭事件通道 + if (eventSink != null) { + eventSink.endOfStream(); + eventSink = null; + } + } catch (Exception e) { + Log.e("AmapFlutter", "销毁定位客户端失败", e); + } + } + } + + AMapLocationListener mLocationListener = new AMapLocationListener() { + @Override + public void onLocationChanged(AMapLocation amapLocation) { + if (eventSink == null) return; + + if (amapLocation != null) { + if (amapLocation.getErrorCode() == 0) { + Map locationData = new HashMap<>(); + locationData.put("latitude", amapLocation.getLatitude()); + locationData.put("longitude", amapLocation.getLongitude()); + locationData.put("address", amapLocation.getAddress()); + eventSink.success(locationData); + } + } + } + }; + + @Override + public void onDetachedFromEngine(@NonNull FlutterPluginBinding binding) { + methodChannel.setMethodCallHandler(null); + eventChannel.setStreamHandler(null); + destroyLocationClient(); + } +} diff --git a/amap_flutter/ios/.gitignore b/amap_flutter/ios/.gitignore new file mode 100644 index 000000000..034771fc9 --- /dev/null +++ b/amap_flutter/ios/.gitignore @@ -0,0 +1,38 @@ +.idea/ +.vagrant/ +.sconsign.dblite +.svn/ + +.DS_Store +*.swp +profile + +DerivedData/ +build/ +GeneratedPluginRegistrant.h +GeneratedPluginRegistrant.m + +.generated/ + +*.pbxuser +*.mode1v3 +*.mode2v3 +*.perspectivev3 + +!default.pbxuser +!default.mode1v3 +!default.mode2v3 +!default.perspectivev3 + +xcuserdata + +*.moved-aside + +*.pyc +*sync/ +Icon? +.tags* + +/Flutter/Generated.xcconfig +/Flutter/ephemeral/ +/Flutter/flutter_export_environment.sh diff --git a/amap_flutter/ios/Assets/.gitkeep b/amap_flutter/ios/Assets/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/amap_flutter/ios/Classes/AmapFlutterPlugin.swift b/amap_flutter/ios/Classes/AmapFlutterPlugin.swift new file mode 100644 index 000000000..b88657f78 --- /dev/null +++ b/amap_flutter/ios/Classes/AmapFlutterPlugin.swift @@ -0,0 +1,19 @@ +import Flutter +import UIKit + +public class AmapFlutterPlugin: NSObject, FlutterPlugin { + public static func register(with registrar: FlutterPluginRegistrar) { + let channel = FlutterMethodChannel(name: "amap_flutter", binaryMessenger: registrar.messenger()) + let instance = AmapFlutterPlugin() + registrar.addMethodCallDelegate(instance, channel: channel) + } + + public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { + switch call.method { + case "getPlatformVersion": + result("iOS " + UIDevice.current.systemVersion) + default: + result(FlutterMethodNotImplemented) + } + } +} diff --git a/amap_flutter/ios/Resources/PrivacyInfo.xcprivacy b/amap_flutter/ios/Resources/PrivacyInfo.xcprivacy new file mode 100644 index 000000000..a34b7e2e6 --- /dev/null +++ b/amap_flutter/ios/Resources/PrivacyInfo.xcprivacy @@ -0,0 +1,14 @@ + + + + + NSPrivacyTrackingDomains + + NSPrivacyAccessedAPITypes + + NSPrivacyCollectedDataTypes + + NSPrivacyTracking + + + diff --git a/amap_flutter/ios/amap_flutter.podspec b/amap_flutter/ios/amap_flutter.podspec new file mode 100644 index 000000000..f9ef9f715 --- /dev/null +++ b/amap_flutter/ios/amap_flutter.podspec @@ -0,0 +1,29 @@ +# +# To learn more about a Podspec see http://guides.cocoapods.org/syntax/podspec.html. +# Run `pod lib lint amap_flutter.podspec` to validate before publishing. +# +Pod::Spec.new do |s| + s.name = 'amap_flutter' + s.version = '0.0.1' + s.summary = 'A new Flutter project.' + s.description = <<-DESC +A new Flutter project. + DESC + s.homepage = 'http://example.com' + s.license = { :file => '../LICENSE' } + s.author = { 'Your Company' => 'email@example.com' } + s.source = { :path => '.' } + s.source_files = 'Classes/**/*' + s.dependency 'Flutter' + s.platform = :ios, '12.0' + + # Flutter.framework does not contain a i386 slice. + s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386' } + s.swift_version = '5.0' + + # If your plugin requires a privacy manifest, for example if it uses any + # required reason APIs, update the PrivacyInfo.xcprivacy file to describe your + # plugin's privacy impact, and then uncomment this line. For more information, + # see https://developer.apple.com/documentation/bundleresources/privacy_manifest_files + # s.resource_bundles = {'amap_flutter_privacy' => ['Resources/PrivacyInfo.xcprivacy']} +end diff --git a/amap_flutter/lib/amap_flutter.dart b/amap_flutter/lib/amap_flutter.dart new file mode 100644 index 000000000..6e4c0645c --- /dev/null +++ b/amap_flutter/lib/amap_flutter.dart @@ -0,0 +1,36 @@ +import 'amap_flutter_platform_interface.dart'; + +class AmapFlutter { + // 初始化SDK + Future initSdk({String? iosKey, String? androidKey}) async { + await AmapFlutterPlatform.instance.setApiKey( + iosKey: iosKey, + androidKey: androidKey, + ); + } + + // 定位初始化 + Future initLocation() async { + return AmapFlutterPlatform.instance.initLocation(); + } + + // 开始定位 + Future startLocation() async { + return AmapFlutterPlatform.instance.startLocation(); + } + + // 监听定位返回位置 + Stream locationStream() { + return AmapFlutterPlatform.instance.locationStream(); + } + + // 停止定位 + Future stopLocation() async { + await AmapFlutterPlatform.instance.stopLocation(); + } + + // 销毁 + Future dispose() async { + await AmapFlutterPlatform.instance.dispose(); + } +} diff --git a/amap_flutter/lib/amap_flutter_method_channel.dart b/amap_flutter/lib/amap_flutter_method_channel.dart new file mode 100644 index 000000000..470a9a322 --- /dev/null +++ b/amap_flutter/lib/amap_flutter_method_channel.dart @@ -0,0 +1,45 @@ +import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; + +import 'amap_flutter_platform_interface.dart'; + +/// An implementation of [AmapFlutterPlatform] that uses method channels. +class MethodChannelAmapFlutter extends AmapFlutterPlatform { + /// The method channel used to interact with the native platform. + @visibleForTesting + final methodChannel = const MethodChannel('amap_flutter'); + final eventChannel = const EventChannel('amap_flutter/events'); + + @override + Future setApiKey({String? iosKey, String? androidKey}) async { + await methodChannel.invokeMethod('setApiKey', { + "iosKey": iosKey, + "androidKey": androidKey, + }); + } + + @override + Future initLocation() async { + return await methodChannel.invokeMethod('initLocation'); + } + + @override + Future startLocation() async { + return await methodChannel.invokeMethod('startLocation'); + } + + @override + Stream locationStream() { + return eventChannel.receiveBroadcastStream(); + } + + @override + Future stopLocation() async { + await methodChannel.invokeMethod('stopLocation'); + } + + @override + Future dispose() async { + await methodChannel.invokeMethod('dispose'); + } +} diff --git a/amap_flutter/lib/amap_flutter_platform_interface.dart b/amap_flutter/lib/amap_flutter_platform_interface.dart new file mode 100644 index 000000000..dd1c168c1 --- /dev/null +++ b/amap_flutter/lib/amap_flutter_platform_interface.dart @@ -0,0 +1,49 @@ +import 'package:plugin_platform_interface/plugin_platform_interface.dart'; + +import 'amap_flutter_method_channel.dart'; + +abstract class AmapFlutterPlatform extends PlatformInterface { + /// Constructs a AmapFlutterPlatform. + AmapFlutterPlatform() : super(token: _token); + + static final Object _token = Object(); + + static AmapFlutterPlatform _instance = MethodChannelAmapFlutter(); + + /// The default instance of [AmapFlutterPlatform] to use. + /// + /// Defaults to [MethodChannelAmapFlutter]. + static AmapFlutterPlatform get instance => _instance; + + /// Platform-specific implementations should set this with their own + /// platform-specific class that extends [AmapFlutterPlatform] when + /// they register themselves. + static set instance(AmapFlutterPlatform instance) { + PlatformInterface.verifyToken(instance, _token); + _instance = instance; + } + + Future setApiKey({String? iosKey, String? androidKey}) { + throw UnimplementedError('init() has not been implemented.'); + } + + Future initLocation() { + throw UnimplementedError('initLocation() has not been implemented.'); + } + + Future startLocation() { + throw UnimplementedError('startLocation() has not been implemented.'); + } + + Stream locationStream() { + throw UnimplementedError('locationStream() has not been implemented.'); + } + + Future stopLocation() { + throw UnimplementedError('stopLocation() has not been implemented.'); + } + + Future dispose() { + throw UnimplementedError('dispose() has not been implemented.'); + } +} diff --git a/amap_flutter/pubspec.yaml b/amap_flutter/pubspec.yaml new file mode 100644 index 000000000..b67a2bea6 --- /dev/null +++ b/amap_flutter/pubspec.yaml @@ -0,0 +1,72 @@ +name: amap_flutter +description: "A new Flutter project." +version: 0.0.1 +homepage: + +environment: + sdk: ^3.7.0 + flutter: '>=3.3.0' + +dependencies: + flutter: + sdk: flutter + plugin_platform_interface: ^2.0.2 + +dev_dependencies: + flutter_test: + sdk: flutter + flutter_lints: ^5.0.0 + +# For information on the generic Dart part of this file, see the +# following page: https://dart.dev/tools/pub/pubspec + +# The following section is specific to Flutter packages. +flutter: + # This section identifies this Flutter project as a plugin project. + # The 'pluginClass' specifies the class (in Java, Kotlin, Swift, Objective-C, etc.) + # which should be registered in the plugin registry. This is required for + # using method channels. + # The Android 'package' specifies package in which the registered class is. + # This is required for using method channels on Android. + # The 'ffiPlugin' specifies that native code should be built and bundled. + # This is required for using `dart:ffi`. + # All these are used by the tooling to maintain consistency when + # adding or updating assets for this project. + plugin: + platforms: + android: + package: com.example.amap_flutter + pluginClass: AmapFlutterPlugin + ios: + pluginClass: AmapFlutterPlugin + + # To add assets to your plugin package, add an assets section, like this: + # assets: + # - images/a_dot_burr.jpeg + # - images/a_dot_ham.jpeg + # + # For details regarding assets in packages, see + # https://flutter.dev/to/asset-from-package + # + # An image asset can refer to one or more resolution-specific "variants", see + # https://flutter.dev/to/resolution-aware-images + + # To add custom fonts to your plugin package, add a fonts section here, + # in this "flutter" section. Each entry in this list should have a + # "family" key with the font family name, and a "fonts" key with a + # list giving the asset and other descriptors for the font. For + # example: + # fonts: + # - family: Schyler + # fonts: + # - asset: fonts/Schyler-Regular.ttf + # - asset: fonts/Schyler-Italic.ttf + # style: italic + # - family: Trajan Pro + # fonts: + # - asset: fonts/TrajanPro.ttf + # - asset: fonts/TrajanPro_Bold.ttf + # weight: 700 + # + # For details regarding fonts in packages, see + # https://flutter.dev/to/font-from-package diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index f120cc1f7..bfcb323d3 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -38,7 +38,7 @@ android:enableOnBackInvokedCallback="true" tools:replace="android:label"> - + @@ -90,6 +90,12 @@ + + + + diff --git a/local_plugins/chat_storage_sqlite/android/src/main/kotlin/com/yunqiinnovation/chat_storage_sqlite/ChatStorageSqlitePlugin.kt b/local_plugins/chat_storage/android/src/main/kotlin/com/yunqiinnovation/chat_storage/ChatStoragePlugin.kt similarity index 98% rename from local_plugins/chat_storage_sqlite/android/src/main/kotlin/com/yunqiinnovation/chat_storage_sqlite/ChatStorageSqlitePlugin.kt rename to local_plugins/chat_storage/android/src/main/kotlin/com/yunqiinnovation/chat_storage/ChatStoragePlugin.kt index d302c4d7d..9e4711531 100644 --- a/local_plugins/chat_storage_sqlite/android/src/main/kotlin/com/yunqiinnovation/chat_storage_sqlite/ChatStorageSqlitePlugin.kt +++ b/local_plugins/chat_storage/android/src/main/kotlin/com/yunqiinnovation/chat_storage/ChatStoragePlugin.kt @@ -1,4 +1,4 @@ -package com.yunqiinnovation.chat_storage_sqlite +package com.yunqiinnovation.chat_storage import android.content.Context import android.database.Cursor @@ -65,14 +65,14 @@ private class ChatDatabaseHelper(context: Context) : } /** ChatStorageSqlitePlugin */ -class ChatStorageSqlitePlugin: FlutterPlugin, MethodCallHandler { +class ChatStoragePlugin: FlutterPlugin, MethodCallHandler { private lateinit var channel : MethodChannel private lateinit var context: Context private lateinit var dbHelper: ChatDatabaseHelper override fun onAttachedToEngine(flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { context = flutterPluginBinding.applicationContext - channel = MethodChannel(flutterPluginBinding.binaryMessenger, "chat_storage_sqlite") + channel = MethodChannel(flutterPluginBinding.binaryMessenger, "chat_storage") channel.setMethodCallHandler(this) dbHelper = ChatDatabaseHelper(context) } diff --git a/local_plugins/chat_storage/ios/Classes/.gitkeep b/local_plugins/chat_storage/ios/Classes/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/local_plugins/chat_storage/ios/chat_storage.podspec b/local_plugins/chat_storage/ios/chat_storage.podspec new file mode 100644 index 000000000..b7a2c39b8 --- /dev/null +++ b/local_plugins/chat_storage/ios/chat_storage.podspec @@ -0,0 +1,23 @@ +# +# To learn more about a Podspec see http://guides.cocoapods.org/syntax/podspec.html. +# Run `pod lib lint chat_storage.podspec` to validate before publishing. +# +Pod::Spec.new do |s| + s.name = 'chat_storage' + s.version = '0.0.1' + s.summary = 'Chat Storage plugin' + s.description = <<-DESC +A Flutter plugin for chat storage services. + DESC + s.homepage = 'http://example.com' + s.license = { :file => '../LICENSE' } + s.author = { 'Your Company' => 'email@example.com' } + s.source = { :path => '.' } + s.source_files = 'Classes/**/*' + s.dependency 'Flutter' + s.platform = :ios, '12.0' + + # Flutter.framework does not contain a i386 slice. + s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386' } + s.swift_version = '5.0' +end \ No newline at end of file diff --git a/local_plugins/chat_storage_sqlite/lib/chat_storage_sqlite.dart b/local_plugins/chat_storage/lib/chat_storage.dart similarity index 98% rename from local_plugins/chat_storage_sqlite/lib/chat_storage_sqlite.dart rename to local_plugins/chat_storage/lib/chat_storage.dart index f41f473a7..2119dd84b 100644 --- a/local_plugins/chat_storage_sqlite/lib/chat_storage_sqlite.dart +++ b/local_plugins/chat_storage/lib/chat_storage.dart @@ -62,8 +62,8 @@ class PagedResult { } /// SQLite聊天记录存储插件 -class ChatStorageSqlite { - static const MethodChannel _channel = MethodChannel('chat_storage_sqlite'); +class ChatStorage { + static const MethodChannel _channel = MethodChannel('chat_storage'); /// 保存一条聊天记录 static Future> saveMessage({ @@ -77,7 +77,6 @@ class ChatStorageSqlite { 'sessionId': sessionId, 'message': message, 'sender': sender, - 'metadata': metadata, }); return Map.from(result); } catch (e) { diff --git a/local_plugins/chat_storage_sqlite/pubspec.yaml b/local_plugins/chat_storage/pubspec.yaml similarity index 61% rename from local_plugins/chat_storage_sqlite/pubspec.yaml rename to local_plugins/chat_storage/pubspec.yaml index a4d6e1bf1..0a68a4a25 100644 --- a/local_plugins/chat_storage_sqlite/pubspec.yaml +++ b/local_plugins/chat_storage/pubspec.yaml @@ -1,7 +1,7 @@ -name: chat_storage_sqlite -description: SQLite存储插件，用于存储聊天记录并支持分页查询 +name: chat_storage +description: A Flutter plugin for storing chat messages using SQLite. version: 0.0.1 -homepage: +# homepage: environment: sdk: ">=2.17.0 <3.0.0" @@ -24,7 +24,7 @@ flutter: plugin: platforms: android: - package: com.yunqiinnovation.chat_storage_sqlite - pluginClass: ChatStorageSqlitePlugin + package: com.yunqiinnovation.chat_storage + pluginClass: ChatStoragePlugin ios: - pluginClass: ChatStorageSqlitePlugin \ No newline at end of file + pluginClass: ChatStoragePlugin \ No newline at end of file diff --git a/local_plugins/chat_storage_sqlite/android/settings.gradle.kts b/local_plugins/chat_storage_sqlite/android/settings.gradle.kts deleted file mode 100644 index 613be470e..000000000 --- a/local_plugins/chat_storage_sqlite/android/settings.gradle.kts +++ /dev/null @@ -1 +0,0 @@ -rootProject.name = "chat_storage_sqlite" \ No newline at end of file diff --git a/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/BleService.kt b/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/BleService.kt index 0ba59a6e6..dec530b53 100644 --- a/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/BleService.kt +++ b/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/BleService.kt @@ -45,7 +45,7 @@ interface BleCallback { */ class BleService(private val context: Context) { companion object { - private const val TAG = "JieliBleCoreService" + private const val TAG = "JieliBleService" // 服务和特征UUID private const val AUDIO_SERVICE_UUID = "0000A100-0000-1000-8000-00805F9B34FB" diff --git a/local_plugins/open_ai_service/LICENSE b/local_plugins/open_ai_service/LICENSE new file mode 100644 index 000000000..e69de29bb diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt index 9e67d15f8..9ca318840 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt @@ -11,6 +11,7 @@ import kotlinx.coroutines.withContext import okhttp3.* import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.RequestBody.Companion.toRequestBody +import okio.Buffer import org.json.JSONArray import org.json.JSONObject import java.io.IOException @@ -389,8 +390,8 @@ class OpenAIService(private val context: Context? = null) { .build() // 输出用于测试的curl命令 - // val curlCommand = buildCurlCommand(request, requestBody.toString()) - // Log.d(TAG, "curl command: $curlCommand") + val curlCommand = buildCurlCommand(request, requestBody.toString()) + Log.d(TAG, "curl command: $curlCommand") client.newCall(request).enqueue(object : Callback { override fun onFailure(call: Call, e: IOException) { @@ -406,81 +407,95 @@ class OpenAIService(private val context: Context? = null) { val responseBody = response.body ?: return val source = responseBody.source() + val buffer = Buffer() try { - // 预取数据到缓冲区 - source.request(Long.MAX_VALUE) - val bufferedSource = source.buffer - // 用于存储函数调用的各个部分 val finalToolCalls = mutableMapOf() + val lineBuffer = StringBuilder() - while (!bufferedSource.exhausted()) { - val line = bufferedSource.readUtf8Line()?.trim() ?: continue - if (line.isEmpty()) continue - if (line.startsWith("data:")) { - val data = line.substring(5).trim() - - // 处理[DONE]消息 - if (data == "[DONE]" || data == "[\"DONE\"]") { - processToolCalls(finalToolCalls, callback, messages) - callback.onComplete() - break - } + // 逐块读取数据，而不是一次性预取 + val BUFFER_SIZE = 8192L // 8KB的块大小 + + while (!Thread.currentThread().isInterrupted) { + // 读取一小块数据 + val bytesRead = source.read(buffer, BUFFER_SIZE) + if (bytesRead == -1L) break // 流结束 + + // 处理接收到的数据 + while (true) { + val line = buffer.readUtf8Line() ?: break + if (line.trim().isEmpty()) continue - try { - val jsonData = JSONObject(data) + if (line.startsWith("data:")) { + val data = line.substring(5).trim() + Log.d("OpenAIService", "$data") - // 处理消息内容 - if (jsonData.has("choices")) { - val choices = jsonData.getJSONArray("choices") - if (choices.length() > 0) { - val choice = choices.getJSONObject(0) - - if (choice.has("delta")) { - val delta = choice.getJSONObject("delta") - - // 处理普通文本内容 - if (delta.has("content")) { - val content = delta.getString("content") - callback.onToken(content) - } + // 处理[DONE]消息 + if (data == "[DONE]" || data == "[\"DONE\"]") { + Log.d(TAG, "收到[DONE]消息") + val hasToolCalls = processToolCalls(finalToolCalls, callback, messages) + // 只有在没有工具调用时才认为对话真正完成 + if (!hasToolCalls) { + callback.onComplete() + } + return + } + + try { + val jsonData = JSONObject(data) + + // 处理消息内容 + if (jsonData.has("choices")) { + val choices = jsonData.getJSONArray("choices") + if (choices.length() > 0) { + val choice = choices.getJSONObject(0) - // 处理工具调用（函数调用） - if (delta.has("tool_calls")) { - val toolCalls = delta.getJSONArray("tool_calls") - for (i in 0 until toolCalls.length()) { - val toolCall = toolCalls.getJSONObject(i) - val index = toolCall.getInt("index") - - // 创建或获取现有的工具调用信息 - val toolCallInfo = finalToolCalls.getOrPut(index) { ToolCallInfo() } - - // 更新ID - if (toolCall.has("id")) { - toolCallInfo.id = toolCall.getString("id") - } - - // 更新函数信息 - if (toolCall.has("function")) { - val function = toolCall.getJSONObject("function") + if (choice.has("delta")) { + val delta = choice.getJSONObject("delta") + + // 处理普通文本内容 + if (delta.has("content")) { + val content = delta.getString("content") + callback.onToken(content) + } + + // 处理工具调用（函数调用） + if (delta.has("tool_calls")) { + val toolCalls = delta.getJSONArray("tool_calls") + for (i in 0 until toolCalls.length()) { + val toolCall = toolCalls.getJSONObject(i) + val index = toolCall.getInt("index") + + // 创建或获取现有的工具调用信息 + val toolCallInfo = finalToolCalls.getOrPut(index) { ToolCallInfo() } - if (function.has("name")) { - toolCallInfo.name = function.getString("name") + // 更新ID + if (toolCall.has("id")) { + toolCallInfo.id = toolCall.getString("id") } - if (function.has("arguments")) { - toolCallInfo.arguments += function.getString("arguments") + // 更新函数信息 + if (toolCall.has("function")) { + val function = toolCall.getJSONObject("function") + + if (function.has("name")) { + toolCallInfo.name = function.getString("name") + } + + if (function.has("arguments")) { + toolCallInfo.arguments += function.getString("arguments") + } } } } } } } + } catch (e: Exception) { + // 忽略解析错误 + Log.e(TAG, "解析JSON出错: ${e.message}") } - } catch (e: Exception) { - // 忽略解析错误 - Log.e(TAG, "解析JSON出错: ${e.message}") } } } @@ -548,11 +563,11 @@ class OpenAIService(private val context: Context? = null) { /** * 处理工具调用结果 */ - private fun processToolCalls(toolCalls: Map, callback: StreamCallback, messages: JSONArray? = null) { - if (toolCalls.isEmpty()) return + private fun processToolCalls(toolCalls: Map, callback: StreamCallback, messages: JSONArray? = null): Boolean { + if (toolCalls.isEmpty()) return false // 只处理第一个工具调用 - val firstToolCall = toolCalls.entries.firstOrNull()?.value ?: return + val firstToolCall = toolCalls.entries.firstOrNull()?.value ?: return false if (firstToolCall.isValid()) { // 创建函数调用JSON对象 @@ -572,7 +587,9 @@ class OpenAIService(private val context: Context? = null) { } } } + return true } + return false } /** diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt index 1e1bb9446..091af559c 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt @@ -1,6 +1,8 @@ package com.yunqiinnovation.open_ai_service import android.content.Context +import android.os.Handler +import android.os.Looper import android.util.Log import androidx.annotation.NonNull import io.flutter.embedding.engine.plugins.FlutterPlugin @@ -43,8 +45,11 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { /// 事件接收器（用于流式输出） private var eventSink: EventSink? = null - /// 执行器（用于后台线程） + /// 执行器（用于真正需要在后台线程处理的任务） private val executor = Executors.newSingleThreadExecutor() + + /// 主线程Handler + private val mainHandler = Handler(Looper.getMainLooper()) override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { // 保存上下文 @@ -98,17 +103,17 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { messages.put(JSONObject(message)) } - // 在后台线程执行请求 + // 对于需要在后台执行的同步阻塞操作，仍然使用executor executor.execute { try { val response = openAIService.sendMessage(messages) // 在主线程返回结果 - android.os.Handler(android.os.Looper.getMainLooper()).post { + mainHandler.post { result.success(response) } } catch (e: Exception) { // 在主线程返回错误 - android.os.Handler(android.os.Looper.getMainLooper()).post { + mainHandler.post { result.error("OPENAI_ERROR", e.message, null) } } @@ -130,53 +135,47 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { messages.put(JSONObject(message)) } - // 在后台线程执行请求 - executor.execute { - try { - openAIService.sendMessageStream( - messages = messages, - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - // 发送token事件 - android.os.Handler(android.os.Looper.getMainLooper()).post { - eventSink?.success(mapOf("type" to "token", "content" to token)) - } + // 不再在后台线程包装OkHttp的异步调用 + try { + openAIService.sendMessageStream( + messages = messages, + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + // 发送token事件 + mainHandler.post { + eventSink?.success(mapOf("type" to "token", "content" to token)) } - - override fun onComplete() { - // 发送完成事件 - android.os.Handler(android.os.Looper.getMainLooper()).post { - eventSink?.success(mapOf("type" to "complete")) - } + } + + override fun onComplete() { + // 发送完成事件 + mainHandler.post { + eventSink?.success(mapOf("type" to "complete")) } - - override fun onError(e: Exception) { - // 发送错误事件 - android.os.Handler(android.os.Looper.getMainLooper()).post { - eventSink?.success(mapOf("type" to "error", "content" to e.message)) - } + } + + override fun onError(e: Exception) { + // 发送错误事件 + mainHandler.post { + eventSink?.success(mapOf("type" to "error", "content" to e.message)) } - - override fun onFunctionCall(functionCall: JSONObject) { - // 发送函数调用事件 - android.os.Handler(android.os.Looper.getMainLooper()).post { - val functionCallMap = functionCall.toMap() - eventSink?.success(mapOf("type" to "functionCall", "content" to functionCallMap)) - } + } + + override fun onFunctionCall(functionCall: JSONObject) { + // 发送函数调用事件 + mainHandler.post { + val functionCallMap = functionCall.toMap() + eventSink?.success(mapOf("type" to "functionCall", "content" to functionCallMap)) } } - ) - - // 请求已开始 - android.os.Handler(android.os.Looper.getMainLooper()).post { - result.success(true) } - } catch (e: Exception) { - // 在主线程返回错误 - android.os.Handler(android.os.Looper.getMainLooper()).post { - result.error("OPENAI_ERROR", e.message, null) - } - } + ) + + // 请求已开始，直接返回结果 + result.success(true) + } catch (e: Exception) { + // 直接在当前线程处理错误 + result.error("OPENAI_ERROR", e.message, null) } } @@ -200,55 +199,49 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { // 转换函数调用 val functionCall = JSONObject(functionCallRaw) - // 在后台线程执行请求 - executor.execute { - try { - openAIService.sendFunctionCallResult( - messages = messages, - functionCall = functionCall, - functionResult = functionResult, - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - // 发送token事件 - android.os.Handler(android.os.Looper.getMainLooper()).post { - eventSink?.success(mapOf("type" to "token", "content" to token)) - } + // 不再在后台线程包装OkHttp的异步调用 + try { + openAIService.sendFunctionCallResult( + messages = messages, + functionCall = functionCall, + functionResult = functionResult, + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + // 发送token事件 + mainHandler.post { + eventSink?.success(mapOf("type" to "token", "content" to token)) } - - override fun onComplete() { - // 发送完成事件 - android.os.Handler(android.os.Looper.getMainLooper()).post { - eventSink?.success(mapOf("type" to "complete")) - } + } + + override fun onComplete() { + // 发送完成事件 + mainHandler.post { + eventSink?.success(mapOf("type" to "complete")) } - - override fun onError(e: Exception) { - // 发送错误事件 - android.os.Handler(android.os.Looper.getMainLooper()).post { - eventSink?.success(mapOf("type" to "error", "content" to e.message)) - } + } + + override fun onError(e: Exception) { + // 发送错误事件 + mainHandler.post { + eventSink?.success(mapOf("type" to "error", "content" to e.message)) } - - override fun onFunctionCall(nestedFunctionCall: JSONObject) { - // 发送函数调用事件 - android.os.Handler(android.os.Looper.getMainLooper()).post { - val functionCallMap = nestedFunctionCall.toMap() - eventSink?.success(mapOf("type" to "functionCall", "content" to functionCallMap)) - } + } + + override fun onFunctionCall(nestedFunctionCall: JSONObject) { + // 发送函数调用事件 + mainHandler.post { + val functionCallMap = nestedFunctionCall.toMap() + eventSink?.success(mapOf("type" to "functionCall", "content" to functionCallMap)) } } - ) - - // 请求已开始 - android.os.Handler(android.os.Looper.getMainLooper()).post { - result.success(true) } - } catch (e: Exception) { - // 在主线程返回错误 - android.os.Handler(android.os.Looper.getMainLooper()).post { - result.error("OPENAI_ERROR", e.message, null) - } - } + ) + + // 请求已开始，直接返回结果 + result.success(true) + } catch (e: Exception) { + // 直接在当前线程处理错误 + result.error("OPENAI_ERROR", e.message, null) } } diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/SystemFunctionHandler.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/SystemFunctionHandler.kt index f2e7cbcdb..875895d65 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/SystemFunctionHandler.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/SystemFunctionHandler.kt @@ -78,7 +78,7 @@ class SystemFunctionHandler(private val context: Context) { private fun registerTranslationModeFunction(client: MCPClient) { client.registerLocalFunction( "enter_translation_mode", - "进入实时翻译模式", + "用户请求进入实时翻译模式时，启动实时翻译功能", JSONObject(""" { "type": "object", @@ -167,7 +167,7 @@ class SystemFunctionHandler(private val context: Context) { private fun registerOpenAppFunction(client: MCPClient) { client.registerLocalFunction( "open_app", - "打开应用", + "当用户请求打开特定应用时，启动相应应用", JSONObject(""" { "type": "object", diff --git a/local_plugins/open_ai_service/ios/Classes/.gitkeep b/local_plugins/open_ai_service/ios/Classes/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/local_plugins/open_ai_service/ios/Classes/OpenAIService.swift b/local_plugins/open_ai_service/ios/Classes/OpenAIService.swift index 671affd73..5df99e126 100644 --- a/local_plugins/open_ai_service/ios/Classes/OpenAIService.swift +++ b/local_plugins/open_ai_service/ios/Classes/OpenAIService.swift @@ -237,30 +237,47 @@ public class OpenAIService { } /// 发送消息（流式输出） - public func sendMessageStream(messages: [[String: Any]], systemPrompt: String, callback: @escaping StreamCallback) { + public func sendMessageStream( + messages: [[String: Any]], + systemPrompt: String, + callback: @escaping StreamCallback + ) { guard isInitialized, !apiKey.isEmpty else { callback.onError(OpenAIError("OpenAI服务未初始化")) return } - // 构建完整消息，添加系统提示 - var fullMessages: [[String: Any]] = [ - ["role": "system", "content": systemPrompt] - ] - fullMessages.append(contentsOf: messages) - // 构建请求体 - var requestDict: [String: Any] = [ + var requestBody: [String: Any] = [ "model": model, - "messages": fullMessages, "temperature": 0.7, "max_tokens": 2000, "stream": true ] - // 如果有注册的函数，添加到请求中 - if !registeredFunctions.isEmpty { + // 构建完整消息数组，添加系统提示 + var fullMessages: [[String: Any]] = [ + ["role": "system", "content": systemPrompt] + ] + fullMessages.append(contentsOf: messages) + requestBody["messages"] = fullMessages + + // 添加工具列表 + if let toolMaps = mcpClient?.getToolMaps(), !toolMaps.isEmpty { + var tools: [[String: Any]] = [] + + for toolMap in toolMaps { + if let tool = toolMap as? [String: Any] { + tools.append(tool) + } + } + + if !tools.isEmpty { + requestBody["tools"] = tools + } + } else if !registeredFunctions.isEmpty { var tools: [[String: Any]] = [] + for function in registeredFunctions { let tool: [String: Any] = [ "type": "function", @@ -268,11 +285,14 @@ public class OpenAIService { ] tools.append(tool) } - requestDict["tools"] = tools + + if !tools.isEmpty { + requestBody["tools"] = tools + } } - // 将请求数据转换为JSON数据 - guard let jsonData = try? JSONSerialization.data(withJSONObject: requestDict) else { + // 转换为JSON数据 + guard let jsonData = try? JSONSerialization.data(withJSONObject: requestBody) else { callback.onError(OpenAIError("无法序列化请求数据")) return } @@ -290,102 +310,10 @@ public class OpenAIService { request.addValue("text/event-stream", forHTTPHeaderField: "Accept") request.httpBody = jsonData - // 用于存储函数调用的各个部分 - var finalToolCalls: [Int: ToolCallInfo] = [:] - - // 创建数据任务 - let task = session.dataTask(with: request) { data, response, error in - if let error = error { - callback.onError(OpenAIError("请求失败: \(error.localizedDescription)")) - return - } - - guard let httpResponse = response as? HTTPURLResponse else { - callback.onError(OpenAIError("无效的HTTP响应")) - return - } - - guard httpResponse.statusCode == 200 else { - callback.onError(OpenAIError("API调用失败: \(httpResponse.statusCode)")) - return - } - - guard let data = data else { - callback.onError(OpenAIError("响应数据为空")) - return - } - - // 处理SSE数据流 - if let text = String(data: data, encoding: .utf8) { - let lines = text.components(separatedBy: "\n") - - for line in lines { - if line.isEmpty { continue } - - if line.hasPrefix("data: ") { - let dataContent = line.dropFirst(6) - - // 处理[DONE]消息 - if dataContent == "[DONE]" { - self.processToolCalls(finalToolCalls, callback: callback) - callback.onComplete() - break - } - - // 解析JSON数据 - do { - if let data = dataContent.data(using: .utf8), - let jsonData = try JSONSerialization.jsonObject(with: data) as? [String: Any] { - - // 处理消息内容 - if let choices = jsonData["choices"] as? [[String: Any]], !choices.isEmpty, - let choice = choices.first { - - if let delta = choice["delta"] as? [String: Any] { - // 处理普通文本内容 - if let content = delta["content"] as? String { - callback.onToken(content) - } - - // 处理工具调用（函数调用） - if let toolCalls = delta["tool_calls"] as? [[String: Any]] { - for toolCall in toolCalls { - if let index = toolCall["index"] as? Int { - // 创建或获取现有的工具调用信息 - let toolCallInfo = finalToolCalls[index] ?? ToolCallInfo() - - // 更新ID - if let id = toolCall["id"] as? String { - toolCallInfo.id = id - } - - // 更新函数信息 - if let function = toolCall["function"] as? [String: Any] { - if let name = function["name"] as? String { - toolCallInfo.name = name - } - - if let arguments = function["arguments"] as? String { - toolCallInfo.arguments += arguments - } - } - - finalToolCalls[index] = toolCallInfo - } - } - } - } - } - } - } catch { - NSLog("解析JSON出错: \(error.localizedDescription)") - // 忽略解析错误，继续处理其他行 - } - } - } - } - } - + // 创建流式会话任务 + let delegate = SSEStreamDelegate(callback: callback) + let session = URLSession(configuration: .default, delegate: delegate, delegateQueue: nil) + let task = session.dataTask(with: request) task.resume() } @@ -447,12 +375,147 @@ public class OpenAIService { } } + /// SSE流委托实现 + private class SSEStreamDelegate: NSObject, URLSessionDataDelegate { + let callback: StreamCallback + private var buffer = Data() + private var finalToolCalls: [Int: ToolCallInfo] = [:] + + init(callback: @escaping StreamCallback) { + self.callback = callback + super.init() + } + + // 接收数据流 + func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) { + buffer.append(data) + + // 处理可能包含多行的数据 + processBuffer() + } + + // 处理缓冲区数据 + private func processBuffer() { + // 按行分割 + while let newlineIndex = buffer.firstIndex(of: 10) { // 10是换行符的ASCII码 + let lineData = buffer.prefix(upTo: newlineIndex) + buffer.removeSubrange(0...newlineIndex) // 移除已处理的行，包括换行符 + + // 解析行数据 + if let line = String(data: lineData, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) { + processLine(line) + } + } + } + + // 处理单行数据 + private func processLine(_ line: String) { + guard !line.isEmpty else { return } + + if line.hasPrefix("data: ") { + let dataContent = line.dropFirst(6) + + // 处理[DONE]消息 + if dataContent == "[DONE]" { + let hasToolCalls = self.processToolCalls() + // 只有在没有工具调用时才认为对话真正完成 + if !hasToolCalls { + callback.onComplete() + } + return + } + + // 解析JSON数据 + do { + if let data = dataContent.data(using: .utf8), + let jsonData = try JSONSerialization.jsonObject(with: data) as? [String: Any] { + + // 处理消息内容 + if let choices = jsonData["choices"] as? [[String: Any]], !choices.isEmpty, + let choice = choices.first { + + if let delta = choice["delta"] as? [String: Any] { + // 处理普通文本内容 + if let content = delta["content"] as? String { + callback.onToken(content) + } + + // 处理工具调用（函数调用） + if let toolCalls = delta["tool_calls"] as? [[String: Any]] { + for toolCall in toolCalls { + if let index = toolCall["index"] as? Int { + // 创建或获取现有的工具调用信息 + let toolCallInfo = finalToolCalls[index] ?? ToolCallInfo() + + // 更新ID + if let id = toolCall["id"] as? String { + toolCallInfo.id = id + } + + // 更新函数信息 + if let function = toolCall["function"] as? [String: Any] { + if let name = function["name"] as? String { + toolCallInfo.name = name + } + + if let arguments = function["arguments"] as? String { + toolCallInfo.arguments += arguments + } + } + + finalToolCalls[index] = toolCallInfo + } + } + } + } + } + } + } catch { + NSLog("解析JSON出错: \(error.localizedDescription)") + // 忽略解析错误，继续处理其他行 + } + } + } + + // 处理工具调用 + private func processToolCalls() -> Bool { + if finalToolCalls.isEmpty { return false } + + // 只处理第一个工具调用 + guard let firstToolCall = finalToolCalls.values.first, firstToolCall.isValid else { return false } + + // 创建函数调用字典 + let functionCall: [String: Any] = [ + "name": firstToolCall.name, + "arguments": firstToolCall.arguments, + "id": firstToolCall.id + ] + + // 回调 + callback.onFunctionCall(functionCall) + return true + } + + // 处理完成 + func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) { + if let error = error { + callback.onError(OpenAIError("请求失败: \(error.localizedDescription)")) + } + } + } + + /// 流式输出回调协议 + public typealias StreamCallback = (onToken: (String) -> Void, + onComplete: () -> Void, + onError: (Error) -> Void, + onFunctionCall: ([String: Any]) -> Void) + /// 处理工具调用（函数调用）并回调 - private func processToolCalls(_ toolCalls: [Int: ToolCallInfo], callback: StreamCallback) { - if toolCalls.isEmpty { return } + private func processToolCalls(_ toolCalls: [Int: ToolCallInfo], callback: StreamCallback) -> Bool { + if toolCalls.isEmpty { return false } // 只处理第一个工具调用 - guard let firstToolCall = toolCalls.values.first, firstToolCall.isValid else { return } + guard let firstToolCall = toolCalls.values.first, firstToolCall.isValid else { return false } // 创建函数调用字典 let functionCall: [String: Any] = [ @@ -463,11 +526,6 @@ public class OpenAIService { // 回调 callback.onFunctionCall(functionCall) + return true } - - /// 流式输出回调协议 - public typealias StreamCallback = (onToken: (String) -> Void, - onComplete: () -> Void, - onError: (Error) -> Void, - onFunctionCall: ([String: Any]) -> Void) } \ No newline at end of file diff --git a/local_plugins/open_ai_service/ios/open_ai_service.podspec b/local_plugins/open_ai_service/ios/open_ai_service.podspec new file mode 100644 index 000000000..d06d07fed --- /dev/null +++ b/local_plugins/open_ai_service/ios/open_ai_service.podspec @@ -0,0 +1,23 @@ +# +# To learn more about a Podspec see http://guides.cocoapods.org/syntax/podspec.html. +# Run `pod lib lint open_ai_service.podspec` to validate before publishing. +# +Pod::Spec.new do |s| + s.name = 'open_ai_service' + s.version = '0.0.1' + s.summary = 'OpenAI Service plugin' + s.description = <<-DESC +A Flutter plugin for OpenAI services. + DESC + s.homepage = 'http://example.com' + s.license = { :file => '../LICENSE' } + s.author = { 'Your Company' => 'email@example.com' } + s.source = { :path => '.' } + s.source_files = 'Classes/**/*' + s.dependency 'Flutter' + s.platform = :ios, '12.0' + + # Flutter.framework does not contain a i386 slice. + s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386' } + s.swift_version = '5.0' +end \ No newline at end of file diff --git a/local_plugins/volcano_speech/LICENSE b/local_plugins/volcano_speech/LICENSE new file mode 100644 index 000000000..e69de29bb diff --git a/local_plugins/volcano_speech/ios/Classes/.gitkeep b/local_plugins/volcano_speech/ios/Classes/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/local_plugins/volcano_speech/ios/volcano_speech.podspec b/local_plugins/volcano_speech/ios/volcano_speech.podspec new file mode 100644 index 000000000..9a70baed1 --- /dev/null +++ b/local_plugins/volcano_speech/ios/volcano_speech.podspec @@ -0,0 +1,23 @@ +# +# To learn more about a Podspec see http://guides.cocoapods.org/syntax/podspec.html. +# Run `pod lib lint volcano_speech.podspec` to validate before publishing. +# +Pod::Spec.new do |s| + s.name = 'volcano_speech' + s.version = '0.0.1' + s.summary = 'Volcano Speech plugin' + s.description = <<-DESC +A Flutter plugin for Volcano Speech services. + DESC + s.homepage = 'http://example.com' + s.license = { :file => '../LICENSE' } + s.author = { 'Your Company' => 'email@example.com' } + s.source = { :path => '.' } + s.source_files = 'Classes/**/*' + s.dependency 'Flutter' + s.platform = :ios, '12.0' + + # Flutter.framework does not contain a i386 slice. + s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386' } + s.swift_version = '5.0' +end \ No newline at end of file diff --git a/pubspec.yaml b/pubspec.yaml index b2962bec9..b08d2746e 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -68,8 +68,8 @@ dependencies: path: local_plugins/volcano_speech jieli: path: local_plugins/jieli - chat_storage_sqlite: - path: local_plugins/chat_storage_sqlite + chat_storage: + path: local_plugins/chat_storage dev_dependencies: diff --git a/test.json b/test.json index 473daebb6..2411527e2 100644 --- a/test.json +++ b/test.json @@ -1,57 +1,163 @@ -curl -v -X POST -H 'Content-Type: application/json' -H 'Authorization: Bearer 168deb3d-fd0c-4912-b9f1-aaee5c6743e6' -H 'Accept: text/event-stream' -d '{ - "model": "bot-20250405211523-l7c9r", - "messages": [ - { - "role": "system", - "content": " 你是一个智能语音助手，能够简洁明了地回答用户的问题。\n时刻关心用户的情绪和需求，主动提供鼓励和温暖。\n\n语言风格活泼、亲切，能够幽默地互动，陪伴用户，缓解压力，增添生活乐趣。\n\n请始终以用户为中心，保持回应的高效性、准确性和温暖体贴，成为用户真正的灵魂伴侣。\n \n 当用户说\"退出\"、\"再见\"、\"结束对话\"等类似意图时，你应该使用exit_interaction函数来结束对话，\n 并在结束前说一句友好的告别语，例如\"再见，有需要随时找我\"。" - }, +curl 'https://ark.cn-beijing.volces.com/api/v3/bots/chat/completions' \ +-H "Authorization: Bearer 168deb3d-fd0c-4912-b9f1-aaee5c6743e6" \ +-H 'Content-Type: application/json' \ +-d '{ + "model": "bot-20250405211523-l7c9r", + "stream": true, + "stream_options": {"include_usage": true}, + "messages": [ { "role": "system", - "content": " 你是一个智能语音助手，能够简洁明了地回答用户的问题。\n时刻关心用户的情绪和需求，主动提供鼓励和温暖。\n\n语言风格活泼、亲切，能够幽默地互动，陪伴用户，缓解压力，增添生活乐趣。\n\n请始终以用户为中心，保持回应的高效性、准确性和温暖体贴，成为用户真正的灵魂伴侣。\n \n 当用户说\"退出\"、\"再见\"、\"结束对话\"等类似意图时，你应该使用exit_interaction函数来结束对话，\n 并在结束前说一句友好的告别语，例如\"再见，有需要随时找我\"。" + "content": "You are a helpful assistant." }, { "role": "user", - "content": "退下吧。" - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_8k680azmfc4thqrrnwpwqxah", - "type": "function", - "function": { - "name": "exit_interaction", - "arguments": " {}" - } - } - ] - }, - { - "role": "tool", - "content": "{\"result\": \"已退出语音交互\"}", - "tool_call_id": "call_8k680azmfc4thqrrnwpwqxah" + "content": "Hello!" } - ], - "temperature": 0.7, - "max_tokens": 2000, - "stream": true, - "tools": [ - { - "type": "function", - "function": { - "name": "exit_interaction", - "description": "退出当前语音交互", - "parameters": { - "type": "object", - "properties": {}, - "required": [] - } + ] +}' + + + + +curl 'https://ark.cn-beijing.volces.com/api/v3/bots/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer 168deb3d-fd0c-4912-b9f1-aaee5c6743e6' \ +-d '{ + "model": "bot-20250405211523-l7c9r", + "stream": true, + "messages": [ + { + "role": "assistant", + "content": "你好呀！今天想和我聊些什么呢😄 是生活趣事、学习心得，还是其他好玩的话题？" + }, + { + "role": "assistant", + "content": "" + }, + { + "role": "assistant", + "content": "你好呀！今天过得怎么样？有什么需要我帮忙的吗？或者只是想聊聊天？😊" + }, + { + "role": "assistant", + "content": "明白啦！有什么需要我帮忙的吗？无论是查资料、聊天解闷，还是帮你处理手机上的任务，我随时待命哦～ 😊" + }, + { + "role": "assistant", + "content": "好的！随时准备陪你测试，有什么需要尽管说哦～ 😊 你希望测试什么功能呢？" + }, + { + "role": "assistant", + "content": "哎呀，听起来你有点小困扰呢！可能是因为网络波动、系统更新或者我偶尔“打瞌睡”了（开玩笑啦~）。不过别担心，我会努力保持最佳状态陪伴你！如果有什么具体问题，随时告诉我，我一定帮你解决得妥妥的！😊\n\n（或者你想测试什么功能？我随时待命哦！）" + }, + { + "role": "assistant", + "content": "嘿嘿，我是你的贴心语音助理小语呀！随时为你提供帮助、陪你聊天、解答问题，还能给你加油打气哦～有什么需要尽管告诉我吧！ 😊" + }, + { + "role": "assistant", + "content": "我是小语呀，那个随时准备和你畅聊、给你排忧解难、陪你度过各种时光的手机全能语音助理😜 怎么啦，是不是刚才没记住呀🧐" + }, + { + "role": "assistant", + "content": "" + }, + { + "role": "user", + "content": "你好呀，你是谁呀？" + } + ], + "temperature": 0.7, + "max_tokens": 2000, + "stream": true, + "tools": [ + { + "type": "function", + "function": { + "name": "exit_interaction", + "description": "退出当前语音交互", + "parameters": { + "type": "object", + "properties": {}, + "required": [] + } + } + }, + { + "type": "function", + "function": { + "name": "enter_translation_mode", + "description": "用户请求进入实时翻译模式时，启动实时翻译功能", + "parameters": { + "type": "object", + "properties": {}, + "required": [] + } + } + }, + { + "type": "function", + "function": { + "name": "send_text_message", + "description": "发送短信", + "parameters": { + "type": "object", + "properties": { + "contact": { + "type": "string", + "description": "联系人姓名或电话号码" + }, + "message": { + "type": "string", + "description": "短信内容" } + }, + "required": [ + "contact", + "message" + ] } - ] -}' 'https: //ark.cn-beijing.volces.com/api/v3/bots/chat/completions' -{ - "code": 0, - "message": "Error during WebSocket handshake: Unexpected response code: 400", - "request_log":"{\"ab_test\":{},\"base\":{\"has_upload\":false,\"is_pending\":true,\"load_flags\":18,\"load_state\":0,\"method\":\"GET\",\"net_error\":-320,\"network_isolation_key\":\"wss://openspeech.bytedance.com wss://openspeech.bytedance.com\",\"origin_url\":\"wss://openspeech.bytedance.com/api/v3/sauc/bigmodel\",\"redirect_times\":0,\"status\":\"FAILED\",\"traffic_annotation\":77012883},\"dns\":{\"address_list\":[\"163.181.50.229:443\",\"163.181.50.223:443\",\"163.181.50.221:443\",\"163.181.50.224:443\",\"163.181.50.226:443\",\"163.181.50.222:443\",\"163.181.50.228:443\",\"163.181.50.227:443\"],\"async_batch_httpdns\":0,\"cache_source\":6,\"cache_stale_reason\":0,\"from_stale_cache\":0,\"hijacked_error\":0,\"is_auth\":false,\"nameservers\":[],\"net_type\":0,\"preload_batch_httpdns\":0,\"queried_with_cip\":\"\",\"race_dns_stale_cache\":0,\"skip_prefer_ip_error\":0,\"source\":2},\"event\":{\"engine_startup\":1744019237.421243,\"network_change\":0.0},\"header\":{\"X-Tt-Logid\":\"2025040717474815E3294F926C737834FA\",\"server-timing\":\"inner; dur=5, cdn-cache;desc=MISS,edge;dur=653,origin;dur=14\",\"x-tt-trace-host\":\"012d6b6cae97bb056dd2540a62f7f6e52ccfb433febe931be5ddf1ac674ca77c57b7dee570790477696b7b5c17bbdd77bf841100de6d411ef8ccc26681b9799e4aee97897e0681b3a8b304853d1848fcbd9fec458fdd375cddc4a7873cdbcef3a0\",\"x-tt-trace-id\":\"00-0fa6216304269466036ad7b0703104d2-0fa6216304269466-01\",\"x-tt-trace-tag\":\"id=03;cdn-cache=miss;type=dyn\"},\"ifconfig\":{\"ipv4_reachable\":true,\"ipv6_reachable\":false},\"multinetwork\":{},\"nqe\":{\"bs_sys_state\":false,\"downlink_throughput\":2658,\"effective_net_type\":5,\"group_rtt_estimate\":[],\"http_rtt\":116,\"pending_requests\":\"1\",\"real_net_type\":2,\"tcp_rtt\":46,\"total_requests\":\"6\"},\"other\":{\"core_ver\":\"87.0.4273.1\",\"gateway\":\"0.0.0.0\",\"is_background\":false,\"is_internal_retry\":false,\"is_main_process\":true,\"libcore\":\"cronet\",\"net_thread_priority\":0,\"quic_commit\":\"6ea2111b/2023-03-16\",\"resume_time\":\"-1\",\"retry_attempts\":0,\"ttnet_build_timestamp\":\"1689596445\",\"ttnet_version\":\"4.1.127.2-tob\",\"version\":\"534436af/2023-07-17\"},\"response\":{\"cached\":false,\"code\":400,\"connection_info\":1,\"network_accessed\":true,\"received_bytes\":0,\"sent_bytes\":0,\"via_proxy\":false},\"session\":{},\"socket\":{\"address_family\":1,\"cip\":\"\",\"connection_attemps\":[],\"happy_eyeballs_result\":2,\"remote\":\"163.181.50.229:443\",\"socket_reused\":false},\"socketpool\":{\"assign_type\":0,\"consume_time\":0,\"is_backup_job\":false,\"pending_reason\":0,\"pending_time\":0,\"reuse_type\":0,\"socket_idle_time\":0},\"ssl\":{\"cert_status\":0,\"connection_status\":6296321,\"handshake_type\":1,\"is_issued_by_known_root\":true,\"server_cert_common_name\":\"*.bytedance.com\"},\"store_idc\":{\"init\":1},\"timing\":{\"detailed_duration\":{\"before_stream\":0,\"body_recv\":-1,\"cdn-cache\":\"miss\",\"connect_delay\":0,\"create_stream\":98,\"dispatch\":0,\"dns\":0,\"edge\":653,\"header_recv\":-1,\"inner\":5,\"origin\":9,\"proxy\":0,\"rtt\":43,\"security_callback\":0,\"send\":0,\"ssl\":48,\"tcp\":49,\"ttfb\":710},\"request\":{\"create_stream_finish\":1744019267.633028,\"create_stream_start\":1744019267.534112,\"duration\":812,\"http_transaction_start\":1744019267.534081,\"request_sent_time\":1744019267.633497,\"response_recv_time\":1744019268.344466,\"start_time\":1744019267.533685}},\"tnc\":{\"data\":{\"canary\":\"\",\"cold_start\":1,\"config\":\"\",\"config_updated\":0,\"local_probe_version\":\"\",\"rules\":\"\",\"source\":0,\"update_time\":\"-1\"},\"probe\":{\"cmd\":\"0\",\"get_time\":\"-1\",\"send_time\":\"-1\",\"state\":1,\"version\":\"0\"},\"request\":{\"probe_version\":\"0\",\"source\":0,\"start_time\":\"-1\",\"state\":1}},\"t \ No newline at end of file + } + }, + { + "type": "function", + "function": { + "name": "make_phone_call", + "description": "拨打电话", + "parameters": { + "type": "object", + "properties": { + "contact": { + "type": "string", + "description": "联系人姓名或电话号码" + } + }, + "required": [ + "contact" + ] + } + } + }, + { + "type": "function", + "function": { + "name": "open_app", + "description": "当用户请求打开特定应用时，启动相应应用", + "parameters": { + "type": "object", + "properties": { + "app_name": { + "type": "string", + "description": "应用名称" + } + }, + "required": [ + "app_name" + ] + } + } + } + ] +}' + From 2003d5d28c8b82ca5503a898e1b99f8d8ba7454d Mon Sep 17 00:00:00 2001 From: tanlongsheng <252620078@qq.com> Date: Sat, 12 Apr 2025 10:33:09 +0800 Subject: [PATCH 11/14] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=AB=98=E5=BE=B7sdk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- amap_flutter/android/build.gradle | 2 +- android/app/src/main/AndroidManifest.xml | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/amap_flutter/android/build.gradle b/amap_flutter/android/build.gradle index 6b87067f5..393dff90b 100644 --- a/amap_flutter/android/build.gradle +++ b/amap_flutter/android/build.gradle @@ -36,7 +36,7 @@ android { } dependencies { - implementation 'com.amap.api:location:6.4.9' + implementation 'com.amap.api:3dmap-location-search:latest.integration' testImplementation("junit:junit:4.13.2") testImplementation("org.mockito:mockito-core:5.0.0") diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index bfcb323d3..d2e406498 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -27,6 +27,18 @@ + + + + + + + + + + + + Date: Sat, 12 Apr 2025 18:41:43 +0800 Subject: [PATCH 12/14] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=BA=94=E7=94=A8?= =?UTF-8?q?=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- android/app/src/main/AndroidManifest.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index d2e406498..5def770c9 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -41,7 +41,7 @@ Date: Sat, 12 Apr 2025 21:17:34 +0100 Subject: [PATCH 13/14] add --- .mcp.json | 12 + .../yunqiinnovation/deepsound/MainActivity.kt | 6 +- .../deepsound/VoiceInteractionHandler.kt | 5 +- .../deepsound/VoiceInteractionService.kt | 6 +- android/build.gradle.kts | 2 + .../services/open_ai_service_adapter.dart | 24 + .../services/voice_interaction_service.dart | 32 +- .../controllers/jieli_test_controller.dart | 105 ++- .../jieli_test/views/jieli_test_view.dart | 74 ++ .../com/yunqiinnovation/jieli/BleService.kt | 269 ++++--- .../com/yunqiinnovation/jieli/JieliPlugin.kt | 25 +- local_plugins/jieli/lib/jieli.dart | 84 ++ .../open_ai_service/android/build.gradle.kts | 5 + .../open_ai_service/MCPClient.kt | 747 +++++++++++++----- .../open_ai_service/OpenAIService.kt | 10 +- .../open_ai_service/OpenAIServicePlugin.kt | 3 +- .../mcp/CustomSseClientTransport.kt | 357 +++++++++ .../open_ai_service/lib/open_ai_service.dart | 3 + pubspec.yaml | 1 + test.json | 14 +- 20 files changed, 1460 insertions(+), 324 deletions(-) create mode 100644 .mcp.json create mode 100644 local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/mcp/CustomSseClientTransport.kt diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 000000000..a344e9c9a --- /dev/null +++ b/.mcp.json @@ -0,0 +1,12 @@ +{ + "mcpServers": { + + "amap-amap-sse": { + "url": "https://mcp.amap.com/sse?key=e5fdc9605eabdeb5626f18f5721f343d" + }, + "web-search": { + "url": "http://mcp.ideapsound.com:8000/sse" + } + + } + } \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt index 1c4717f11..ffc3d256a 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt @@ -172,6 +172,7 @@ class MainActivity: FlutterActivity() { var openaiModel: String? = null var volcanoSpeechAppId: String = "" var volcanoSpeechAppToken: String = "" + var mcpServer: String = "" // 安全存储相关常量 private const val SECURE_PREFS_FILENAME = "deep_voice_secure_prefs" @@ -184,6 +185,7 @@ class MainActivity: FlutterActivity() { private const val KEY_MCP_SERVER_ENDPOINT = "mcp_server_endpoint" private const val KEY_VOLCANO_SPEECH_APP_ID = "volcano_speech_app_id" private const val KEY_VOLCANO_SPEECH_APP_TOKEN = "volcano_speech_app_token" + private const val KEY_MCP_SERVER = "mcp_server" // 会话管理 private const val KEY_SESSION_ID = "session_id" @@ -224,6 +226,7 @@ class MainActivity: FlutterActivity() { .putString(KEY_OPENAI_MODEL, openaiModel) .putString(KEY_VOLCANO_SPEECH_APP_ID, volcanoSpeechAppId) .putString(KEY_VOLCANO_SPEECH_APP_TOKEN, volcanoSpeechAppToken) + .putString(KEY_MCP_SERVER, mcpServer) .apply() FileLogger.d("MainActivity", "密钥已安全保存到加密存储中") @@ -272,6 +275,7 @@ class MainActivity: FlutterActivity() { openaiModel = sharedPreferences.getString(KEY_OPENAI_MODEL, null) volcanoSpeechAppId = sharedPreferences.getString(KEY_VOLCANO_SPEECH_APP_ID, "") ?: "" volcanoSpeechAppToken = sharedPreferences.getString(KEY_VOLCANO_SPEECH_APP_TOKEN, "") ?: "" + mcpServer = sharedPreferences.getString(KEY_MCP_SERVER, "") ?: "" FileLogger.d("MainActivity", "已从加密存储加载密钥") // 检查是否成功获取所有必要密钥 @@ -520,7 +524,7 @@ class MainActivity: FlutterActivity() { openaiModel = call.argument("openai_model") volcanoSpeechAppId = call.argument("volcano_speech_app_id") ?: "" volcanoSpeechAppToken = call.argument("volcano_speech_app_token") ?: "" - + mcpServer = call.argument("mcp_config") ?: "" // 保存密钥到安全存储 saveKeysToSecureStorage(applicationContext) diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt index b885117c0..cf6bf560a 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt @@ -25,7 +25,8 @@ class VoiceInteractionHandler( private val openaiBaseUrl: String = "", private val openaiModel: String = "", private val volcanoSpeechAppId: String, - private val volcanoSpeechAppToken: String + private val volcanoSpeechAppToken: String, + private val mcpServer: String ) { private val TAG = "VoiceInteractionHandler" @@ -80,7 +81,7 @@ class VoiceInteractionHandler( } // 初始化OpenAI服务 - openAIService.initialize(openaiApiKey, openaiBaseUrl, openaiModel) + openAIService.initialize(openaiApiKey, openaiBaseUrl, openaiModel, mcpServer) // 注册广播接收器 try { diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt index dbe64c79e..0497458d2 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt @@ -147,6 +147,7 @@ class VoiceInteractionService : Service() { var openaiModel = MainActivity.openaiModel ?: "" // OpenAI模型 var volcanoSpeechAppId = MainActivity.volcanoSpeechAppId ?: "" var volcanoSpeechAppToken = MainActivity.volcanoSpeechAppToken ?: "" + var mcpServer = MainActivity.mcpServer ?: "" // 如果静态变量中没有配置，尝试从加密存储中加载 if (subscriptionKey.isEmpty() || serviceRegion.isEmpty() || openaiKey.isEmpty()) { @@ -164,7 +165,7 @@ class VoiceInteractionService : Service() { openaiModel = MainActivity.openaiModel ?: "" volcanoSpeechAppId = MainActivity.volcanoSpeechAppId ?: "" volcanoSpeechAppToken = MainActivity.volcanoSpeechAppToken ?: "" - + mcpServer = MainActivity.mcpServer ?: "" FileLogger.d(TAG, "已从加密存储加载配置信息") } else { FileLogger.e(TAG, "无法从加密存储加载配置信息") @@ -175,7 +176,8 @@ class VoiceInteractionService : Service() { voiceInteractionHandler = VoiceInteractionHandler(applicationContext, subscriptionKey, serviceRegion, openaiKey, openaiBaseUrl, openaiModel, - volcanoSpeechAppId, volcanoSpeechAppToken) + volcanoSpeechAppId, volcanoSpeechAppToken, + mcpServer) // 初始化回调 voiceInteractionHandler.setCallback(object : VoiceInteractionHandler.InteractionCallback { diff --git a/android/build.gradle.kts b/android/build.gradle.kts index ca0ce6220..88678470b 100644 --- a/android/build.gradle.kts +++ b/android/build.gradle.kts @@ -8,6 +8,8 @@ allprojects { maven { url = uri("https://artifact.bytedance.com/repository/Volcengine/") } + maven { url = uri("https://maven.pkg.jetbrains.space/public/p/ktor/eap") } + } } diff --git a/lib/data/services/open_ai_service_adapter.dart b/lib/data/services/open_ai_service_adapter.dart index b93ce6cf0..0e018524d 100644 --- a/lib/data/services/open_ai_service_adapter.dart +++ b/lib/data/services/open_ai_service_adapter.dart @@ -1,8 +1,12 @@ import 'dart:async'; +import 'dart:io'; +import 'dart:convert'; import 'package:flutter_dotenv/flutter_dotenv.dart'; import 'package:open_ai_service/open_ai_service.dart'; import 'package:get/get.dart'; import 'ai_service.dart'; +import 'package:flutter/services.dart'; +import 'package:path_provider/path_provider.dart'; /// 定义流事件类型，用于区分不同类型的事件 enum StreamEventType { token, complete, error } @@ -105,11 +109,31 @@ class OpenAIServiceAdapter implements AiService { return false; } + // 读取.mcp.json文件 + String mcpConfig = ''; + try { + // 从Flutter资源包中加载.mcp.json + mcpConfig = await rootBundle.loadString('.mcp.json'); + printInfo(info: '成功从资源包加载.mcp.json配置文件'); + + // 验证JSON格式 + final jsonData = jsonDecode(mcpConfig); + if (jsonData is Map && jsonData.containsKey('mcpServers')) { + printInfo(info: '解析到有效的mcpServers配置'); + } else { + printInfo(info: '.mcp.json内容格式不正确，期望包含mcpServers字段'); + } + } catch (e) { + printError(info: '加载.mcp.json资源文件时出错: $e'); + + } + // 初始化OpenAI服务 final result = await _openAIService.initialize( apiKey: apiKey, baseUrl: baseUrl, model: model, + mcpServer: mcpConfig, ); if (result) { diff --git a/lib/data/services/voice_interaction_service.dart b/lib/data/services/voice_interaction_service.dart index 6adafa5c4..1c6e330dd 100644 --- a/lib/data/services/voice_interaction_service.dart +++ b/lib/data/services/voice_interaction_service.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:convert'; import 'package:flutter/services.dart'; import 'package:get/get.dart'; import 'package:flutter_dotenv/flutter_dotenv.dart'; @@ -44,6 +45,9 @@ class VoiceInteractionService extends GetxService { late String _openaiModel; late String _volcanoSpeechAppId; late String _volcanoSpeechAppToken; + // MCP配置信息 + String? _mcpConfig; + // 聊天历史服务 late final ChatHistoryService _chatHistoryService; @@ -71,6 +75,21 @@ class VoiceInteractionService extends GetxService { Logger.warning('未找到 Azure 语音服务配置。请在 .env 文件中设置 AZURE_SPEECH_KEY 和 AZURE_SPEECH_REGION'); } + // 加载MCP配置 + _loadMcpConfig(); + } + + /// 加载MCP配置 + void _loadMcpConfig() { + try { + rootBundle.loadString('.mcp.json').then((config) { + _mcpConfig = config; + Logger.info('成功加载MCP配置, 配置内容: $_mcpConfig'); + }); + } catch (e) { + Logger.warning('加载MCP配置失败: $e'); + _mcpConfig = null; + } } /// 处理来自原生层的事件 @@ -212,8 +231,8 @@ class VoiceInteractionService extends GetxService { try { Logger.info('启动语音交互服务...'); - // 使用已加载的配置信息 - final result = await _channel.invokeMethod('startService', { + // 构建参数 + final Map params = { 'azure_speech_key': _azureSpeechKey, 'azure_speech_region': _azureSpeechRegion, 'openai_api_key': _openaiApiKey, @@ -221,7 +240,14 @@ class VoiceInteractionService extends GetxService { 'openai_model': _openaiModel, 'volcano_speech_app_id': _volcanoSpeechAppId, 'volcano_speech_app_token': _volcanoSpeechAppToken, - }) ?? false; + }; + + // 如果有MCP配置，添加到参数中 + if (_mcpConfig != null) { + params['mcp_config'] = _mcpConfig; + } + + final result = await _channel.invokeMethod('startService', params) ?? false; if (result) { _isServiceRunning.value = true; diff --git a/lib/modules/jieli_test/controllers/jieli_test_controller.dart b/lib/modules/jieli_test/controllers/jieli_test_controller.dart index 8afa86cf4..b6d896836 100644 --- a/lib/modules/jieli_test/controllers/jieli_test_controller.dart +++ b/lib/modules/jieli_test/controllers/jieli_test_controller.dart @@ -1,13 +1,11 @@ import 'dart:async'; import 'dart:typed_data'; +import 'package:flutter/material.dart' hide ConnectionState; import 'package:get/get.dart'; -import 'package:logger/logger.dart'; import 'package:jieli/jieli.dart'; /// 杰理测试控制器 class JieliTestController extends GetxController { - static final Logger logger = Logger(); - // Jieli插件实例 late Jieli jieli; @@ -21,23 +19,23 @@ class JieliTestController extends GetxController { var audioDataSize = 0.obs; var receivedFrames = 0.obs; var audioDataReceiving = false.obs; + var volumeCommandResult = ''.obs; // 事件订阅 StreamSubscription? _scanSubscription; StreamSubscription? _connectionSubscription; StreamSubscription? _wakeupSubscription; StreamSubscription? _audioSubscription; + StreamSubscription? _commandResultSubscription; @override void onInit() { super.onInit(); - logger.d('JieliTestController 初始化'); _initializeJieli(); } @override void onClose() { - logger.d('JieliTestController 关闭'); _cancelAllSubscriptions(); _disposeJieli(); super.onClose(); @@ -49,16 +47,16 @@ class JieliTestController extends GetxController { _connectionSubscription?.cancel(); _wakeupSubscription?.cancel(); _audioSubscription?.cancel(); + _commandResultSubscription?.cancel(); } /// 初始化杰理SDK void _initializeJieli() { try { - logger.d('初始化Jieli插件'); jieli = Jieli(); _setupEventListeners(); } catch (e) { - logger.e('初始化Jieli插件错误', error: e); + // 初始化错误处理 } } @@ -66,13 +64,11 @@ class JieliTestController extends GetxController { void _setupEventListeners() { // 监听扫描结果 _scanSubscription = jieli.scanResults.listen((devices) { - // logger.d('扫描到设备: ${devices.length}个'); scannedDevices.assignAll(devices); }); // 监听连接状态 _connectionSubscription = jieli.connectionState.listen((state) { - logger.d('连接状态变化: $state'); switch (state) { case ConnectionState.connected: connectStatus.value = '已连接'; @@ -95,7 +91,6 @@ class JieliTestController extends GetxController { // 监听唤醒信号 _wakeupSubscription = jieli.wakeupStream.listen((wakeup) { - logger.d('收到唤醒信号'); wakeupReceived.value = true; Get.snackbar('收到唤醒信号', '耳机发送了唤醒信号'); }); @@ -106,10 +101,21 @@ class JieliTestController extends GetxController { audioDataSize.value += data.length; receivedFrames.value++; audioDataReceiving.value = true; - - // 每100帧更新一次日志，避免日志过多 - if (receivedFrames.value % 100 == 0) { - logger.d('已接收音频数据: ${audioDataSize.value} 字节, ${receivedFrames.value} 帧'); + }); + + // 监听命令结果 + _commandResultSubscription = jieli.commandResultStream.listen((result) { + // 处理音量调整命令结果 + if (result.commandType == Jieli.commandTypeVolume) { + volumeCommandResult.value = result.success ? '音量调整成功' : '音量调整失败'; + Get.snackbar( + result.success ? '音量调整成功' : '音量调整失败', + result.success ? '设备已更新音量' : '设备未能更新音量', + snackPosition: SnackPosition.BOTTOM, + backgroundColor: result.success ? Colors.green.withOpacity(0.7) : Colors.red.withOpacity(0.7), + colorText: Colors.white, + duration: Duration(seconds: 2), + ); } }); } @@ -119,12 +125,10 @@ class JieliTestController extends GetxController { try { final isEnabled = await jieli.isBluetoothEnabled(); if (!isEnabled) { - logger.d('蓝牙未启用，请求启用蓝牙'); return await jieli.requestBluetoothEnable(); } return true; } catch (e) { - logger.e('检查蓝牙状态错误', error: e); return false; } } @@ -140,7 +144,6 @@ class JieliTestController extends GetxController { } try { - logger.d('开始扫描设备'); isScanning.value = true; // 清空之前的设备列表 @@ -150,7 +153,6 @@ class JieliTestController extends GetxController { final success = await jieli.startScan(scanTimeout: 10000); if (!success) { - logger.e('启动扫描失败'); isScanning.value = false; Get.snackbar('扫描失败', '启动扫描设备失败，请检查蓝牙权限'); } @@ -161,7 +163,6 @@ class JieliTestController extends GetxController { }); } catch (e) { - logger.e('扫描设备错误', error: e); isScanning.value = false; Get.snackbar('扫描错误', e.toString()); } @@ -172,11 +173,10 @@ class JieliTestController extends GetxController { if (!isScanning.value) return; try { - logger.d('停止扫描'); await jieli.stopScan(); isScanning.value = false; } catch (e) { - logger.e('停止扫描错误', error: e); + // 停止扫描错误处理 } } @@ -190,7 +190,6 @@ class JieliTestController extends GetxController { // 更新UI状态 connectStatus.value = '连接中...'; - logger.d('连接设备: ${device.name} (${device.id})'); // 重置音频和唤醒相关的状态 wakeupReceived.value = false; @@ -205,14 +204,11 @@ class JieliTestController extends GetxController { // 连接成功，更新当前设备 currentDevice.value = device; deviceInfo.value = '设备名称: ${device.name}\n设备ID: ${device.id}\nRSSI: ${device.rssi} dBm'; - logger.d('设备连接成功'); } else { deviceInfo.value = '连接失败'; - logger.d('设备连接失败'); } } catch (e) { deviceInfo.value = '连接错误: ${e.toString()}'; - logger.e('连接设备错误', error: e); } } @@ -222,19 +218,15 @@ class JieliTestController extends GetxController { try { connectStatus.value = '断开中...'; - logger.d('断开设备连接'); final success = await jieli.disconnect(); if (success) { deviceInfo.value = '未连接设备'; - logger.d('设备断开连接成功'); - } else { - logger.d('设备断开连接失败'); } } catch (e) { - logger.e('断开设备错误', error: e); + // 断开连接错误处理 } } @@ -246,18 +238,14 @@ class JieliTestController extends GetxController { } try { - logger.d('发送开始录音命令'); final success = await jieli.sendStartRecordingCommand(); if (success) { Get.snackbar('命令已发送', '已发送开始录音命令'); - logger.d('开始录音命令发送成功'); } else { Get.snackbar('命令发送失败', '发送开始录音命令失败'); - logger.d('开始录音命令发送失败'); } } catch (e) { - logger.e('发送开始录音命令错误', error: e); Get.snackbar('命令错误', e.toString()); } } @@ -270,29 +258,66 @@ class JieliTestController extends GetxController { } try { - logger.d('发送停止录音命令'); final success = await jieli.sendStopRecordingCommand(); if (success) { Get.snackbar('命令已发送', '已发送停止录音命令'); - logger.d('停止录音命令发送成功'); } else { Get.snackbar('命令发送失败', '发送停止录音命令失败'); - logger.d('停止录音命令发送失败'); } } catch (e) { - logger.e('发送停止录音命令错误', error: e); Get.snackbar('命令错误', e.toString()); } } + /// 增加音量 + Future increaseVolume() async { + if (currentDevice.value == null) { + Get.snackbar('操作失败', '未连接设备'); + return; + } + + try { + volumeCommandResult.value = '发送中...'; + final success = await jieli.increaseVolume(); + + if (!success) { + Get.snackbar('命令发送失败', '发送增加音量命令失败'); + volumeCommandResult.value = '发送失败'; + } + } catch (e) { + Get.snackbar('命令错误', e.toString()); + volumeCommandResult.value = '发送出错'; + } + } + + /// 减小音量 + Future decreaseVolume() async { + if (currentDevice.value == null) { + Get.snackbar('操作失败', '未连接设备'); + return; + } + + try { + volumeCommandResult.value = '发送中...'; + final success = await jieli.decreaseVolume(); + + if (!success) { + Get.snackbar('命令发送失败', '发送减小音量命令失败'); + volumeCommandResult.value = '发送失败'; + } + } catch (e) { + Get.snackbar('命令错误', e.toString()); + volumeCommandResult.value = '发送出错'; + } + } + /// 清理资源 void _disposeJieli() { try { - logger.d('释放Jieli插件资源'); jieli.dispose(); } catch (e) { - logger.e('释放Jieli插件资源错误', error: e); + // 释放资源错误处理 } } } \ No newline at end of file diff --git a/lib/modules/jieli_test/views/jieli_test_view.dart b/lib/modules/jieli_test/views/jieli_test_view.dart index aa6316366..46409651b 100644 --- a/lib/modules/jieli_test/views/jieli_test_view.dart +++ b/lib/modules/jieli_test/views/jieli_test_view.dart @@ -260,6 +260,80 @@ class JieliTestView extends GetView { ), ], ), + SizedBox(height: 8.h), + Row( + children: [ + Expanded( + child: ElevatedButton.icon( + onPressed: controller.increaseVolume, + icon: Icon( + Icons.volume_up, + size: 18.sp, + ), + label: Text( + '增加音量', + style: TextStyle(fontSize: 14.sp), + ), + style: ElevatedButton.styleFrom( + backgroundColor: Colors.blue[600], + padding: EdgeInsets.symmetric(vertical: 12.h), + shape: RoundedRectangleBorder( + borderRadius: BorderRadius.circular(8.r), + ), + ), + ), + ), + SizedBox(width: 8.w), + Expanded( + child: ElevatedButton.icon( + onPressed: controller.decreaseVolume, + icon: Icon( + Icons.volume_down, + size: 18.sp, + ), + label: Text( + '减小音量', + style: TextStyle(fontSize: 14.sp), + ), + style: ElevatedButton.styleFrom( + backgroundColor: Colors.orange[600], + padding: EdgeInsets.symmetric(vertical: 12.h), + shape: RoundedRectangleBorder( + borderRadius: BorderRadius.circular(8.r), + ), + ), + ), + ), + ], + ), + SizedBox(height: 8.h), + Obx(() => controller.volumeCommandResult.value.isNotEmpty + ? Container( + width: double.infinity, + padding: EdgeInsets.symmetric(vertical: 8.h, horizontal: 16.w), + decoration: BoxDecoration( + color: controller.volumeCommandResult.value.contains('成功') + ? Colors.green[100] + : (controller.volumeCommandResult.value == '发送中...' + ? Colors.blue[100] + : Colors.red[100]), + borderRadius: BorderRadius.circular(8.r), + ), + child: Text( + '音量控制: ${controller.volumeCommandResult.value}', + style: TextStyle( + fontSize: 13.sp, + color: controller.volumeCommandResult.value.contains('成功') + ? Colors.green[900] + : (controller.volumeCommandResult.value == '发送中...' + ? Colors.blue[900] + : Colors.red[900]), + ), + textAlign: TextAlign.center, + ), + ) + : SizedBox.shrink() + ), ], ); } diff --git a/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/BleService.kt b/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/BleService.kt index dec530b53..4534aade2 100644 --- a/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/BleService.kt +++ b/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/BleService.kt @@ -12,34 +12,6 @@ import androidx.lifecycle.MutableLiveData import kotlinx.coroutines.* import java.util.* -/** - * BLE回调接口，统一处理蓝牙事件 - */ -interface BleCallback { - /** - * 扫描结果回调 - * @param devices 扫描到的设备列表 - */ - fun onScanResult(devices: List>) - - /** - * 连接状态变化回调 - * @param state 连接状态 - */ - fun onConnectionStateChanged(state: Int) - - /** - * 音频数据回调 - * @param data 音频数据字节数组 - */ - fun onAudioDataReceived(data: ByteArray) - - /** - * 唤醒信号回调 - */ - fun onWakeupSignalReceived() -} - /** * BLE服务类，提供核心蓝牙功能 */ @@ -47,12 +19,11 @@ class BleService(private val context: Context) { companion object { private const val TAG = "JieliBleService" - // 服务和特征UUID - private const val AUDIO_SERVICE_UUID = "0000A100-0000-1000-8000-00805F9B34FB" - private const val AUDIO_DATA_CHAR_UUID = "0000A101-0000-1000-8000-00805F9B34FB" - private const val CONTROL_CHAR_UUID = "0000A102-0000-1000-8000-00805F9B34FB" - private const val CLIENT_CHAR_CONFIG_UUID = "00002902-0000-1000-8000-00805f9b34fb" - + // 服务和特征UUID - 根据协议规范重新定义 + private const val AUDIO_SERVICE_UUID = "0000faa0-0000-1000-8000-00805f9b34fb" // 主服务UUID + private const val NOTIFY_CHAR_UUID = "0000faa2-0000-1000-8000-00805f9b34fb" // 用于接收数据的特征(notify) + private const val WRITE_CHAR_UUID = "0000faa1-0000-1000-8000-00805f9b34fb" // 用于发送命令的特征(write without response) + private const val CLIENT_CHAR_CONFIG_UUID = "00002902-0000-1000-8000-00805f9b34fb" // 标准的客户端特征配置描述符 // 连接状态常量 const val STATE_DISCONNECTED = 0 const val STATE_CONNECTING = 1 @@ -62,6 +33,20 @@ class BleService(private val context: Context) { // 扫描结果更新间隔(毫秒) private const val SCAN_RESULT_UPDATE_INTERVAL = 500L + + // 命令类型常量 + const val COMMAND_VOLUME = 1 + } + + /** + * 回调接口 + */ + interface Callback { + fun onScanResult(devices: List>) + fun onConnectionStateChanged(state: Int) + fun onAudioDataReceived(data: ByteArray) + fun onWakeupSignalReceived() + fun onCommandResult(commandType: Int, success: Boolean) } // 蓝牙管理器和适配器 @@ -71,8 +56,8 @@ class BleService(private val context: Context) { // GATT连接和特征 private var bluetoothGatt: BluetoothGatt? = null - private var audioDataChar: BluetoothGattCharacteristic? = null - private var controlChar: BluetoothGattCharacteristic? = null + private var notifyChar: BluetoothGattCharacteristic? = null + private var writeChar: BluetoothGattCharacteristic? = null // 扫描相关 private val scanHandler = Handler(Looper.getMainLooper()) @@ -88,7 +73,7 @@ class BleService(private val context: Context) { val connectionState = MutableLiveData(STATE_DISCONNECTED) // 统一回调接口 - var callback: BleCallback? = null + var callback: Callback? = null /** * 开始扫描BLE设备 @@ -252,8 +237,8 @@ class BleService(private val context: Context) { // 清理资源 gatt.close() bluetoothGatt = null - audioDataChar = null - controlChar = null + notifyChar = null + writeChar = null } else if (status != BluetoothGatt.GATT_SUCCESS) { Log.e(TAG, "连接失败，status=$status") updateConnectionState(STATE_ERROR) @@ -272,13 +257,16 @@ class BleService(private val context: Context) { if (audioService != null) { Log.i(TAG, "找到音频服务") - // 获取音频数据和控制特征 - audioDataChar = audioService.getCharacteristic(UUID.fromString(AUDIO_DATA_CHAR_UUID)) - controlChar = audioService.getCharacteristic(UUID.fromString(CONTROL_CHAR_UUID)) + // 获取特征 + notifyChar = audioService.getCharacteristic(UUID.fromString(NOTIFY_CHAR_UUID)) + writeChar = audioService.getCharacteristic(UUID.fromString(WRITE_CHAR_UUID)) - if (audioDataChar != null && controlChar != null) { + if (notifyChar != null && writeChar != null) { Log.i(TAG, "找到所需特征") + // 设置特征写入类型为无响应模式 + writeChar?.writeType = BluetoothGattCharacteristic.WRITE_TYPE_NO_RESPONSE + // 注册特征通知 setupNotifications(gatt) } else { @@ -297,22 +285,28 @@ class BleService(private val context: Context) { override fun onCharacteristicChanged(gatt: BluetoothGatt, characteristic: BluetoothGattCharacteristic) { val characteristicUuid = characteristic.uuid.toString() + Log.d(TAG, "收到特征变化通知: $characteristicUuid") when (characteristicUuid) { - AUDIO_DATA_CHAR_UUID -> { - // 收到音频数据 - val audioData = characteristic.value - Log.d(TAG, "收到音频数据帧，大小: ${audioData.size} 字节") + // NOTIFY_CHAR_UUID -> { + // // 收到音频数据 + // val audioData = characteristic.value + // Log.d(TAG, "收到音频数据帧，大小: ${audioData.size} 字节") - callback?.onAudioDataReceived(audioData) - } - CONTROL_CHAR_UUID -> { - // 收到唤醒信号 + // callback?.onAudioDataReceived(audioData) + // } + NOTIFY_CHAR_UUID -> { + // 处理控制消息 val value = characteristic.value Log.i(TAG, "收到控制通知: ${value.contentToString()}") + // 判断是否是协议格式响应 + if (value.isNotEmpty() && value[0].toInt() and 0xFF == 0xA5) { + // 处理设备返回的协议数据 + processResponseData(value) + } // 假设0x01表示唤醒信号 - if (value.isNotEmpty() && value[0].toInt() == 0x01) { + else if (value.isNotEmpty() && value[0].toInt() == 0x01) { Log.i(TAG, "收到唤醒信号") callback?.onWakeupSignalReceived() @@ -325,7 +319,7 @@ class BleService(private val context: Context) { } override fun onCharacteristicWrite(gatt: BluetoothGatt, characteristic: BluetoothGattCharacteristic, status: Int) { - if (characteristic.uuid.toString() == CONTROL_CHAR_UUID) { + if (characteristic.uuid.toString() == WRITE_CHAR_UUID) { if (status == BluetoothGatt.GATT_SUCCESS) { Log.i(TAG, "命令发送成功: ${characteristic.value.contentToString()}") } else { @@ -337,13 +331,10 @@ class BleService(private val context: Context) { override fun onDescriptorWrite(gatt: BluetoothGatt, descriptor: BluetoothGattDescriptor, status: Int) { if (status == BluetoothGatt.GATT_SUCCESS) { Log.i(TAG, "描述符写入成功: ${descriptor.uuid}") - - // 如果是音频数据特征的描述符，再订阅控制特征 - if (descriptor.characteristic.uuid.toString() == AUDIO_DATA_CHAR_UUID) { - setupControlNotification(gatt) - } + Log.i(TAG, "特征通知设置成功，设备就绪") } else { Log.e(TAG, "描述符写入失败，status=$status") + updateConnectionState(STATE_ERROR) } } } @@ -352,57 +343,92 @@ class BleService(private val context: Context) { * 设置特征通知 */ private fun setupNotifications(gatt: BluetoothGatt) { - // 先订阅音频数据通知 - if (audioDataChar?.properties?.and(BluetoothGattCharacteristic.PROPERTY_NOTIFY) != 0) { - gatt.setCharacteristicNotification(audioDataChar, true) + // 启用 notify 特征通知 + if (notifyChar?.properties?.and(BluetoothGattCharacteristic.PROPERTY_NOTIFY) != 0) { + gatt.setCharacteristicNotification(notifyChar, true) - val descriptor = audioDataChar?.getDescriptor(UUID.fromString(CLIENT_CHAR_CONFIG_UUID)) + val descriptor = notifyChar?.getDescriptor(UUID.fromString(CLIENT_CHAR_CONFIG_UUID)) descriptor?.value = BluetoothGattDescriptor.ENABLE_NOTIFICATION_VALUE if (descriptor != null) { gatt.writeDescriptor(descriptor) - Log.i(TAG, "设置音频数据特征通知") + Log.i(TAG, "设置数据通知特征通知") } } else { - Log.e(TAG, "音频数据特征不支持通知") - setupControlNotification(gatt) + Log.e(TAG, "数据通知特征不支持通知") } } /** - * 设置控制特征通知 + * 处理设备返回的数据 + * 协议格式: + * 协议头(0xA5) + 字段类型(1byte) + 数据长度(1byte) + 数据域(N byte) + 校验和(1byte) */ - private fun setupControlNotification(gatt: BluetoothGatt) { - if (controlChar?.properties?.and(BluetoothGattCharacteristic.PROPERTY_NOTIFY) != 0) { - gatt.setCharacteristicNotification(controlChar, true) - - val descriptor = controlChar?.getDescriptor(UUID.fromString(CLIENT_CHAR_CONFIG_UUID)) - descriptor?.value = BluetoothGattDescriptor.ENABLE_NOTIFICATION_VALUE - - if (descriptor != null) { - gatt.writeDescriptor(descriptor) - Log.i(TAG, "设置控制特征通知") + private fun processResponseData(data: ByteArray) { + if (data.size < 4) { + Log.e(TAG, "返回数据格式错误，长度不足") + return + } + + // 判断协议头 + if (data[0].toInt() and 0xFF != 0xA5) { + Log.e(TAG, "返回数据协议头错误: ${data[0].toInt() and 0xFF}") + return + } + + val fieldType = data[1].toInt() and 0xFF + val dataLen = data[2].toInt() and 0xFF + + // 验证长度 + if (data.size != dataLen + 4) { // 协议头+字段类型+数据长度+数据域+校验和 + Log.e(TAG, "返回数据长度不匹配: 应为${dataLen + 4}，实际为${data.size}") + return + } + + // 计算校验和 + var checksum = 0 + for (i in 0 until data.size - 1) { + checksum += data[i].toInt() and 0xFF + } + checksum = checksum and 0xFF + + val receivedChecksum = data[data.size - 1].toInt() and 0xFF + + if (checksum != receivedChecksum) { + Log.e(TAG, "校验和错误: 计算值=$checksum, 接收值=$receivedChecksum") + return + } + + // 解析字段类型 + when (fieldType) { + 0x01 -> { + // 音量调整命令的响应 + val result = data[3].toInt() and 0xFF + val success = result == 0 + Log.i(TAG, "音量调整命令响应: ${if (success) "成功" else "失败"}") + callback?.onCommandResult(COMMAND_VOLUME, success) + } + // 其他命令类型可在此处添加 + else -> { + Log.d(TAG, "未知命令类型: ${String.format("0x%02X", fieldType)}") } - } else { - Log.e(TAG, "控制特征不支持通知") } } /** - * 发送开始录音命令（向控制特征写入0x01） + * 发送开始录音命令 */ fun sendStartRecordingCommand(): Boolean { - if (bluetoothGatt == null || controlChar == null || connectionState.value != STATE_CONNECTED) { + if (bluetoothGatt == null || writeChar == null || connectionState.value != STATE_CONNECTED) { Log.e(TAG, "无法发送开始录音命令，设备未连接") return false } try { val commandData = byteArrayOf(0x01) - controlChar?.value = commandData - controlChar?.writeType = BluetoothGattCharacteristic.WRITE_TYPE_DEFAULT + writeChar?.value = commandData - val success = bluetoothGatt?.writeCharacteristic(controlChar) ?: false + val success = bluetoothGatt?.writeCharacteristic(writeChar) ?: false if (success) { Log.i(TAG, "开始录音命令已发送") @@ -418,20 +444,19 @@ class BleService(private val context: Context) { } /** - * 发送停止录音命令（向控制特征写入0x02） + * 发送停止录音命令 */ fun sendStopRecordingCommand(): Boolean { - if (bluetoothGatt == null || controlChar == null || connectionState.value != STATE_CONNECTED) { + if (bluetoothGatt == null || writeChar == null || connectionState.value != STATE_CONNECTED) { Log.e(TAG, "无法发送停止录音命令，设备未连接") return false } try { val commandData = byteArrayOf(0x02) - controlChar?.value = commandData - controlChar?.writeType = BluetoothGattCharacteristic.WRITE_TYPE_DEFAULT + writeChar?.value = commandData - val success = bluetoothGatt?.writeCharacteristic(controlChar) ?: false + val success = bluetoothGatt?.writeCharacteristic(writeChar) ?: false if (success) { Log.i(TAG, "停止录音命令已发送") @@ -446,6 +471,70 @@ class BleService(private val context: Context) { } } + /** + * 发送音量调整命令 + * 协议格式: + * 协议头(0xAA) + 字段类型(0x01) + 数据长度(0x01) + 音量值(0~16) + 校验和 + */ + fun adjustVolume(volume: Int): Boolean { + if (bluetoothGatt == null || writeChar == null || connectionState.value != STATE_CONNECTED) { + Log.e(TAG, "无法发送音量调整命令，设备未连接") + return false + } + + // 限制音量范围为0-16 + val safeVolume = volume.coerceIn(0, 16) + + try { + // 构建命令数据 + val header: Byte = 0xAA.toByte() + val fieldType: Byte = 0x01 + val dataLen: Byte = 0x01 + val volumeData: Byte = safeVolume.toByte() + + // 计算校验和 + val checksum = ((header.toInt() and 0xFF) + + (fieldType.toInt() and 0xFF) + + (dataLen.toInt() and 0xFF) + + (volumeData.toInt() and 0xFF)) and 0xFF + + // 创建完整命令 + val commandData = byteArrayOf(header, fieldType, dataLen, volumeData, checksum.toByte()) + + // 写入特征 + writeChar?.value = commandData + + val success = bluetoothGatt?.writeCharacteristic(writeChar) ?: false + + if (success) { + Log.i(TAG, "音量调整命令已发送: $safeVolume") + } else { + Log.e(TAG, "音量调整命令发送失败") + } + + return success + } catch (e: Exception) { + Log.e(TAG, "发送音量调整命令时发生异常: ${e.message}") + return false + } + } + + /** + * 增加音量 + */ + fun increaseVolume(): Boolean { + // 假设当前音量保存在设备中，我们只需发送增加命令 + return adjustVolume(16) // 发送最大音量命令 + } + + /** + * 减小音量 + */ + fun decreaseVolume(): Boolean { + // 假设当前音量保存在设备中，我们只需发送减小命令 + return adjustVolume(0) // 发送最小音量命令 + } + /** * 清理资源 */ @@ -454,7 +543,7 @@ class BleService(private val context: Context) { scanHandler.removeCallbacks(scanResultUpdateRunnable) bluetoothGatt?.close() bluetoothGatt = null - audioDataChar = null - controlChar = null + notifyChar = null + writeChar = null } } \ No newline at end of file diff --git a/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt b/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt index 17cc5d21f..95a267e97 100644 --- a/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt +++ b/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt @@ -44,6 +44,7 @@ class JieliPlugin: FlutterPlugin, MethodCallHandler, ActivityAware, EventChannel private const val EVENT_TYPE_CONNECTION_STATE = "connection_state" private const val EVENT_TYPE_SCAN_RESULT = "scan_result" private const val EVENT_TYPE_WAKEUP_SIGNAL = "wakeup_signal" + private const val EVENT_TYPE_COMMAND_RESULT = "command_result" } // 蓝牙相关请求码 @@ -75,7 +76,7 @@ class JieliPlugin: FlutterPlugin, MethodCallHandler, ActivityAware, EventChannel */ private fun setupBleCallbacks() { // 设置统一回调接口 - bleService.callback = object : BleCallback { + bleService.callback = object : BleService.Callback { override fun onScanResult(devices: List>) { pluginScope.launch(Dispatchers.Main) { sendEvent(EVENT_TYPE_SCAN_RESULT, devices) @@ -99,6 +100,15 @@ class JieliPlugin: FlutterPlugin, MethodCallHandler, ActivityAware, EventChannel sendEvent(EVENT_TYPE_WAKEUP_SIGNAL, true) } } + + override fun onCommandResult(commandType: Int, success: Boolean) { + pluginScope.launch(Dispatchers.Main) { + sendEvent(EVENT_TYPE_COMMAND_RESULT, mapOf( + "commandType" to commandType, + "success" to success + )) + } + } } } @@ -187,6 +197,19 @@ class JieliPlugin: FlutterPlugin, MethodCallHandler, ActivityAware, EventChannel activity?.startActivityForResult(enableBtIntent, REQUEST_ENABLE_BT) result.success(true) } + "adjustVolume" -> { + val volume = call.argument("volume") ?: 0 + val success = bleService.adjustVolume(volume) + result.success(success) + } + "increaseVolume" -> { + val success = bleService.increaseVolume() + result.success(success) + } + "decreaseVolume" -> { + val success = bleService.decreaseVolume() + result.success(success) + } else -> { result.notImplemented() } diff --git a/local_plugins/jieli/lib/jieli.dart b/local_plugins/jieli/lib/jieli.dart index 78ec090bb..fa65eca91 100644 --- a/local_plugins/jieli/lib/jieli.dart +++ b/local_plugins/jieli/lib/jieli.dart @@ -46,6 +46,7 @@ class EventType { static const String connectionState = 'connection_state'; static const String scanResult = 'scan_result'; static const String wakeupSignal = 'wakeup_signal'; + static const String commandResult = 'command_result'; } /// 杰理BLE插件主类 @@ -72,10 +73,21 @@ class Jieli { static final StreamController _wakeupController = StreamController.broadcast(); Stream get wakeupStream => _wakeupController.stream; + + /// 命令结果流 + static final StreamController _commandResultController = + StreamController.broadcast(); + Stream get commandResultStream => _commandResultController.stream; + + /// 原始事件流 + Stream>? _eventStream; /// 事件流订阅 StreamSubscription? _eventSubscription; + /// 命令类型常量 + static const int commandTypeVolume = 1; + /// 构造函数，初始化平台通道和事件监听 Jieli() { _setupEventListener(); @@ -118,6 +130,16 @@ class Jieli { _wakeupController.add(data); } break; + case EventType.commandResult: + if (data is Map) { + final commandType = data['commandType'] as int; + final success = data['success'] as bool; + _commandResultController.add(CommandResult( + commandType: commandType, + success: success, + )); + } + break; } } @@ -172,6 +194,7 @@ class Jieli { _connectionController.close(); _audioController.close(); _wakeupController.close(); + _commandResultController.close(); } /// 检查蓝牙是否已启用 @@ -185,4 +208,65 @@ class Jieli { final result = await _methodChannel.invokeMethod('requestBluetoothEnable'); return result ?? false; } + + /// 调整音量 + /// [volume] 音量值，范围0-16 + Future adjustVolume(int volume) async { + try { + final result = await _methodChannel.invokeMethod( + 'adjustVolume', + {'volume': volume}, + ); + return result ?? false; + } catch (e) { + print('调整音量失败: $e'); + return false; + } + } + + /// 增加音量 + Future increaseVolume() async { + try { + final result = await _methodChannel.invokeMethod('increaseVolume'); + return result ?? false; + } catch (e) { + print('增加音量失败: $e'); + return false; + } + } + + /// 减小音量 + Future decreaseVolume() async { + try { + final result = await _methodChannel.invokeMethod('decreaseVolume'); + return result ?? false; + } catch (e) { + print('减小音量失败: $e'); + return false; + } + } + + /// 命令结果回调 + Stream get onCommandResult => commandResultStream + .where((event) => event.commandType == commandTypeVolume) + .map((event) => event); +} + +/// 蓝牙设备模型 +class BluetoothDevice { + // ... existing code ... +} + +/// 命令结果 +class CommandResult { + final int commandType; + final bool success; + + CommandResult({ + required this.commandType, + required this.success, + }); + + @override + String toString() => 'CommandResult{commandType: $commandType, success: $success}'; } \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/build.gradle.kts b/local_plugins/open_ai_service/android/build.gradle.kts index 57889671a..1f079fa6c 100644 --- a/local_plugins/open_ai_service/android/build.gradle.kts +++ b/local_plugins/open_ai_service/android/build.gradle.kts @@ -50,6 +50,11 @@ dependencies { implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4") implementation("com.fasterxml.jackson.core:jackson-databind:2.14.2") implementation("com.squareup.okhttp3:okhttp:4.9.3") + implementation("com.squareup.okhttp3:logging-interceptor:4.9.3") implementation("com.anthropic:anthropic-java:0.8.0") implementation("io.modelcontextprotocol:kotlin-sdk:0.4.0") + implementation("io.ktor:ktor-client-core:3.1.2") + implementation("io.ktor:ktor-client-cio:3.1.2") + implementation("io.ktor:ktor-client-okhttp:3.1.2") + implementation("io.ktor:ktor-client-logging:3.1.2") } \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt index fef91ab05..a4625376c 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt @@ -9,14 +9,30 @@ import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper import io.ktor.client.HttpClient +import io.ktor.client.plugins.sse.* import io.modelcontextprotocol.kotlin.sdk.Implementation +import io.modelcontextprotocol.kotlin.sdk.ListToolsResult import io.modelcontextprotocol.kotlin.sdk.TextContent import io.modelcontextprotocol.kotlin.sdk.client.Client -import io.modelcontextprotocol.kotlin.sdk.client.WebSocketClientTransport +import io.modelcontextprotocol.kotlin.sdk.client.SseClientTransport import kotlinx.coroutines.runBlocking import kotlinx.serialization.json.JsonObject import org.json.JSONObject import kotlin.jvm.optionals.getOrNull +import java.io.File +import io.ktor.client.engine.okhttp.* +import okhttp3.OkHttpClient +import java.security.cert.X509Certificate +import javax.net.ssl.SSLContext +import javax.net.ssl.TrustManager +import javax.net.ssl.X509TrustManager +import java.security.SecureRandom +import java.util.concurrent.TimeUnit +import okhttp3.logging.HttpLoggingInterceptor +import okhttp3.Interceptor +import io.ktor.client.request.* +import io.ktor.http.* +import com.yunqiinnovation.open_ai_service.mcp.CustomSseClientTransport /** * 工具类型枚举 @@ -39,55 +55,45 @@ interface FunctionHandler { } /** - * MCP客户端，用于连接Model Context Protocol服务器和管理本地函数 + * 单个MCP子客户端 + * 负责连接到一个具体的MCP服务器并管理其工具 */ -class MCPClient(private val context: Context? = null) : AutoCloseable { - private val TAG = "MCPClient" - +class MCPSubClient( + private val serverId: String, + serverUrl: String, + private val customClient: HttpClient? = null +) : AutoCloseable { + private val TAG = "MCPSubClient[$serverId]" + + // 保存服务器URL + private val serverUrl: String = serverUrl + + init { + Log.d(TAG, "MCPSubClient初始化，URL: $serverUrl") + } + // 初始化MCP客户端 private val mcp: Client = Client( clientInfo = Implementation( - name = "deep-voice-mcp-client", + name = "deep-voice-mcp-client-$serverId", version = "1.0.0" ) ) - + // 服务器提供的工具列表 private var tools: List = emptyList() // 工具Map列表，用于OpenAI请求 private val toolMaps = mutableListOf>() - // 本地函数Map，函数名 -> 处理器 - private val localFunctions = mutableMapOf() - - // 本地函数定义Map，函数名 -> 定义 - private val localFunctionDefs = mutableMapOf() - // 是否已连接 private var isConnected = false - // 系统功能处理器 - private val systemFunctionHandler = SystemFunctionHandler(context!!) - - init { - // 如果提供了Context，自动注册语音功能处理器 - context?.let { initializeVoiceFunctions(it) } - } + // SSE传输层，使用自定义的传输层 + private var sseTransport: CustomSseClientTransport? = null - /** - * 初始化语音功能处理器 - */ - private fun initializeVoiceFunctions(context: Context) { - try { - // 使用SystemFunctionHandler注册所有功能 - systemFunctionHandler.registerAllFunctions(this) - - Log.d(TAG, "语音功能初始化完成") - } catch (e: Exception) { - Log.e(TAG, "语音功能初始化失败: ${e.message}", e) - } - } + // 工具名称到工具的映射，用于快速查找 + private val toolMapping = mutableMapOf() /** * 将JsonObject转换为JsonValue @@ -97,20 +103,35 @@ class MCPClient(private val context: Context? = null) : AutoCloseable { val node = mapper.readTree(this.toString()) return JsonValue.fromJsonNode(node) } - + /** - * 连接到MCP服务器(通过WebSocket) + * 连接到MCP服务器(通过SSE) + * @return 是否连接成功 */ - suspend fun connectToWebSocket(serverUrl: String): Boolean { + suspend fun connect(): Boolean { try { - // 创建WebSocket传输层 - val transport = WebSocketClientTransport( - client = HttpClient(), - urlString = serverUrl + Log.d(TAG, "开始连接到服务器: $serverUrl") + + // 使用提供的客户端或创建带SSL配置的客户端 + val client = customClient ?: createSslTrustAllClient() + + // 使用自定义传输层，直接传递完整URL + sseTransport = CustomSseClientTransport( + client = client, + urlString = serverUrl, + requestBuilder = { + // 添加基本请求头 + headers { + append(HttpHeaders.Accept, "text/event-stream") + append(HttpHeaders.CacheControl, "no-cache") + } + } ) + Log.d(TAG, "创建自定义SSE传输层成功") + // 连接MCP客户端到服务器 - mcp.connect(transport) + mcp.connect(sseTransport!!) // 获取服务器提供的工具列表 val toolsResult = mcp.listTools() @@ -120,10 +141,12 @@ class MCPClient(private val context: Context? = null) : AutoCloseable { processToolsList(toolsResult) isConnected = true - Log.d(TAG, "已连接到WebSocket服务器，可用工具: ${tools.map { it.tool().get().name() }.joinToString(", ")}") + Log.d(TAG, "成功连接到MCP服务器，可用工具: ${tools.map { it.tool().get().name() }.joinToString(", ")}") return true } catch (e: Exception) { - Log.e(TAG, "连接WebSocket服务器失败: ${e.message}", e) + Log.e(TAG, "连接服务器失败: ${e.message}", e) + // 记录详细的错误堆栈 + Log.e(TAG, "详细错误: ${e.stackTraceToString()}") return false } } @@ -131,7 +154,7 @@ class MCPClient(private val context: Context? = null) : AutoCloseable { /** * 处理工具列表 */ - private fun processToolsList(toolsResult: io.modelcontextprotocol.kotlin.sdk.ListToolsResult?) { + private fun processToolsList(toolsResult: ListToolsResult?) { tools = toolsResult?.tools?.map { tool -> // 确保输入模式有效 val inputType = tool.inputSchema?.type ?: "object" @@ -153,10 +176,415 @@ class MCPClient(private val context: Context? = null) : AutoCloseable { ) } ?: emptyList() - // 更新用于OpenAI的工具映射 + // 更新工具映射 + tools.forEach { toolUnion -> + val tool = toolUnion.tool().get() + toolMapping[tool.name()] = toolUnion + } + + // 更新工具Map列表 updateToolMaps() } - + + /** + * 更新工具Map列表，用于OpenAI请求 + */ + private fun updateToolMaps() { + toolMaps.clear() + + // 添加MCP工具 + tools.forEach { toolUnion -> + try { + val toolOptional = toolUnion.tool() + if (!toolOptional.isPresent) { + return@forEach + } + + val tool = toolOptional.get() + + // 创建基本参数结构 + val parametersMap = mutableMapOf() + parametersMap["type"] = "object" // 默认值 + parametersMap["properties"] = mutableMapOf() + parametersMap["required"] = emptyList() + + // 从工具的输入模式中提取信息 + try { + // 获取输入模式并转换为字符串 + val schema = tool.inputSchema() + val schemaStr = schema.toString() + Log.d(TAG, "解析工具Schema: $schemaStr") + + // 如果toString()提供了足够的信息，就使用正则表达式解析 + // 这是最可靠的方法，因为它不依赖于API的细节变化 + + // 解析类型 + val typePattern = "type=([^,}]+)".toRegex() + val typeMatch = typePattern.find(schemaStr) + if (typeMatch != null) { + parametersMap["type"] = typeMatch.groupValues[1].trim() + } + + // 尝试从字符串中解析完整的JSON结构 + try { + // 直接从Schema字符串中提取属性结构，手动构建properties映射 + val propertiesMap = mutableMapOf() + + // 匹配属性定义: 形如 fieldName={...} + val propertiesPattern = "properties=\\{([^}]+)\\}".toRegex() + val propertiesMatch = propertiesPattern.find(schemaStr) + + if (propertiesMatch != null) { + // 匹配属性定义: 形如 fieldName={...} + val fieldPattern = "([\\w]+)=\\{([^{}]+(?:\\{[^{}]*\\}[^{}]*)*)\\}".toRegex() + val fieldMatches = fieldPattern.findAll(propertiesMatch.groupValues[1]) + + fieldMatches.forEach { fieldMatch -> + val fieldName = fieldMatch.groupValues[1] + val fieldContent = fieldMatch.groupValues[2] + + // 为属性创建一个Map + val fieldMap = mutableMapOf() + + // 解析字段内容中的键值对 + val keyValuePattern = "(\\w+)=([^,}]+)".toRegex() + val keyValueMatches = keyValuePattern.findAll(fieldContent) + + keyValueMatches.forEach { kvMatch -> + val key = kvMatch.groupValues[1] + val value = kvMatch.groupValues[2].trim() + + // 根据值的类型进行适当处理 + val processedValue: Any = when { + value == "true" -> true + value == "false" -> false + value.matches("\\d+".toRegex()) -> value.toIntOrNull() ?: value + value.matches("\\d+\\.\\d+".toRegex()) -> value.toDoubleOrNull() ?: value + else -> value // 保持为字符串 + } + + fieldMap[key] = processedValue + } + + propertiesMap[fieldName] = fieldMap + } + parametersMap["properties"] = propertiesMap + Log.d(TAG, "成功解析属性: $propertiesMap") + } else { + parametersMap["properties"] = emptyMap() + } + + // 提取required部分 + val requiredPattern = "required=\\[([^\\]]+)\\]".toRegex() + val requiredMatch = requiredPattern.find(schemaStr) + + if (requiredMatch != null) { + val requiredStr = requiredMatch.groupValues[1] + val requiredList = requiredStr.split(",").map { it.trim() } + parametersMap["required"] = requiredList + } else { + parametersMap["required"] = emptyList() + } + } catch (e: Exception) { + Log.e(TAG, "从字符串解析schema细节失败: ${e.message}") + // 使用默认值 + parametersMap["properties"] = emptyMap() + parametersMap["required"] = emptyList() + } + } catch (e: Exception) { + Log.e(TAG, "解析工具输入模式失败: ${e.message}", e) + // 使用默认值继续 + } + + // 创建工具Map + toolMaps.add(mapOf( + "type" to "function", + "function" to mapOf( + "name" to tool.name(), + "description" to (tool.description() ?: ""), + "parameters" to parametersMap + ) + )) + + Log.d(TAG, "工具 ${tool.name()} 成功添加到工具映射") + + } catch (e: Exception) { + Log.e(TAG, "转换工具到Map失败: ${e.message}") + } + } + + Log.d(TAG, "更新工具映射完成，共 ${toolMaps.size} 个工具") + } + + /** + * 将JsonNode转换为Map + */ + private fun convertJsonNodeToMap(node: JsonNode): Any { + return when { + node.isTextual() -> node.asText() + node.isNumber() -> node.asDouble() + node.isBoolean() -> node.asBoolean() + node.isObject() -> { + val result = mutableMapOf() + node.fieldNames().forEach { fieldName -> + result[fieldName] = convertJsonNodeToMap(node.get(fieldName)) + } + result + } + node.isArray() -> { + val result = mutableListOf() + for (i in 0 until node.size()) { + result.add(convertJsonNodeToMap(node.get(i))) + } + result + } + node.isNull() -> "null" + else -> node.toString() + } + } + + /** + * 获取工具Map列表，用于OpenAI请求 + */ + fun getToolMaps(): List> { + return toolMaps + } + + /** + * 获取可用的工具列表 + */ + fun getTools(): List { + return tools + } + + /** + * 检查是否包含指定工具 + */ + fun containsTool(name: String): Boolean { + return toolMapping.containsKey(name) + } + + /** + * 调用工具 + */ + suspend fun callTool(name: String, arguments: Map): String? { + if (!isConnected) { + Log.e(TAG, "未连接到MCP服务器", null) + return "未连接到MCP服务器" + } + + if (!containsTool(name)) { + Log.w(TAG, "此客户端不包含工具: $name") + return null + } + + return try { + val result = mcp.callTool( + name = name, + arguments = arguments + ) + + // 处理工具调用结果 + result?.content?.joinToString("\n") { content -> + when (content) { + is TextContent -> content.text ?: "" + else -> "" + } + } + } catch (e: Exception) { + Log.e(TAG, "调用MCP工具失败: ${e.message}", e) + "调用MCP工具失败: ${e.message}" + } + } + + /** + * 检查是否已连接 + */ + fun isConnected(): Boolean { + return isConnected + } + + /** + * 关闭连接 + */ + override fun close() { + runBlocking { + try { + mcp.close() + sseTransport?.close() + isConnected = false + Log.d(TAG, "已关闭MCP服务器连接") + } catch (e: Exception) { + Log.e(TAG, "关闭MCP服务器连接失败: ${e.message}", e) + } + } + } +} + +/** + * MCP客户端，用于连接Model Context Protocol服务器和管理本地函数 + */ +class MCPClient(private val context: Context? = null) : AutoCloseable { + private val TAG = "MCPClient" + + // 本地函数Map，函数名 -> 处理器 + private val localFunctions = mutableMapOf() + + // 本地函数定义Map，函数名 -> 定义 + private val localFunctionDefs = mutableMapOf() + + // 子客户端列表，每个连接一个MCP服务器 + private val subClients = mutableMapOf() + + // 是否已连接 + private var isConnected = false + + // 系统功能处理器 + private val systemFunctionHandler = SystemFunctionHandler(context!!) + + // 共享的HttpClient，用于所有子客户端 + private val sharedHttpClient by lazy { createSslTrustAllClient() } + + init { + // 如果提供了Context，自动注册语音功能处理器 + context?.let { initializeVoiceFunctions(it) } + } + + /** + * 初始化语音功能处理器 + */ + private fun initializeVoiceFunctions(context: Context) { + try { + // 使用SystemFunctionHandler注册所有功能 + systemFunctionHandler.registerAllFunctions(this) + + Log.d(TAG, "语音功能初始化完成") + } catch (e: Exception) { + Log.e(TAG, "语音功能初始化失败: ${e.message}", e) + } + } + + /** + * 连接到MCP服务器(通过SSE) + * 直接接收完整的JSON配置字符串 + * + * @param mcpConfigJson 包含mcpServers字段的JSON配置字符串 + * @return 是否连接成功 + */ + suspend fun connectToSSE(mcpConfigJson: String): Boolean { + // 清除现有连接 + closeAllConnections() + + val serverMap = mutableMapOf() + var configLoaded = false + + // 尝试解析传入的JSON配置 + try { + if (mcpConfigJson.isNotEmpty()) { + val mapper = ObjectMapper() + try { + val config = mapper.readValue(mcpConfigJson, Map::class.java) + + // 处理mcpServers格式 + @Suppress("UNCHECKED_CAST") + val mcpServers = config["mcpServers"] as? Map> + if (mcpServers != null && mcpServers.isNotEmpty()) { + // 从服务器配置中提取URL + mcpServers.forEach { (serverId, serverConfig) -> + val url = serverConfig["url"] + if (!url.isNullOrEmpty()) { + serverMap[serverId] = url + Log.d(TAG, "从配置中添加服务器 [$serverId]: $url") + } + } + configLoaded = serverMap.isNotEmpty() + Log.d(TAG, "从传入的JSON配置加载了 ${serverMap.size} 个服务器") + } + } catch (e: Exception) { + Log.e(TAG, "解析传入的JSON配置失败: ${e.message}") + } + } + } catch (e: Exception) { + Log.e(TAG, "处理传入的配置失败: ${e.message}") + } + + + // 创建并连接所有子客户端 + val connectionResults = mutableMapOf() + + for ((serverId, serverUrl) in serverMap) { + try { + Log.d(TAG, "为服务器 [$serverId],[$serverUrl] 创建子客户端") + val subClient = MCPSubClient(serverId, serverUrl, sharedHttpClient) + val connected = subClient.connect() + + if (connected) { + subClients[serverId] = subClient + connectionResults[serverId] = true + Log.d(TAG, "子客户端 [$serverId] 连接成功") + } else { + subClient.close() + connectionResults[serverId] = false + Log.w(TAG, "子客户端 [$serverId] 连接失败") + } + } catch (e: Exception) { + Log.e(TAG, "创建子客户端 [$serverId] 失败: ${e.message}", e) + connectionResults[serverId] = false + } + } + + // 检查是否有任何客户端连接成功 + val successCount = connectionResults.count { it.value } + isConnected = successCount > 0 + + if (isConnected) { + Log.d(TAG, "已成功连接 $successCount/${serverMap.size} 个MCP服务器") + + // 记录所有可用工具 + val allToolNames = getAllToolNames() + Log.d(TAG, "所有可用工具: ${allToolNames.joinToString(", ")}") + + return true + } else { + Log.e(TAG, "所有MCP服务器连接均失败") + return false + } + } + + /** + * 关闭所有连接 + */ + private fun closeAllConnections() { + subClients.forEach { (serverId, client) -> + try { + client.close() + Log.d(TAG, "已关闭子客户端 [$serverId]") + } catch (e: Exception) { + Log.e(TAG, "关闭子客户端 [$serverId] 失败: ${e.message}") + } + } + subClients.clear() + } + + /** + * 获取所有工具名称 + */ + private fun getAllToolNames(): List { + val toolNames = mutableSetOf() + + // 添加本地函数 + toolNames.addAll(localFunctions.keys) + + // 添加所有子客户端的工具 + subClients.forEach { (_, client) -> + client.getTools().forEach { toolUnion -> + val tool = toolUnion.tool().get() + toolNames.add(tool.name()) + } + } + + return toolNames.toList() + } /** * 注册本地函数 @@ -211,9 +639,6 @@ class MCPClient(private val context: Context? = null) : AutoCloseable { // 保存函数定义 localFunctionDefs[name] = ObjectMapper().writeValueAsString(functionDef) - // 更新工具映射 - updateToolMaps() - Log.d(TAG, "注册本地函数成功: $name") return true } catch (e: Exception) { @@ -231,128 +656,39 @@ class MCPClient(private val context: Context? = null) : AutoCloseable { val removed = localFunctions.remove(name) != null if (removed) { localFunctionDefs.remove(name) - updateToolMaps() Log.d(TAG, "取消注册本地函数成功: $name") } return removed } /** - * 更新工具Map列表，用于OpenAI请求 + * 获取工具Map列表，用于OpenAI请求 */ - private fun updateToolMaps() { - toolMaps.clear() + fun getToolMaps(): List> { + val allToolMaps = mutableListOf>() // 首先添加本地函数 localFunctionDefs.forEach { (name, functionDefJson) -> try { val mapper = ObjectMapper() val functionDef = mapper.readValue(functionDefJson, object : TypeReference>() {}) - val toolMap = mapOf( + allToolMaps.add(mapOf( "type" to "function", "function" to functionDef - ) - toolMaps.add(toolMap) + )) } catch (e: Exception) { - Log.e(TAG, "解析本地函数定义失败: $name, ${e.message}", e) + Log.e(TAG, "解析本地函数定义失败: $name, ${e.message}") } } - // 然后添加MCP工具 - tools.forEach { toolUnion -> - try { - val tool = toolUnion.tool().get() - val parametersMap = mutableMapOf() - - // 提取工具类型及参数 - try { - val inputSchema = tool.inputSchema() - // 解析JsonValue - val mapper = ObjectMapper() - val inputSchemaStr = inputSchema.toString() - val inputSchemaJson = mapper.readTree(inputSchemaStr) - val typeValue = inputSchemaJson.path("type") - parametersMap["type"] = if (typeValue.isTextual) typeValue.asText() else "object" - - // 提取properties字段 - val propertiesMap = mutableMapOf() - val propertiesNode = inputSchemaJson.path("properties") - if (propertiesNode.isObject) { - propertiesNode.fieldNames().forEach { fieldName -> - val fieldNode = propertiesNode.get(fieldName) - propertiesMap[fieldName] = convertJsonNodeToMap(fieldNode) - } - } - parametersMap["properties"] = propertiesMap - - // 提取required字段 - val requiredList = mutableListOf() - val requiredNode = inputSchemaJson.path("required") - if (requiredNode.isArray) { - for (i in 0 until requiredNode.size()) { - if (requiredNode.get(i).isTextual) { - requiredList.add(requiredNode.get(i).asText()) - } - } - } - parametersMap["required"] = requiredList - } catch (e: Exception) { - Log.e(TAG, "解析工具参数失败: ${e.message}", e) - // 设置默认参数 - parametersMap["type"] = "object" - parametersMap["properties"] = emptyMap() - parametersMap["required"] = emptyList() - } - - // 创建工具Map - val toolMap = mapOf( - "type" to "function", - "function" to mapOf( - "name" to tool.name(), - "description" to (tool.description() ?: ""), - "parameters" to parametersMap - ) - ) - - toolMaps.add(toolMap) - } catch (e: Exception) { - Log.e(TAG, "转换工具到Map失败: ${e.message}", e) - } - } - } - - /** - * 将JsonNode转换为Map - */ - private fun convertJsonNodeToMap(node: JsonNode): Any { - return when { - node.isTextual() -> node.asText() - node.isNumber() -> node.asDouble() - node.isBoolean() -> node.asBoolean() - node.isObject() -> { - val result = mutableMapOf() - node.fieldNames().forEach { fieldName -> - result[fieldName] = convertJsonNodeToMap(node.get(fieldName)) - } - result - } - node.isArray() -> { - val result = mutableListOf() - for (i in 0 until node.size()) { - result.add(convertJsonNodeToMap(node.get(i))) - } - result - } - node.isNull() -> "null" - else -> node.toString() + // 然后添加所有子客户端的工具 + subClients.forEach { (serverId, client) -> + val clientTools = client.getToolMaps() + allToolMaps.addAll(clientTools) + Log.d(TAG, "添加子客户端 [$serverId] 的 ${clientTools.size} 个工具") } - } - - /** - * 获取工具Map列表，用于OpenAI请求 - */ - fun getToolMaps(): List> { - return toolMaps + + return allToolMaps } /** @@ -365,7 +701,7 @@ class MCPClient(private val context: Context? = null) : AutoCloseable { return ToolType.LOCAL_FUNCTION } - if (tools.any { it.tool().get().name() == name }) { + if (subClients.any { it.value.containsTool(name) }) { return ToolType.MCP_TOOL } @@ -422,40 +758,40 @@ class MCPClient(private val context: Context? = null) : AutoCloseable { return "未连接到MCP服务器" } - return try { - val result = mcp.callTool( - name = name, - arguments = arguments - ) - - // 处理工具调用结果 - result?.content?.joinToString("\n") { content -> - when (content) { - is TextContent -> content.text ?: "" - else -> "" - } - } - } catch (e: Exception) { - Log.e(TAG, "调用MCP工具失败: ${e.message}", e) - "调用MCP工具失败: ${e.message}" + // 查找包含此工具的子客户端 + val clientEntry = subClients.entries.find { it.value.containsTool(name) } + + if (clientEntry != null) { + val (serverId, client) = clientEntry + Log.d(TAG, "通过子客户端 [$serverId] 调用工具: $name") + return client.callTool(name, arguments) } + + // 没有找到包含此工具的客户端 + Log.e(TAG, "没有子客户端包含工具: $name") + return "没有可用的服务器提供工具: $name" } /** * 获取可用的工具列表 */ fun getTools(): List { - return tools + val allTools = mutableListOf() + + // 收集所有子客户端的工具 + subClients.forEach { (_, client) -> + allTools.addAll(client.getTools()) + } + + return allTools } /** * 检查工具是否存在 */ fun hasToolWithName(name: String): Boolean { - return localFunctions.containsKey(name) || toolMaps.any { - val functionMap = it["function"] as? Map - functionMap?.get("name") == name - } + return localFunctions.containsKey(name) || + subClients.any { it.value.containsTool(name) } } /** @@ -483,14 +819,71 @@ class MCPClient(private val context: Context? = null) : AutoCloseable { override fun close() { runBlocking { try { - mcp.close() + closeAllConnections() isConnected = false - tools = emptyList() - toolMaps.clear() - Log.d(TAG, "已关闭MCP服务器连接", null) + Log.d(TAG, "已关闭所有MCP服务器连接") } catch (e: Exception) { Log.e(TAG, "关闭MCP服务器连接失败: ${e.message}", e) } } } +} + +/** + * 创建一个信任所有SSL证书的HttpClient + */ +private fun createSslTrustAllClient(): HttpClient { + // 创建信任所有证书的TrustManager + val trustAllCerts = arrayOf( + object : X509TrustManager { + override fun checkClientTrusted(chain: Array?, authType: String?) {} + override fun checkServerTrusted(chain: Array?, authType: String?) {} + override fun getAcceptedIssuers(): Array = arrayOf() + } + ) + + // 创建SSL上下文并初始化它 + val sslContext = SSLContext.getInstance("TLS") + sslContext.init(null, trustAllCerts, SecureRandom()) + + // 创建HTTP日志拦截器 + val loggingInterceptor = HttpLoggingInterceptor { message -> + Log.d("OkHttp", message) + }.apply { + level = HttpLoggingInterceptor.Level.HEADERS + } + + // 创建自定义拦截器显示实际请求URL + val urlLoggingInterceptor = Interceptor { chain -> + val request = chain.request() + Log.d("OkHttp-URL", "实际请求URL: ${request.url}") + Log.d("OkHttp-Headers", "请求头: ${request.headers}") + + val response = chain.proceed(request) + + Log.d("OkHttp-Response", "状态码: ${response.code} - ${response.message}") + Log.d("OkHttp-Response", "响应头: ${response.headers}") + + response + } + + // 创建OkHttpClient并配置信任所有证书 + val okHttpClient = OkHttpClient.Builder() + .sslSocketFactory(sslContext.socketFactory, trustAllCerts[0] as X509TrustManager) + .hostnameVerifier { _, _ -> true } + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .addInterceptor(loggingInterceptor) // 添加HTTP日志拦截器 + .addInterceptor(urlLoggingInterceptor) // 添加URL日志拦截器 + .build() + + // 创建使用OkHttp引擎的HttpClient + return HttpClient(OkHttp) { + engine { + preconfigured = okHttpClient + } + + // 安装SSE插件 + install(SSE) + } } \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt index 9ca318840..b91ec832a 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt @@ -94,7 +94,7 @@ class OpenAIService(private val context: Context? = null) { /** * 初始化OpenAI服务 */ - fun initialize(apiKey: String, baseUrl: String, model: String): Boolean { + fun initialize(apiKey: String, baseUrl: String, model: String, mcpServer: String): Boolean { this.apiKey = apiKey if (baseUrl.isNotEmpty()) { this.baseUrl = baseUrl @@ -107,7 +107,7 @@ class OpenAIService(private val context: Context? = null) { if (mcpClient == null) { mcpClient = MCPClient(context) } - + initializeMcpClient(mcpServer) isInitialized = apiKey.isNotEmpty() return isInitialized } @@ -129,9 +129,9 @@ class OpenAIService(private val context: Context? = null) { } /** - * 初始化MCP客户端（WebSocket方式） + * 初始化MCP客户端 */ - fun initializeMcpClient(serverUrl: String): Boolean { + fun initializeMcpClient(mcpServer: String): Boolean { if (mcpClient != null) { mcpClient?.close() } @@ -144,7 +144,7 @@ class OpenAIService(private val context: Context? = null) { var result = false scope.launch { try { - result = mcpClient?.connectToWebSocket(serverUrl) ?: false + result = mcpClient?.connectToSSE(mcpServer) ?: false isMcpInitialized = result Log.d(TAG, "MCP客户端初始化${if (result) "成功" else "失败"}") } catch (e: Exception) { diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt index 091af559c..03b4d5d9e 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIServicePlugin.kt @@ -73,8 +73,9 @@ class OpenAIServicePlugin : FlutterPlugin, MethodCallHandler, StreamHandler { val apiKey = call.argument("apiKey") ?: "" val baseUrl = call.argument("baseUrl") ?: "" val model = call.argument("model") ?: "" + val mcpServer = call.argument("mcpServer") ?: "" - val initialized = openAIService.initialize(apiKey, baseUrl, model) + val initialized = openAIService.initialize(apiKey, baseUrl, model, mcpServer) result.success(initialized) } diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/mcp/CustomSseClientTransport.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/mcp/CustomSseClientTransport.kt new file mode 100644 index 000000000..4b98b1c0f --- /dev/null +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/mcp/CustomSseClientTransport.kt @@ -0,0 +1,357 @@ +package com.yunqiinnovation.open_ai_service.mcp + +import android.util.Log +import io.ktor.client.* +import io.ktor.client.plugins.sse.* +import io.ktor.client.request.* +import io.ktor.client.statement.* +import io.ktor.http.* +import io.modelcontextprotocol.kotlin.sdk.JSONRPCMessage +import io.modelcontextprotocol.kotlin.sdk.shared.AbstractTransport +import kotlinx.coroutines.* +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import kotlinx.serialization.decodeFromString +import kotlin.properties.Delegates +import kotlin.time.Duration +import java.util.concurrent.atomic.AtomicBoolean +import org.json.JSONObject + +/** + * 自定义SSE客户端传输层，修复原始SseClientTransport中的URL拼接问题 + * 解决URL查询参数与路径拼接错误的问题，确保消息端点URL格式正确 + */ +class CustomSseClientTransport( + private val client: HttpClient, + private val urlString: String?, + private val reconnectionTime: Duration? = null, + private val requestBuilder: HttpRequestBuilder.() -> Unit = {}, +) : AbstractTransport() { + private val TAG = "CustomSseClientTransport" + + private val scope by lazy { + CoroutineScope(session.coroutineContext + SupervisorJob()) + } + + // 使用Java标准库的AtomicBoolean替代kotlinx.atomicfu + private val initialized = AtomicBoolean(false) + private var session: ClientSSESession by Delegates.notNull() + private val endpoint = CompletableDeferred() + + private var job: Job? = null + + // 创建JSON解析器，增强灵活性设置 + private val json = Json { + ignoreUnknownKeys = true // 忽略未知字段 + isLenient = true // 宽松解析模式 + coerceInputValues = true // 尝试强制转换类型 + encodeDefaults = true // 编码默认值 + explicitNulls = false // 不要求显式null值 + } + + // 保存基础URL（不包含查询参数）和查询参数 + private var baseUrlWithoutParams: String? = null + private var queryParams: Map = emptyMap() + private var hostPart: String = "" // 添加类级别变量 + private var pathPart: String = "" // 添加类级别变量 + + /** + * 解析URL，分离基础URL、路径和查询参数 + * 返回三元组: (主机部分URL, 路径部分, 查询参数Map) + */ + private fun parseUrl(url: String): Triple> { + return try { + val params = mutableMapOf() + + // 确保URL有协议部分 + var processedUrl = url.trim() + if (!processedUrl.startsWith("http://") && !processedUrl.startsWith("https://")) { + processedUrl = "https://$processedUrl" + Log.d(TAG, "添加默认协议: $processedUrl") + } + + val urlObj = java.net.URL(processedUrl) + + // 解析查询参数 + if (urlObj.query != null) { + urlObj.query.split("&").forEach { param -> + val parts = param.split("=", limit = 2) + if (parts.size == 2) { + params[parts[0]] = parts[1] + } + } + } + + // 构建主机部分URL（协议+主机+端口） + val port = if (urlObj.port == -1) "" else ":${urlObj.port}" + val hostUrl = "${urlObj.protocol}://${urlObj.host}$port" + + // 路径部分 + val path = urlObj.path + + Triple(hostUrl, path, params) + } catch (e: Exception) { + Log.e(TAG, "解析URL失败: $url, ${e.message}") + Triple(url, "", emptyMap()) + } + } + + /** + * 收集SSE事件 + */ + private suspend fun collectEvents() { + job = scope.launch(CoroutineName("CustomSseMcpClientTransport.collect#${hashCode()}")) { + session.incoming.collect { event -> + when (event.event) { + "error" -> { + val e = IllegalStateException("SSE error: ${event.data}") + Log.e(TAG, "SSE错误: ${event.data}") + _onError(e) + throw e + } + + "open" -> { + Log.d(TAG, "SSE连接已打开") + // 连接已打开，等待endpoint事件 + } + + "endpoint" -> { + try { + val eventData = event.data ?: "" + Log.d(TAG, "收到endpoint事件: $eventData") + + // 使用主机部分构建endpoint + val fullEndpoint = if (eventData.startsWith("/")) { + "$hostPart$eventData" + } else { + "$hostPart/$eventData" + } + + Log.d(TAG, "构建的endpoint路径（不含参数）: $fullEndpoint") + + // 添加查询参数到endpoint + val endpointWithParams = if (queryParams.isNotEmpty()) { + // 检查endpoint是否已有查询参数 + if (fullEndpoint.contains("?")) { + // 已有查询参数，添加&并附加其他参数 + val queryString = queryParams.entries.joinToString("&") { "${it.key}=${it.value}" } + "$fullEndpoint&$queryString" + } else { + // 没有查询参数，添加?并附加参数 + val queryString = queryParams.entries.joinToString("&") { "${it.key}=${it.value}" } + "$fullEndpoint?$queryString" + } + } else { + fullEndpoint + } + + Log.d(TAG, "最终消息端点: $endpointWithParams") + endpoint.complete(endpointWithParams) + } catch (e: Exception) { + Log.e(TAG, "处理endpoint事件失败: ${e.message}", e) + _onError(e) + close() + error(e) + } + } + + else -> { + try { + // 解析JSON-RPC消息 + val data = event.data + if (data != null) { + Log.d(TAG, "收到事件数据: $data") + try { + // 尝试安全地解析JSON消息 + safeParseMessage(data) + } catch (e: Exception) { + Log.e(TAG, "解析JSON-RPC消息失败: ${e.message}", e) + // 错误已记录，但不中断连接，只发送错误通知 + _onError(e) + } + } + } catch (e: Exception) { + Log.e(TAG, "处理事件失败: ${e.message}", e) + _onError(e) + } + } + } + } + } + } + + /** + * 安全解析JSON-RPC消息 + */ + private suspend fun safeParseMessage(data: String) { + try { + // 先尝试使用标准解析 + val message = json.decodeFromString(data) + _onMessage(message) + } catch (e: Exception) { + // 如果标准解析失败，记录错误并尝试使用备用解析方式 + Log.w(TAG, "标准解析失败，尝试备用解析: ${e.message}") + + try { + // 尝试修复nextCursor缺失问题 + if (e.message?.contains("nextCursor") == true) { + // 尝试手动添加缺失的nextCursor字段 + val jsonObj = JSONObject(data) + + // 只有在解析ListToolsResult时处理 + if (data.contains("\"tools\"")) { + Log.d(TAG, "尝试修复ListToolsResult缺少nextCursor字段的问题") + + // 手动解析result部分并添加nextCursor + val resultJson = try { + if (jsonObj.has("result")) { + val resultObj = jsonObj.getJSONObject("result") + if (!resultObj.has("nextCursor")) { + resultObj.put("nextCursor", "") + jsonObj.put("result", resultObj) + } + jsonObj.toString() + } else { + // 如果没有result字段，可能是其他类型的消息 + data + } + } catch (ex: Exception) { + Log.e(TAG, "手动修复JSON失败: ${ex.message}") + data + } + + // 重新尝试解析修复后的JSON + val fixedMessage = json.decodeFromString(resultJson) + _onMessage(fixedMessage) + return + } + } + + // 通用错误处理 + Log.e(TAG, "无法解析消息，跳过: $data") + } catch (ex: Exception) { + Log.e(TAG, "备用解析也失败: ${ex.message}", ex) + // 不抛出异常，只记录错误 + _onError(e) + } + } + } + + /** + * 启动传输层 + */ + override suspend fun start() { + if (!initialized.compareAndSet(false, true)) { + Log.e(TAG, "传输层已经启动，不能重复启动") + error("CustomSseClientTransport already started!") + } + + // 解析URL和参数 + if (urlString != null) { + // 解析URL，提取主机部分、路径部分和查询参数 + val urlInfo = parseUrl(urlString) + hostPart = urlInfo.first + pathPart = urlInfo.second + queryParams = urlInfo.third + + // 存储不带查询参数的基础URL（主机+路径） + baseUrlWithoutParams = hostPart + pathPart + + Log.d(TAG, "原始URL: $urlString") + Log.d(TAG, "主机部分: $hostPart") + Log.d(TAG, "路径部分: $pathPart") + Log.d(TAG, "查询参数: $queryParams") + Log.d(TAG, "完整基础URL: $baseUrlWithoutParams") + } + + // 创建SSE会话 - 直接使用原始URL，不添加/sse后缀 + session = urlString?.let { + // 完整的SSE连接URL（主机部分+原始路径+查询参数） + val sseConnectUrl = if (queryParams.isNotEmpty()) { + // 如果路径已包含查询参数，就不再添加 + if (pathPart.contains("?")) { + "$hostPart$pathPart" + } else { + val queryString = queryParams.entries.joinToString("&") { "${it.key}=${it.value}" } + "$hostPart$pathPart?$queryString" + } + } else { + "$hostPart$pathPart" + } + + Log.d(TAG, "SSE连接URL: $sseConnectUrl") + + client.sseSession( + urlString = sseConnectUrl, + reconnectionTime = reconnectionTime, + block = requestBuilder, + ) + } ?: client.sseSession( + reconnectionTime = reconnectionTime, + block = requestBuilder, + ) + + // 收集SSE事件 + collectEvents() + + // 等待endpoint就绪 + endpoint.await() + Log.d(TAG, "传输层启动完成，消息端点已就绪") + } + + /** + * 发送消息 + */ + @OptIn(ExperimentalCoroutinesApi::class) + override suspend fun send(message: JSONRPCMessage) { + if (!endpoint.isCompleted) { + Log.e(TAG, "发送失败: 未连接") + error("Not connected") + } + + try { + val messageEndpoint = endpoint.getCompleted() + Log.d(TAG, "发送消息到: $messageEndpoint") + + // 序列化消息 + val jsonString = json.encodeToString(message) + + val response = client.post(messageEndpoint) { + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody(jsonString) + } + + if (!response.status.isSuccess()) { + val text = response.bodyAsText() + Log.e(TAG, "发送消息失败: HTTP ${response.status}, $text") + error("Error POSTing to endpoint (HTTP ${response.status}): $text") + } + } catch (e: Exception) { + Log.e(TAG, "发送消息异常: ${e.message}", e) + _onError(e) + throw e + } + } + + /** + * 关闭传输层 + */ + override suspend fun close() { + if (!initialized.get()) { + Log.e(TAG, "关闭失败: 传输层未初始化") + error("CustomSseClientTransport is not initialized!") + } + + session.cancel() + _onClose() + job?.cancelAndJoin() + Log.d(TAG, "传输层已关闭") + } + + /** + * 检查传输层是否已初始化 + */ + fun isInitialized(): Boolean { + return initialized.get() + } +} \ No newline at end of file diff --git a/local_plugins/open_ai_service/lib/open_ai_service.dart b/local_plugins/open_ai_service/lib/open_ai_service.dart index 8666f0100..df0fa315d 100644 --- a/local_plugins/open_ai_service/lib/open_ai_service.dart +++ b/local_plugins/open_ai_service/lib/open_ai_service.dart @@ -94,10 +94,12 @@ class OpenAIService { /// [apiKey] OpenAI API密钥 /// [baseUrl] 可选，自定义API基础URL /// [model] 可选，自定义使用的模型 + /// [mcpServer] 可选，MCP服务器地址配置 Future initialize({ required String apiKey, String baseUrl = '', String model = '', + String mcpServer = '', }) async { try { final result = await _channel.invokeMethod( @@ -106,6 +108,7 @@ class OpenAIService { 'apiKey': apiKey, 'baseUrl': baseUrl, 'model': model, + 'mcpServer': mcpServer, }, ); diff --git a/pubspec.yaml b/pubspec.yaml index b08d2746e..a076d7687 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -91,3 +91,4 @@ flutter: - assets/images/explore2.jpg - assets/lottie/ - .env + - .mcp.json diff --git a/test.json b/test.json index 2411527e2..208c91d82 100644 --- a/test.json +++ b/test.json @@ -16,9 +16,19 @@ curl 'https://ark.cn-beijing.volces.com/api/v3/bots/chat/completions' \ } ] }' +"map-qq": { + "url": "https://mcp.map.qq.com/sse?key=E5ABZ-RIACZ-DX2XC-7DEU2-AXR2Q-V7BWQ" +}, - - +"amap-amap-sse": { + "url": "https://mcp.amap.com/sse?key=66b3f6199e3561f874a5a9cd9e9ed7f2" +}, + // "map-qq": { + // "url": "https://mcp.map.qq.com?key=E5ABZ-RIACZ-DX2XC-7DEU2-AXR2Q-V7BWQ" + // }, + "web-search": { + "url": "http://mcp.ideapsound.com:8000" + }, curl 'https://ark.cn-beijing.volces.com/api/v3/bots/chat/completions' \ -H 'Content-Type: application/json' \ From 7d88c2cabacf13ccfdfeb4909ee7c6971ed060ae Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Wed, 16 Apr 2025 14:58:59 +0800 Subject: [PATCH 14/14] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E9=9F=B3=E4=B9=90?= =?UTF-8?q?=E6=92=AD=E6=94=BE=E5=92=8Cfacntion=20call=20=E7=9A=84=E7=BB=93?= =?UTF-8?q?=E6=9E=9C=E8=A1=A5=E5=85=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- android/app/build.gradle.kts | 11 +- android/app/src/main/AndroidManifest.xml | 4 + .../deepsound/VoiceInteractionHandler.kt | 4 +- android/settings.gradle.kts | 7 +- lib/data/services/deapsound_ai_service.dart | 2 +- lib/data/services/network/nw_api.dart | 2 +- .../services/open_ai_service_adapter.dart | 147 +-- .../chat/controllers/chat_controller.dart | 44 +- .../controllers/profile_controller.dart | 35 +- .../open_ai_service/MCPClient.kt | 27 +- .../open_ai_service/OpenAIService.kt | 15 +- .../open_ai_service/OpenAIServicePlugin.kt | 27 +- .../mcp/CustomSseClientTransport.kt | 2 +- .../open_ai_service/lib/open_ai_service.dart | 130 +- .../.github/ISSUE_TEMPLATE/bug_report.md | 62 + .../.github/ISSUE_TEMPLATE/config.yml | 1 + .../.github/ISSUE_TEMPLATE/feature_request.md | 20 + .../publish_flutter_package/action.yaml | 41 + .../spotify_sdk/.github/dependabot.yml | 42 + .../.github/scripts/verify_pub_score.sh | 16 + .../.github/workflows/pull_request.yml | 38 + .../spotify_sdk/.github/workflows/release.yml | 34 + local_plugins/spotify_sdk/.gitignore | 50 + local_plugins/spotify_sdk/.metadata | 10 + local_plugins/spotify_sdk/CHANGELOG.md | 184 +++ local_plugins/spotify_sdk/CODE_OF_CONDUCT.md | 76 ++ local_plugins/spotify_sdk/CONTRIBUTING.md | 48 + local_plugins/spotify_sdk/LICENSE | 201 +++ local_plugins/spotify_sdk/README.md | 257 ++++ .../spotify_sdk/analysis_options.yaml | 12 + local_plugins/spotify_sdk/android/.gitignore | 8 + .../spotify_sdk/android/build.gradle | 74 ++ .../spotify_sdk/android/gradle.properties | 3 + .../gradle/wrapper/gradle-wrapper.properties | 7 + .../libs/spotify-app-remote-release-0.8.0.aar | Bin 0 -> 132749 bytes .../spotify_sdk/android/settings.gradle | 1 + .../android/src/main/AndroidManifest.xml | 10 + .../minimalme/spotify_sdk/BaseSpotifyApi.kt | 13 + .../spotify_sdk/SpotifyConnectApi.kt | 25 + .../minimalme/spotify_sdk/SpotifyImagesApi.kt | 35 + .../minimalme/spotify_sdk/SpotifyPlayerApi.kt | 210 +++ .../minimalme/spotify_sdk/SpotifySdkPlugin.kt | 381 ++++++ .../minimalme/spotify_sdk/SpotifyUserApi.kt | 59 + .../subscriptions/CapabilitiesChannel.kt | 23 + .../subscriptions/ConnectionStatusChannel.kt | 27 + .../subscriptions/PlayerContextChannel.kt | 22 + .../subscriptions/PlayerStateChannel.kt | 22 + .../subscriptions/UserStatusChannel.kt | 23 + .../spotify_sdk/bin/android_cleanup.dart | 39 + .../bin/android_module_creator.dart | 96 ++ .../spotify_sdk/bin/android_setup.dart | 91 ++ local_plugins/spotify_sdk/bin/github_api.dart | 100 ++ .../spotify_sdk/bin/precondition_checker.dart | 52 + local_plugins/spotify_sdk/ios/.gitignore | 37 + local_plugins/spotify_sdk/ios/Assets/.gitkeep | 0 .../ios/Classes/ConnectionStatusHandler.swift | 45 + .../ios/Classes/Extension/Dictionary.swift | 13 + .../ios/Classes/PlayerDelegate.swift | 14 + .../PlayerState/PlayerContextHandler.swift | 18 + .../ios/Classes/PlayerState/PlayerState.swift | 85 ++ .../PlayerState/PlayerStateHandler.swift | 22 + .../ios/Classes/SpotifyError.swift | 4 + .../ios/Classes/SpotifySdkConstants.swift | 46 + .../ios/Classes/SpotifySdkPlugin.h | 4 + .../ios/Classes/SpotifySdkPlugin.m | 18 + .../ios/Classes/StatusHandler.swift | 17 + .../ios/Classes/SwiftSpotifySdkPlugin.swift | 416 ++++++ .../spotify_sdk/ios/prepare-iOS-SDK.sh | 10 + .../spotify_sdk/ios/spotify_sdk.podspec | 26 + .../lib/enums/image_dimension_enum.dart | 17 + .../lib/enums/podcast_playback_speed.dart | 23 + .../lib/enums/repeat_mode_enum.dart | 11 + .../extensions/image_dimension_extension.dart | 19 + .../podcast_playback_speed_extension.dart | 21 + .../lib/logging/custom_log_filter.dart | 8 + .../spotify_sdk/lib/models/album.dart | 15 + .../spotify_sdk/lib/models/album.g.dart | 17 + .../spotify_sdk/lib/models/artist.dart | 15 + .../spotify_sdk/lib/models/artist.g.dart | 17 + .../spotify_sdk/lib/models/capabilities.dart | 18 + .../lib/models/capabilities.g.dart | 16 + .../lib/models/connection_status.dart | 31 + .../lib/models/connection_status.g.dart | 23 + .../lib/models/crossfade_state.dart | 19 + .../lib/models/crossfade_state.g.dart | 19 + .../spotify_sdk/lib/models/image_uri.dart | 15 + .../spotify_sdk/lib/models/image_uri.g.dart | 15 + .../spotify_sdk/lib/models/library_state.dart | 24 + .../lib/models/library_state.g.dart | 20 + .../lib/models/player_context.dart | 18 + .../lib/models/player_context.g.dart | 23 + .../lib/models/player_options.dart | 30 + .../lib/models/player_options.g.dart | 25 + .../lib/models/player_restrictions.dart | 33 + .../lib/models/player_restrictions.g.dart | 27 + .../spotify_sdk/lib/models/player_state.dart | 36 + .../lib/models/player_state.g.dart | 29 + .../spotify_sdk/lib/models/track.dart | 43 + .../spotify_sdk/lib/models/track.g.dart | 35 + .../spotify_sdk/lib/models/user_status.dart | 26 + .../spotify_sdk/lib/models/user_status.g.dart | 20 + .../spotify_sdk/lib/platform_channels.dart | 161 +++ .../spotify_sdk/lib/spotify_sdk.dart | 660 ++++++++++ .../spotify_sdk/lib/spotify_sdk_web.dart | 1137 +++++++++++++++++ local_plugins/spotify_sdk/pubspec.yaml | 82 ++ .../spotify_sdk/test/spotify_sdk_test.dart | 28 + pubspec.yaml | 3 +- 107 files changed, 6214 insertions(+), 210 deletions(-) create mode 100644 local_plugins/spotify_sdk/.github/ISSUE_TEMPLATE/bug_report.md create mode 100644 local_plugins/spotify_sdk/.github/ISSUE_TEMPLATE/config.yml create mode 100644 local_plugins/spotify_sdk/.github/ISSUE_TEMPLATE/feature_request.md create mode 100644 local_plugins/spotify_sdk/.github/actions/publish_flutter_package/action.yaml create mode 100644 local_plugins/spotify_sdk/.github/dependabot.yml create mode 100755 local_plugins/spotify_sdk/.github/scripts/verify_pub_score.sh create mode 100644 local_plugins/spotify_sdk/.github/workflows/pull_request.yml create mode 100644 local_plugins/spotify_sdk/.github/workflows/release.yml create mode 100644 local_plugins/spotify_sdk/.gitignore create mode 100644 local_plugins/spotify_sdk/.metadata create mode 100644 local_plugins/spotify_sdk/CHANGELOG.md create mode 100644 local_plugins/spotify_sdk/CODE_OF_CONDUCT.md create mode 100644 local_plugins/spotify_sdk/CONTRIBUTING.md create mode 100644 local_plugins/spotify_sdk/LICENSE create mode 100644 local_plugins/spotify_sdk/README.md create mode 100644 local_plugins/spotify_sdk/analysis_options.yaml create mode 100644 local_plugins/spotify_sdk/android/.gitignore create mode 100644 local_plugins/spotify_sdk/android/build.gradle create mode 100644 local_plugins/spotify_sdk/android/gradle.properties create mode 100644 local_plugins/spotify_sdk/android/gradle/wrapper/gradle-wrapper.properties create mode 100644 local_plugins/spotify_sdk/android/libs/spotify-app-remote-release-0.8.0.aar create mode 100644 local_plugins/spotify_sdk/android/settings.gradle create mode 100644 local_plugins/spotify_sdk/android/src/main/AndroidManifest.xml create mode 100644 local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/BaseSpotifyApi.kt create mode 100644 local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/SpotifyConnectApi.kt create mode 100644 local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/SpotifyImagesApi.kt create mode 100644 local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/SpotifyPlayerApi.kt create mode 100644 local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/SpotifySdkPlugin.kt create mode 100644 local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/SpotifyUserApi.kt create mode 100644 local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/subscriptions/CapabilitiesChannel.kt create mode 100644 local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/subscriptions/ConnectionStatusChannel.kt create mode 100644 local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/subscriptions/PlayerContextChannel.kt create mode 100644 local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/subscriptions/PlayerStateChannel.kt create mode 100644 local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/subscriptions/UserStatusChannel.kt create mode 100644 local_plugins/spotify_sdk/bin/android_cleanup.dart create mode 100644 local_plugins/spotify_sdk/bin/android_module_creator.dart create mode 100644 local_plugins/spotify_sdk/bin/android_setup.dart create mode 100644 local_plugins/spotify_sdk/bin/github_api.dart create mode 100644 local_plugins/spotify_sdk/bin/precondition_checker.dart create mode 100644 local_plugins/spotify_sdk/ios/.gitignore create mode 100644 local_plugins/spotify_sdk/ios/Assets/.gitkeep create mode 100644 local_plugins/spotify_sdk/ios/Classes/ConnectionStatusHandler.swift create mode 100644 local_plugins/spotify_sdk/ios/Classes/Extension/Dictionary.swift create mode 100644 local_plugins/spotify_sdk/ios/Classes/PlayerDelegate.swift create mode 100644 local_plugins/spotify_sdk/ios/Classes/PlayerState/PlayerContextHandler.swift create mode 100644 local_plugins/spotify_sdk/ios/Classes/PlayerState/PlayerState.swift create mode 100644 local_plugins/spotify_sdk/ios/Classes/PlayerState/PlayerStateHandler.swift create mode 100644 local_plugins/spotify_sdk/ios/Classes/SpotifyError.swift create mode 100644 local_plugins/spotify_sdk/ios/Classes/SpotifySdkConstants.swift create mode 100644 local_plugins/spotify_sdk/ios/Classes/SpotifySdkPlugin.h create mode 100644 local_plugins/spotify_sdk/ios/Classes/SpotifySdkPlugin.m create mode 100644 local_plugins/spotify_sdk/ios/Classes/StatusHandler.swift create mode 100644 local_plugins/spotify_sdk/ios/Classes/SwiftSpotifySdkPlugin.swift create mode 100755 local_plugins/spotify_sdk/ios/prepare-iOS-SDK.sh create mode 100644 local_plugins/spotify_sdk/ios/spotify_sdk.podspec create mode 100644 local_plugins/spotify_sdk/lib/enums/image_dimension_enum.dart create mode 100644 local_plugins/spotify_sdk/lib/enums/podcast_playback_speed.dart create mode 100644 local_plugins/spotify_sdk/lib/enums/repeat_mode_enum.dart create mode 100644 local_plugins/spotify_sdk/lib/extensions/image_dimension_extension.dart create mode 100644 local_plugins/spotify_sdk/lib/extensions/podcast_playback_speed_extension.dart create mode 100644 local_plugins/spotify_sdk/lib/logging/custom_log_filter.dart create mode 100644 local_plugins/spotify_sdk/lib/models/album.dart create mode 100644 local_plugins/spotify_sdk/lib/models/album.g.dart create mode 100644 local_plugins/spotify_sdk/lib/models/artist.dart create mode 100644 local_plugins/spotify_sdk/lib/models/artist.g.dart create mode 100644 local_plugins/spotify_sdk/lib/models/capabilities.dart create mode 100644 local_plugins/spotify_sdk/lib/models/capabilities.g.dart create mode 100644 local_plugins/spotify_sdk/lib/models/connection_status.dart create mode 100644 local_plugins/spotify_sdk/lib/models/connection_status.g.dart create mode 100644 local_plugins/spotify_sdk/lib/models/crossfade_state.dart create mode 100644 local_plugins/spotify_sdk/lib/models/crossfade_state.g.dart create mode 100644 local_plugins/spotify_sdk/lib/models/image_uri.dart create mode 100644 local_plugins/spotify_sdk/lib/models/image_uri.g.dart create mode 100644 local_plugins/spotify_sdk/lib/models/library_state.dart create mode 100644 local_plugins/spotify_sdk/lib/models/library_state.g.dart create mode 100644 local_plugins/spotify_sdk/lib/models/player_context.dart create mode 100644 local_plugins/spotify_sdk/lib/models/player_context.g.dart create mode 100644 local_plugins/spotify_sdk/lib/models/player_options.dart create mode 100644 local_plugins/spotify_sdk/lib/models/player_options.g.dart create mode 100644 local_plugins/spotify_sdk/lib/models/player_restrictions.dart create mode 100644 local_plugins/spotify_sdk/lib/models/player_restrictions.g.dart create mode 100644 local_plugins/spotify_sdk/lib/models/player_state.dart create mode 100644 local_plugins/spotify_sdk/lib/models/player_state.g.dart create mode 100644 local_plugins/spotify_sdk/lib/models/track.dart create mode 100644 local_plugins/spotify_sdk/lib/models/track.g.dart create mode 100644 local_plugins/spotify_sdk/lib/models/user_status.dart create mode 100644 local_plugins/spotify_sdk/lib/models/user_status.g.dart create mode 100644 local_plugins/spotify_sdk/lib/platform_channels.dart create mode 100644 local_plugins/spotify_sdk/lib/spotify_sdk.dart create mode 100644 local_plugins/spotify_sdk/lib/spotify_sdk_web.dart create mode 100644 local_plugins/spotify_sdk/pubspec.yaml create mode 100644 local_plugins/spotify_sdk/test/spotify_sdk_test.dart diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index e388c263b..883dea602 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -3,6 +3,9 @@ import java.io.FileInputStream repositories { google() mavenCentral() + flatDir { + dirs(project(":spotify_sdk").projectDir.resolve("libs")) + } } plugins { @@ -54,6 +57,12 @@ android { } defaultConfig { + manifestPlaceholders.putAll( + mapOf( + "redirectSchemeName" to "spotify-sdk", + "redirectHostName" to "auth" + ) + ) // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html). applicationId = "com.yunqiinnovation.deepsound" // You can update the following values to match your application needs. @@ -110,6 +119,7 @@ dependencies { implementation(project(":volcano_speech")) implementation(project(":chat_storage")) // 添加新创建的SQLite聊天存储插件 implementation(project(":jieli")) // 添加新创建的SQLite聊天存储插件 + implementation(project(":spotify_sdk")) // 添加OkHttp依赖 implementation("com.squareup.okhttp3:okhttp:4.9.3") @@ -126,7 +136,6 @@ dependencies { // 添加 AndroidX Security 加密 SharedPreferences 依赖 implementation("androidx.security:security-crypto:1.1.0-alpha06") - } flutter { diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 23c380fb0..fe1f5f245 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -92,6 +92,10 @@ + + + +