From 85c029f60b28bc4cadc0fa3da1d81ea77656d041 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Wed, 23 Apr 2025 16:21:56 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../chat/controllers/chat_controller.dart | 3 +- .../open_ai_service/MCPClient.kt | 2 +- .../spotify_sdk/SpotifySdkService.kt | 41 +++---------------- 3 files changed, 9 insertions(+), 37 deletions(-) diff --git a/lib/modules/chat/controllers/chat_controller.dart b/lib/modules/chat/controllers/chat_controller.dart index 4ce3bf80b..997ef0fbb 100644 --- a/lib/modules/chat/controllers/chat_controller.dart +++ b/lib/modules/chat/controllers/chat_controller.dart @@ -112,8 +112,9 @@ class ChatController extends GetxController { // 获取Agent final Agent? foundAgent = AgentProvider.getAgentById(agentId); if (foundAgent == null) { - Logger.error('无法找到Agent: $agentId'); Get.back(); + Logger.error('无法找到Agent: $agentId'); + return; } diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt index b43013e0a..e2796ee85 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt @@ -168,7 +168,7 @@ class MCPSubClient( private fun processToolsList(toolsResult: ListToolsResult?) { try { // 将MCP工具列表转换为ToolUnion列表 - tools = toolsResult?.tools?.filter { tool -> config.tools == null || config.tools.contains(tool.name) == true + tools = toolsResult?.tools?.filter { tool -> config.tools.isNullOrEmpty() || config.tools.contains(tool.name) }?.map { tool -> // 转换为ToolUnion格式 ToolUnion.ofTool( diff --git a/local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/SpotifySdkService.kt b/local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/SpotifySdkService.kt index 9c97e7545..b5b551219 100644 --- a/local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/SpotifySdkService.kt +++ b/local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/SpotifySdkService.kt @@ -55,8 +55,7 @@ class SpotifySdkService(private val context: Context? = null) { private var spotifyAppRemote: SpotifyAppRemote? = null private var playerApi: PlayerApi? = null - //-- Method implementations - private fun connectToSpotify(clientId: String?, redirectUrl: String?, result: MethodChannel.Result) { + public fun connectToSpotify(clientId: String?, redirectUrl: String?, result: MethodChannel.Result) { if (clientId.isNullOrBlank() || redirectUrl.isNullOrBlank()) { result.error(errorConnecting, "client id or redirectUrl are not set or have invalid format", "") @@ -162,7 +161,7 @@ class SpotifySdkService(private val context: Context? = null) { } } - private fun getAccessToken(clientId: String?, redirectUrl: String?, scope: String?, result: MethodChannel.Result) { + public fun getAccessToken(clientId: String?, redirectUrl: String?, scope: String?, result: MethodChannel.Result) { if (applicationActivity == null) { throw IllegalStateException("getAccessToken needs a foreground activity") } @@ -172,8 +171,6 @@ class SpotifySdkService(private val context: Context? = null) { } else { //Convert String? scope to Array. Delimiter set as comma "," val scopeArray = scope?.split(",")?.toTypedArray() -// methodConnectToSpotify.checkAndSetPendingOperation(result) - val builder = AuthorizationRequest.Builder( clientId, AuthorizationResponse.Type.TOKEN, @@ -190,20 +187,9 @@ class SpotifySdkService(private val context: Context? = null) { } } - private fun disconnectFromSpotify(result: MethodChannel.Result) { + public fun disconnectFromSpotify(result: MethodChannel.Result) { if (spotifyAppRemote != null && spotifyAppRemote!!.isConnected) { SpotifyAppRemote.disconnect(spotifyAppRemote) - - // emit connection terminated event -// connStatusEventChannel( -// ConnectionStatusChannel.ConnectionEvent( -// false, -// "Successfully disconnected from Spotify.", -// null, -// null -// ) -// ) - // method success result.success(true) } else if (!spotifyAppRemote!!.isConnected) { result.error(errorDisconnecting, "could not disconnect spotify remote", "you are not connected, no need to disconnect") @@ -212,16 +198,7 @@ class SpotifySdkService(private val context: Context? = null) { } } - private fun play(spotifyUri: String?){ -// if (playerApi != null && !spotifyUri.isNullOrBlank()) { -// playerApi.play(spotifyUri) -// .setResultCallback { result.success(true) } -// .setErrorCallback { throwable -> result.error(errorPlay, "error when playing uri: $spotifyUri", throwable.toString()) } -// } else if (spotifyUri.isNullOrBlank()) { -// result.error(errorPlay, "spotifyUri has invalid format or is not set", "") -// } else { -// spotifyRemoteAppNotSetError() -// } + public fun play(spotifyUri: String?){ if (playerApi != null && !spotifyUri.isNullOrBlank()) { playerApi!!.play(spotifyUri) } else if (spotifyUri.isNullOrBlank()) { @@ -230,14 +207,8 @@ class SpotifySdkService(private val context: Context? = null) { } } - private fun pause(){ -// if (playerApi != null) { -// playerApi.pause() -// .setResultCallback { result.success(true) } -// .setErrorCallback { throwable -> result.error(errorPause, "error when pausing", throwable.toString()) } -// } else { -// spotifyRemoteAppNotSetError() -// } + + public fun pause(){ if (playerApi != null) { playerApi!!.pause() .setResultCallback {} From 63b44e72ab1e83e112167e90dded8bbe94d6de10 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Wed, 23 Apr 2025 17:23:20 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E8=81=8A=E5=A4=A9?= =?UTF-8?q?=E8=AF=AD=E9=9F=B3=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../deepsound/VoiceInteractionHandler.kt | 19 ++++++- .../chat/controllers/chat_controller.dart | 7 +-- ...otifySdkService.kt => SpotifySdkHelper.kt} | 51 +++++++------------ 3 files changed, 40 insertions(+), 37 deletions(-) rename local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/{SpotifySdkService.kt => SpotifySdkHelper.kt} (81%) diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt index 881b9dc84..f0a7d6d86 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt @@ -12,6 +12,7 @@ import com.yunqiinnovation.azure_speech.AzureAsrHelper import com.yunqiinnovation.azure_speech.AzureTtsHelper import com.yunqiinnovation.open_ai_service.OpenAIService import com.yunqiinnovation.open_ai_service.SystemFunctionHandler +import de.minimalme.spotify_sdk.SpotifySdkHelper /** @@ -38,7 +39,9 @@ class VoiceInteractionHandler( // OpenAI服务 private val openAIService = OpenAIService(context.applicationContext) - + + private var spotifyHelper: SpotifySdkHelper? = null + // 当前用户输入 private var currentUserInput = "" @@ -314,7 +317,19 @@ class VoiceInteractionHandler( notifyStateChanged() } } - + + /* + * 播放音乐。Spotify + * */ + fun playSpotifyMusic(url:String){ + if (spotifyHelper == null){ + spotifyHelper = SpotifySdkHelper(context).apply { + initialize(spotifyclientid,spotifyRedirectUrl) + } + } + spotifyHelper?.play(url) + } + /** * 播放AI回复 */ diff --git a/lib/modules/chat/controllers/chat_controller.dart b/lib/modules/chat/controllers/chat_controller.dart index 997ef0fbb..8bd00e873 100644 --- a/lib/modules/chat/controllers/chat_controller.dart +++ b/lib/modules/chat/controllers/chat_controller.dart @@ -414,7 +414,7 @@ class ChatController extends GetxController { if (chunk['card_spotify'] != null) { play(chunk['card_spotify']); } - } else { + } else if (chunk is String) { fullResponse += chunk; // 更新中间状态的消息内容 @@ -422,7 +422,8 @@ class ChatController extends GetxController { // 如果TTS启用，使用流式处理发送到TTS if (isTtsEnabled.value) { - _ttsService.speakStream(chunk); + String str = chunk.replaceAll('*', ''); + _ttsService.speakStream(str); } } }, @@ -456,7 +457,7 @@ class ChatController extends GetxController { final List> messageHistory = []; // 只取最近的10条消息作为上下文 - final recentMessages = messages.where((m) => !m.isIntermediate).toList(); + final recentMessages = messages.where((m) => m.role == "user").toList(); final startIndex = recentMessages.length > 10 ? recentMessages.length - 10 : 0; diff --git a/local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/SpotifySdkService.kt b/local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/SpotifySdkHelper.kt similarity index 81% rename from local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/SpotifySdkService.kt rename to local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/SpotifySdkHelper.kt index b5b551219..7ae21f8fe 100644 --- a/local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/SpotifySdkService.kt +++ b/local_plugins/spotify_sdk/android/src/main/kotlin/de/minimalme/spotify_sdk/SpotifySdkHelper.kt @@ -2,7 +2,6 @@ package de.minimalme.spotify_sdk import android.app.Activity import android.content.Context -import android.content.Intent import android.util.Log import com.spotify.android.appremote.api.ConnectionParams import com.spotify.android.appremote.api.Connector @@ -20,45 +19,33 @@ import com.spotify.android.appremote.api.error.UserNotAuthorizedException import com.spotify.sdk.android.auth.AuthorizationClient import com.spotify.sdk.android.auth.AuthorizationRequest import com.spotify.sdk.android.auth.AuthorizationResponse -import de.minimalme.spotify_sdk.subscriptions.CapabilitiesChannel -import de.minimalme.spotify_sdk.subscriptions.ConnectionStatusChannel -import de.minimalme.spotify_sdk.subscriptions.PlayerContextChannel -import de.minimalme.spotify_sdk.subscriptions.PlayerStateChannel -import de.minimalme.spotify_sdk.subscriptions.UserStatusChannel -import io.flutter.embedding.engine.plugins.FlutterPlugin -import io.flutter.embedding.engine.plugins.activity.ActivityAware -import io.flutter.embedding.engine.plugins.activity.ActivityPluginBinding -import io.flutter.plugin.common.EventChannel -import io.flutter.plugin.common.MethodCall -import io.flutter.plugin.common.MethodChannel -import io.flutter.plugin.common.PluginRegistry -import kotlinx.event.SetEvent -import kotlinx.event.event -class SpotifySdkService(private val context: Context? = null) { + +class SpotifySdkHelper(private val context: Context? = null) { // application context private var applicationContext : Context? = null private var applicationActivity : Activity? = null // method channel - private lateinit var methodChannel : MethodChannel - private val channelName = "spotify_sdk" - private val loggingTag = "spotify_sdk" + private val loggingTag = "spotify_sdk" - private val errorConnecting = "errorConnecting" - private val errorDisconnecting = "errorDisconnecting" private val errorConnection = "errorConnection" - private val errorAuthenticationToken = "authenticationTokenError" private val requestCodeAuthentication = 1337 private var spotifyAppRemote: SpotifyAppRemote? = null private var playerApi: PlayerApi? = null - public fun connectToSpotify(clientId: String?, redirectUrl: String?, result: MethodChannel.Result) { + fun initialize(clientId: String, redirectUrl: String): Boolean { + getAccessToken(clientId,redirectUrl,""); + connectToSpotify(clientId,redirectUrl) + return true; + } + + public fun connectToSpotify(clientId: String?, redirectUrl: String?) { if (clientId.isNullOrBlank() || redirectUrl.isNullOrBlank()) { - result.error(errorConnecting, "client id or redirectUrl are not set or have invalid format", "") +// result.error(errorConnecting, "client id or redirectUrl are not set or have invalid format", "") } else { val connectionParams = ConnectionParams.Builder(clientId) .setRedirectUri(redirectUrl) @@ -74,7 +61,7 @@ class SpotifySdkService(private val context: Context? = null) { initiallyConnected = true Log.i(loggingTag, "App Remote successfully connected") - result.success(true) +// result.success(true) } override fun onFailure(throwable: Throwable) { @@ -154,20 +141,20 @@ class SpotifySdkService(private val context: Context? = null) { // ) } else { // throw exception as the connect method - result.error(errorCode, errorMessage, errorDetails) +// result.error(errorCode, errorMessage, errorDetails) } } }) } } - public fun getAccessToken(clientId: String?, redirectUrl: String?, scope: String?, result: MethodChannel.Result) { + public fun getAccessToken(clientId: String?, redirectUrl: String?, scope: String?) { if (applicationActivity == null) { throw IllegalStateException("getAccessToken needs a foreground activity") } if (clientId.isNullOrBlank() || redirectUrl.isNullOrBlank()) { - result.error(errorConnecting, "client id or redirectUrl are not set or have invalid format", "") +// result.error(errorConnecting, "client id or redirectUrl are not set or have invalid format", "") } else { //Convert String? scope to Array. Delimiter set as comma "," val scopeArray = scope?.split(",")?.toTypedArray() @@ -187,14 +174,14 @@ class SpotifySdkService(private val context: Context? = null) { } } - public fun disconnectFromSpotify(result: MethodChannel.Result) { + public fun disconnectFromSpotify() { if (spotifyAppRemote != null && spotifyAppRemote!!.isConnected) { SpotifyAppRemote.disconnect(spotifyAppRemote) - result.success(true) +// result.success(true) } else if (!spotifyAppRemote!!.isConnected) { - result.error(errorDisconnecting, "could not disconnect spotify remote", "you are not connected, no need to disconnect") +// result.error(errorDisconnecting, "could not disconnect spotify remote", "you are not connected, no need to disconnect") } else { - result.error(errorDisconnecting, "could not disconnect spotify remote", "spotifyAppRemote is not set") +// result.error(errorDisconnecting, "could not disconnect spotify remote", "spotifyAppRemote is not set") } } From 735d2658b2a25163b39584585aade3bbb9739a36 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Wed, 23 Apr 2025 18:55:32 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E8=81=8A=E5=A4=A9?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E4=B8=8A=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/data/services/spotify_service.dart | 12 +++++----- .../chat/controllers/chat_controller.dart | 22 ++++++++++++------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/lib/data/services/spotify_service.dart b/lib/data/services/spotify_service.dart index 05d439e18..30adbfe97 100644 --- a/lib/data/services/spotify_service.dart +++ b/lib/data/services/spotify_service.dart @@ -20,15 +20,17 @@ class SpotifService { /// 初始化OpenAI服务 Future initialize() async { if (!isInitialized) { - await getAccessToken(); - await connectToSpotifyRemote(); - isInitialized = true; final _env = _storage.read("ENV") as Map; SPOTIFY_CLIENT_ID = _env['SPOTIFY_CLIENT_ID'] ?? ''; SPOTIFY_REDIRECT_URL = _env['SPOTIFY_REDIRECT_URL'] ?? ''; - printInfo(info: "SpotifService----------- initialize"); - } + printInfo( + info: + "SpotifService----------- initialize SPOTIFY_CLIENT_ID:${SPOTIFY_CLIENT_ID} SPOTIFY_REDIRECT_URL:${SPOTIFY_REDIRECT_URL}"); + await getAccessToken(); + await connectToSpotifyRemote(); + isInitialized = true; + } return true; } diff --git a/lib/modules/chat/controllers/chat_controller.dart b/lib/modules/chat/controllers/chat_controller.dart index 180e44e2f..a17b22173 100644 --- a/lib/modules/chat/controllers/chat_controller.dart +++ b/lib/modules/chat/controllers/chat_controller.dart @@ -403,13 +403,18 @@ class ChatController extends GetxController { messages: messageHistory, systemPrompt: agent.systemPrompt, ); - + Logger.info( + 'liwei----------- 发送用户提问 ${text}', + ); // 订阅响应流 _aiResponseSubscription = responseStream.listen( (chunk) { if (_shouldCancelAiResponse) return; Logger.info('chat ${chunk}'); if (chunk is Map) { + Logger.info( + 'liwei-----------收到卡片消息 ${chunk}', + ); _updateIntermediateMessage(meta: chunk); if (chunk['card_spotify'] != null) { play(chunk['card_spotify']); @@ -428,6 +433,9 @@ class ChatController extends GetxController { } }, onDone: () { + Logger.info( + 'liwei-----------结束当前问题 ${text}', + ); if (_shouldCancelAiResponse) return; // 完成响应，将isIntermediate设为false @@ -457,9 +465,13 @@ class ChatController extends GetxController { final List> messageHistory = []; // 只取最近的10条消息作为上下文 - final recentMessages = messages.where((m) => m.role == "user").toList(); + final recentMessages = messages.where((m) => !m.isIntermediate).toList(); final startIndex = recentMessages.length > 10 ? recentMessages.length - 10 : 0; + messageHistory.add({ + 'role': 'user', + 'content': '我当前所在城市是$_city，我的经纬度是$_latLng', + }); for (int i = startIndex; i < recentMessages.length; i++) { final m = recentMessages[i]; @@ -467,12 +479,6 @@ class ChatController extends GetxController { 'role': m.role, 'content': m.content, }); - if (i == recentMessages.length - 2) { - messageHistory.add({ - 'role': 'user', - 'content': '当需要获取我的位置时，我当前在$_city，我的经纬度是$_latLng', - }); - } } return messageHistory; From 804255cf2412803f494a29dd1cb67045ac4e4d53 Mon Sep 17 00:00:00 2001 From: fdp <1286779656@qq.com> Date: Wed, 23 Apr 2025 18:51:51 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E6=8E=A5=E9=80=9A=E7=BF=BB=E8=AF=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/data/services/asr_service.dart | 5 + .../speech_impl/azure_asr_service.dart | 18 + .../speech_impl/flutter_asr_service.dart | 239 +++++++------- .../speech_impl/xunfei_asr_service.dart | 6 + .../controllers/translation_controller.dart | 310 ++++++++++++------ .../azure_speech/AzureAsrHelper.kt | 32 +- .../azure_speech/AzureSpeechPlugin.kt | 16 + .../com/yunqiinnovation/jieli/JieliPlugin.kt | 2 +- local_plugins/jieli/lib/jieli.dart | 2 +- .../yunqiinnovation/jl_opus/JlOpusPlugin.kt | 2 +- 10 files changed, 388 insertions(+), 244 deletions(-) diff --git a/lib/data/services/asr_service.dart b/lib/data/services/asr_service.dart index fee73821c..474b16408 100644 --- a/lib/data/services/asr_service.dart +++ b/lib/data/services/asr_service.dart @@ -16,6 +16,11 @@ abstract class AsrService { /// 返回识别的最终结果 Future recognizeOnce(); + /// 开始连续语音识别 + /// + /// 推送音频数据到连续识别中 + Future pushAudioData(Uint8List data); + /// 开始连续语音识别 /// /// 返回一个包含识别事件的流 diff --git a/lib/data/services/speech_impl/azure_asr_service.dart b/lib/data/services/speech_impl/azure_asr_service.dart index dee08ed5e..878206ef3 100644 --- a/lib/data/services/speech_impl/azure_asr_service.dart +++ b/lib/data/services/speech_impl/azure_asr_service.dart @@ -288,4 +288,22 @@ class AzureAsrService extends GetxService implements AsrService { _isInitialized = false; } } + + @override + Future pushAudioData(Uint8List data) async { + try { + // 开始连续识别 + final bool result = await _channel.invokeMethod('pushAudioData', { + 'AudioData': data // Flutter 会自动将 Uint8List 转换为 ByteBuffer + }); + + if (!result) { + _cleanupEventStream(); + throw Exception('开始pushAudioData失败'); + } + } catch (e) { + Logger.error('开始pushAudioData失败: $e'); + rethrow; + } + } } diff --git a/lib/data/services/speech_impl/flutter_asr_service.dart b/lib/data/services/speech_impl/flutter_asr_service.dart index e17fb9198..7cbf9b88f 100644 --- a/lib/data/services/speech_impl/flutter_asr_service.dart +++ b/lib/data/services/speech_impl/flutter_asr_service.dart @@ -11,26 +11,26 @@ import '../asr_service.dart'; enum AsrState { /// 未初始化 notInitialized, - + /// 已初始化但未开始监听 initialized, - + /// 正在监听 listening, - + /// 正在处理识别结果 processing, - + /// 识别完成 done, - + /// 发生错误 error, } /// Flutter ASR服务，负责本地语音转文本功能 /// 使用speech_to_text插件实现语音识别功能 -class FlutterAsrService extends GetxService implements AsrService { +class FlutterAsrService extends GetxService implements AsrService { static final FlutterAsrService to = Get.put(FlutterAsrService()); // speech_to_text 实例 @@ -40,34 +40,34 @@ class FlutterAsrService extends GetxService implements AsrService { final List _supportedLanguages = []; @override List get supportedLanguages => _supportedLanguages; - + // 当前选择的语言 final RxString _currentLocale = ''.obs; String get currentLocale => _currentLocale.value; - + // 是否可用 final RxBool _isAvailable = false.obs; bool get isAvailable => _isAvailable.value; - + // 是否正在监听 bool _isListening = false; - + // 当前ASR状态 final Rx _asrState = AsrState.notInitialized.obs; AsrState get asrState => _asrState.value; - + // 连续识别模式的结果流控制器 StreamController? _continuousRecognitionController; - + FlutterAsrService() { _initSpeech(); } - + /// 初始化语音识别 Future _initSpeech() async { try { _asrState.value = AsrState.notInitialized; - + // 初始化语音识别，使用较短的最终超时时间，提高响应速度 _isAvailable.value = await _speech.initialize( onStatus: _onSpeechStatus, @@ -75,11 +75,11 @@ class FlutterAsrService extends GetxService implements AsrService { debugLogging: false, finalTimeout: const Duration(milliseconds: 800), // 更短的最终超时时间，加速结果返回 ); - + if (_isAvailable.value) { // 获取支持的语言列表 final systemLocales = await _speech.locales(); - + // 清空并重新填充支持的语言列表 _supportedLanguages.clear(); for (var locale in systemLocales) { @@ -87,7 +87,7 @@ class FlutterAsrService extends GetxService implements AsrService { _supportedLanguages.add(locale.localeId); } } - + // 设置默认语言（优先使用系统语言或中文） var systemLocale = await _speech.systemLocale(); if (systemLocale != null) { @@ -97,7 +97,7 @@ class FlutterAsrService extends GetxService implements AsrService { } else if (_supportedLanguages.isNotEmpty) { _currentLocale.value = _supportedLanguages.first; } - + _asrState.value = AsrState.initialized; Logger.info('语音识别初始化成功，支持 ${_supportedLanguages.length} 种语言'); } else { @@ -110,11 +110,11 @@ class FlutterAsrService extends GetxService implements AsrService { Logger.error('语音识别初始化失败: $e'); } } - + /// 处理语音识别状态变化 void _onSpeechStatus(String status) { Logger.info('语音识别状态: $status'); - + switch (status) { case 'listening': _asrState.value = AsrState.listening; @@ -129,31 +129,33 @@ class FlutterAsrService extends GetxService implements AsrService { case 'done': _asrState.value = AsrState.done; _isListening = false; - + // 在连续识别模式下，当一次识别完成后立即开始下一次识别 - if (_continuousRecognitionController != null && !_continuousRecognitionController!.isClosed) { + if (_continuousRecognitionController != null && + !_continuousRecognitionController!.isClosed) { _restartContinuousRecognition(); } break; } } - + /// 处理语音识别错误 void _onSpeechError(SpeechRecognitionError error) { Logger.error('语音识别错误: ${error.errorMsg} (永久性错误: ${error.permanent})'); _asrState.value = AsrState.error; - + // 向连续识别控制器发送错误事件（如果存在） - _continuousRecognitionController?.add( - RecognitionEvent.error(error.errorMsg) - ); - + _continuousRecognitionController + ?.add(RecognitionEvent.error(error.errorMsg)); + // 在连续识别模式下，尝试从错误中恢复（非永久性错误） - if (!error.permanent && _continuousRecognitionController != null && !_continuousRecognitionController!.isClosed) { + if (!error.permanent && + _continuousRecognitionController != null && + !_continuousRecognitionController!.isClosed) { _restartContinuousRecognition(); } } - + /// 启动语音识别的统一方法 Future _startListening({ required Function(SpeechRecognitionResult) onResult, @@ -168,36 +170,37 @@ class FlutterAsrService extends GetxService implements AsrService { // 延长等待时间，确保资源完全释放 await Future.delayed(const Duration(milliseconds: 800)); } - + // 在启动前再次检查状态 if (_speech.isListening) { Logger.warning('停止识别后，仍处于监听状态，可能有系统冲突'); await _speech.stop(); await Future.delayed(const Duration(milliseconds: 500)); } - + // 启动语音识别前确保状态正确 _asrState.value = AsrState.listening; - + // 启动语音识别 final result = await _speech.listen( - onResult: onResult, - localeId: _currentLocale.value, - listenFor: listenFor, - pauseFor: pauseFor, - listenOptions: stt.SpeechListenOptions( - partialResults: true, - autoPunctuation: autoPunctuation, - // 使用独白模式，减少与系统交互冲突 - listenMode: stt.ListenMode.dictation, - ), - ) ?? false; - + onResult: onResult, + localeId: _currentLocale.value, + listenFor: listenFor, + pauseFor: pauseFor, + listenOptions: stt.SpeechListenOptions( + partialResults: true, + autoPunctuation: autoPunctuation, + // 使用独白模式，减少与系统交互冲突 + listenMode: stt.ListenMode.dictation, + ), + ) ?? + false; + if (!result) { Logger.error('语音识别启动失败'); _asrState.value = AsrState.error; } - + return result; } catch (e) { Logger.error('启动语音识别失败: $e'); @@ -205,33 +208,35 @@ class FlutterAsrService extends GetxService implements AsrService { return false; } } - + /// 重新启动连续识别 Future _restartContinuousRecognition() async { try { // 确保控制器有效 - if (_continuousRecognitionController == null || _continuousRecognitionController!.isClosed) { + if (_continuousRecognitionController == null || + _continuousRecognitionController!.isClosed) { Logger.info('连续识别控制器已关闭，跳过重启'); return; } - + // 检查系统状态，避免频繁重启 if (_speech.isListening) { Logger.info('当前已在监听中，无需重启'); return; } - + // 延长等待时间，避免与系统资源冲突 await Future.delayed(const Duration(milliseconds: 1000)); - + // 再次检查状态 - if (_continuousRecognitionController == null || _continuousRecognitionController!.isClosed) { + if (_continuousRecognitionController == null || + _continuousRecognitionController!.isClosed) { Logger.info('延迟期间控制器已关闭，跳过重启'); return; } - + Logger.info('尝试重新启动连续识别'); - + // 使用统一方法启动识别 final success = await _startListening( onResult: _onSpeechResult, @@ -239,49 +244,42 @@ class FlutterAsrService extends GetxService implements AsrService { listenFor: const Duration(seconds: 30), pauseFor: const Duration(seconds: 4), ); - + if (success) { Logger.info('连续识别重启成功'); } else { Logger.error('连续识别重启失败'); - _continuousRecognitionController?.add( - RecognitionEvent.error('重启语音识别失败，可能存在系统资源冲突') - ); + _continuousRecognitionController + ?.add(RecognitionEvent.error('重启语音识别失败，可能存在系统资源冲突')); } } catch (e) { Logger.error('重启连续识别失败: $e'); - + // 发送错误给控制器 - _continuousRecognitionController?.add( - RecognitionEvent.error('重启失败: $e') - ); + _continuousRecognitionController?.add(RecognitionEvent.error('重启失败: $e')); } } - + /// 处理语音识别结果 void _onSpeechResult(SpeechRecognitionResult result) { // 如果是连续识别，发送到连续识别控制器 if (_continuousRecognitionController != null) { if (result.finalResult) { Logger.info('收到最终识别结果: "${result.recognizedWords}"'); - _continuousRecognitionController!.add( - RecognitionEvent.finalResult( - text: result.recognizedWords, - detectedLanguage: currentLocale, - ) - ); + _continuousRecognitionController!.add(RecognitionEvent.finalResult( + text: result.recognizedWords, + detectedLanguage: currentLocale, + )); } else { - _continuousRecognitionController!.add( - RecognitionEvent( - type: RecognitionEventType.intermediateResult, - text: result.recognizedWords, - detectedLanguage: currentLocale, - ) - ); + _continuousRecognitionController!.add(RecognitionEvent( + type: RecognitionEventType.intermediateResult, + text: result.recognizedWords, + detectedLanguage: currentLocale, + )); } } } - + /// 设置当前语言 void setLocale(String localeId) { if (_supportedLanguages.contains(localeId)) { @@ -291,7 +289,7 @@ class FlutterAsrService extends GetxService implements AsrService { Logger.error('不支持的语言: $localeId'); } } - + @override Future initialize({ List? supportedLanguages, @@ -301,7 +299,7 @@ class FlutterAsrService extends GetxService implements AsrService { _supportedLanguages.clear(); _supportedLanguages.addAll(supportedLanguages); } - + // 初始化语音识别 await _initSpeech(); return _isAvailable.value; @@ -310,24 +308,24 @@ class FlutterAsrService extends GetxService implements AsrService { return false; } } - + @override Future recognizeOnce() async { try { if (!_isAvailable.value) { return RecognitionEvent.error('语音识别不可用'); } - + if (_isListening) { await stopContinuousRecognition(); } - + // 创建Completer来等待最终结果 final completer = Completer(); - + // 开始监听 _asrState.value = AsrState.listening; - + // 定义结果处理函数 void resultListener(SpeechRecognitionResult result) { if (result.finalResult) { @@ -338,21 +336,21 @@ class FlutterAsrService extends GetxService implements AsrService { )); } } - + // 使用统一方法启动识别，单次识别使用更短的时间参数 final success = await _startListening( onResult: resultListener, listenFor: const Duration(seconds: 15), pauseFor: const Duration(seconds: 2), ); - + if (!success) { return RecognitionEvent.error('启动语音识别失败'); } - + // 设置较短的超时时间 return await completer.future.timeout( - const Duration(seconds: 20), // 略长于listenFor以确保获取到结果 + const Duration(seconds: 20), // 略长于listenFor以确保获取到结果 onTimeout: () { _speech.stop(); return RecognitionEvent.error('语音识别超时'); @@ -363,7 +361,7 @@ class FlutterAsrService extends GetxService implements AsrService { return RecognitionEvent.error('$e'); } } - + @override Future> startContinuousRecognition() async { try { @@ -373,43 +371,42 @@ class FlutterAsrService extends GetxService implements AsrService { controller.close(); return controller.stream; } - + // 如果已经在监听，先停止 if (_isListening) { await stopContinuousRecognition(); } - + // 创建新的流控制器 - _continuousRecognitionController = StreamController.broadcast(); - + _continuousRecognitionController = + StreamController.broadcast(); + // 开始会话 - _continuousRecognitionController!.add( - RecognitionEvent(type: RecognitionEventType.sessionStarted) - ); - + _continuousRecognitionController! + .add(RecognitionEvent(type: RecognitionEventType.sessionStarted)); + // 开始连续监听 _asrState.value = AsrState.listening; - + // 使用统一方法启动识别，连续识别使用较长的参数 final success = await _startListening( onResult: _onSpeechResult, ); - + if (!success) { // 添加错误事件，但保留流 - _continuousRecognitionController!.add( - RecognitionEvent.error('启动连续语音识别失败') - ); - + _continuousRecognitionController! + .add(RecognitionEvent.error('启动连续语音识别失败')); + // 返回已有的流控制器 return _continuousRecognitionController!.stream; } - + _isListening = true; return _continuousRecognitionController!.stream; } catch (e) { Logger.error('启动连续语音识别失败: $e'); - + // 如果已经创建了控制器，使用它 if (_continuousRecognitionController != null) { _continuousRecognitionController!.add(RecognitionEvent.error('$e')); @@ -422,7 +419,7 @@ class FlutterAsrService extends GetxService implements AsrService { } } } - + @override Future stopContinuousRecognition() async { try { @@ -430,36 +427,35 @@ class FlutterAsrService extends GetxService implements AsrService { if (_speech.isListening) { await _speech.stop(); } - + _isListening = false; - + // 发送会话结束事件 - _continuousRecognitionController?.add( - RecognitionEvent(type: RecognitionEventType.sessionStopped) - ); - + _continuousRecognitionController + ?.add(RecognitionEvent(type: RecognitionEventType.sessionStopped)); + // 关闭流控制器 await _continuousRecognitionController?.close(); _continuousRecognitionController = null; - + _asrState.value = AsrState.done; return true; } catch (e) { Logger.error('停止连续语音识别失败: $e'); - + // 强制重置状态 _isListening = false; _asrState.value = AsrState.error; - + return false; } } - + @override bool isContinuousRecognitionActive() { return _isListening; } - + @override Future dispose() async { try { @@ -467,15 +463,18 @@ class FlutterAsrService extends GetxService implements AsrService { if (_speech.isListening) { await _speech.stop(); } - + // 关闭控制器 await _continuousRecognitionController?.close(); _continuousRecognitionController = null; - + _isListening = false; _asrState.value = AsrState.notInitialized; } catch (e) { Logger.error('释放语音识别资源失败: $e'); } } -} \ No newline at end of file + + @override + Future pushAudioData(Uint8List data) async {} +} diff --git a/lib/data/services/speech_impl/xunfei_asr_service.dart b/lib/data/services/speech_impl/xunfei_asr_service.dart index 1201f7c66..cadb48b2f 100644 --- a/lib/data/services/speech_impl/xunfei_asr_service.dart +++ b/lib/data/services/speech_impl/xunfei_asr_service.dart @@ -269,4 +269,10 @@ class XunfeiAsrService extends GetxService implements AsrService { _isInitialized = false; } } + + @override + Future pushAudioData(Uint8List data) { + // TODO: implement pushAudioData + throw UnimplementedError(); + } } diff --git a/lib/modules/translation/controllers/translation_controller.dart b/lib/modules/translation/controllers/translation_controller.dart index 59822e45f..d07fe9a22 100644 --- a/lib/modules/translation/controllers/translation_controller.dart +++ b/lib/modules/translation/controllers/translation_controller.dart @@ -10,14 +10,37 @@ import '../../../core/utils/logger.dart'; import '../../../data/models/events/voice_interaction_event.dart'; import '../../../data/services/voice_interaction_service.dart'; import '../../../data/services/asr_service.dart'; +import 'package:jieli/jieli.dart'; + class TranslationController extends GetxController { + late Jieli jieli; + // 事件订阅 + StreamSubscription? _scanSubscription; + StreamSubscription? _connectionSubscription; + StreamSubscription? _wakeupSubscription; + StreamSubscription? _audioSubscription; + StreamSubscription? _commandResultSubscription; + // 状态变量 + var isScanning = false.obs; + var deviceInfo = '未获取'.obs; + var connectStatus = '未连接'.obs; + var currentDevice = Rx(null); + var scannedDevices = RxList([]); + var wakeupReceived = false.obs; + var audioDataSize = 0.obs; + var receivedFrames = 0.obs; + var audioDataReceiving = false.obs; + var volumeCommandResult = ''.obs; + // 服务 final AsrService _asrService = Get.find(); - final VolcanoTranslationService _translationService = Get.find(); + final VolcanoTranslationService _translationService = + Get.find(); final TtsService _ttsService = Get.find(); final LanguageManager _languageManager = Get.find(); final GetStorage _storage = GetStorage(); - final VoiceInteractionService _voiceInteractionService = Get.find(); + final VoiceInteractionService _voiceInteractionService = + Get.find(); // 存储相关 static const String _historyKey = 'translation_history'; @@ -29,75 +52,80 @@ class TranslationController extends GetxController { final isRecognizing = false.obs; final isTranslating = false.obs; final isTtsEnabled = true.obs; - + // 语言相关 final sourceLanguage = '中文(简体)'.obs; final targetLanguage = '英语'.obs; final sourceLanguageCode = 'zh-CN'.obs; final targetLanguageCode = 'en-US'.obs; var detectedLanguageCode = ''; - + // 翻译历史和当前项 final translationHistory = [].obs; String? currentSessionId; - + // 兼容旧版视图的计算属性 Rx get currentItem { - if (translationHistory.isNotEmpty && translationHistory.last.isIntermediate) { + if (translationHistory.isNotEmpty && + translationHistory.last.isIntermediate) { return Rx(translationHistory.last); } return Rx(null); } - + // UI相关 final ScrollController scrollController = ScrollController(); - + // 其他 StreamSubscription? _recognitionSubscription; StreamSubscription? _voiceInteractionSubscription; Timer? _translationDebounceTimer; // 获取支持的语言列表 - Map get supportedLanguages => _languageManager.getChineseNameToAsrCodeMap(); + Map get supportedLanguages => + _languageManager.getChineseNameToAsrCodeMap(); @override void onInit() { super.onInit(); - + jieli = Jieli(); + //_setupEventListeners(); // 加载上次保存的语言设置 _loadSavedLanguages(); - + // 初始化语言代码 - final sourceAsrCode = _languageManager.getAsrCodeByChineseName(sourceLanguage.value); + final sourceAsrCode = + _languageManager.getAsrCodeByChineseName(sourceLanguage.value); if (sourceAsrCode != null) { sourceLanguageCode.value = sourceAsrCode; } - - final targetAsrCode = _languageManager.getAsrCodeByChineseName(targetLanguage.value); + + final targetAsrCode = + _languageManager.getAsrCodeByChineseName(targetLanguage.value); if (targetAsrCode != null) { targetLanguageCode.value = targetAsrCode; } - + // 订阅原生服务事件 _subscribeToVoiceInteractionService(); // 加载历史记录 loadTranslationHistory(); - + // 初始化服务 _initServices(); - + // 使用微任务在布局完成后自动滚动到底部 WidgetsBinding.instance.addPostFrameCallback((_) { _scrollToBottom(); - }); } - + /// 订阅原生服务事件 void _subscribeToVoiceInteractionService() { if (_voiceInteractionService.eventStream != null) { - _voiceInteractionSubscription = _voiceInteractionService.eventStream!.listen((event) { + _voiceInteractionSubscription = + _voiceInteractionService.eventStream!.listen((event) { if (event is RecognitionStartedEvent) { _handleVoiceInteractionStarted(); } @@ -109,35 +137,54 @@ class TranslationController extends GetxController { void _loadSavedLanguages() { final savedSourceLanguage = _storage.read(_sourceLanguageKey); final savedTargetLanguage = _storage.read(_targetLanguageKey); - - if (savedSourceLanguage != null && supportedLanguages.containsKey(savedSourceLanguage)) { + + if (savedSourceLanguage != null && + supportedLanguages.containsKey(savedSourceLanguage)) { sourceLanguage.value = savedSourceLanguage; } - - if (savedTargetLanguage != null && supportedLanguages.containsKey(savedTargetLanguage)) { + + if (savedTargetLanguage != null && + supportedLanguages.containsKey(savedTargetLanguage)) { targetLanguage.value = savedTargetLanguage; } } - + // 保存当前语言设置 void _saveLanguageSettings() { _storage.write(_sourceLanguageKey, sourceLanguage.value); _storage.write(_targetLanguageKey, targetLanguage.value); } - + + // /// 设置事件监听器 + // void _setupEventListeners() { + // // 监听扫描结果 + // _scanSubscription = jieli.scanResults.listen((devices) { + // scannedDevices.assignAll(devices); + // }); + + // // 监听唤醒信号 + // _wakeupSubscription = jieli.wakeupStream.listen((wakeup) { + // wakeupReceived.value = true; + // Get.snackbar('收到唤醒信号', '耳机发送了唤醒信号'); + // }); + // } + // 初始化服务 Future _initServices() async { try { await _translationService.initialize(); - + // 初始化 ASR 服务时，明确指定需要支持的语言 - final List asrSupportedLanguages = [sourceLanguageCode.value, targetLanguageCode.value]; + final List asrSupportedLanguages = [ + sourceLanguageCode.value, + targetLanguageCode.value + ]; Logger.info('初始化ASR服务，支持语言: $asrSupportedLanguages'); - + await _asrService.initialize( supportedLanguages: asrSupportedLanguages, ); - + Logger.info('所有服务初始化完成'); } catch (e) { Logger.error('初始化服务失败: $e'); @@ -159,20 +206,26 @@ class TranslationController extends GetxController { // 开始语音识别 Future startRecognition() async { + Logger.info('开始语音识别'); if (isRecognizing.value) return; _voiceInteractionService.pauseVoiceInteraction(); try { + Logger.info('开始语音识别1'); // 每次开始识别前检查 ASR 服务是否已配置为当前语言对 - final List currentLanguages = [sourceLanguageCode.value, targetLanguageCode.value]; - final bool needsReinit = !_asrService.supportedLanguages.toSet().containsAll(currentLanguages); - + final List currentLanguages = [ + sourceLanguageCode.value, + targetLanguageCode.value + ]; + final bool needsReinit = + !_asrService.supportedLanguages.toSet().containsAll(currentLanguages); + if (needsReinit) { Logger.info('识别开始前发现语言变更，重新初始化ASR服务'); await _reinitializeAsrService(); } - + currentSessionId = DateTime.now().millisecondsSinceEpoch.toString(); // // 初始化新的翻译项 @@ -186,17 +239,25 @@ class TranslationController extends GetxController { // isFirstInSession: true, // isIntermediate: true, // ); - + // // 添加到历史记录 // translationHistory.add(newItem); - + // 开始连续语音识别 final recognitionStream = await _asrService.startContinuousRecognition(); + Logger.info('翻译开启，开始监'); + // 监听音频数据 + _audioSubscription = jieli.audioDataStream.listen((data) async { + // Logger.info('翻译开启，开始监听解码后的数据=${data}'); + final pushAudioDataStream = await _asrService.pushAudioData(data); + }); + isRecognizing.value = true; // 监听识别结果 _recognitionSubscription?.cancel(); - _recognitionSubscription = recognitionStream.listen(_handleRecognitionEvent); + _recognitionSubscription = + recognitionStream.listen(_handleRecognitionEvent); } catch (e) { isRecognizing.value = false; Logger.error('启动语音识别失败: $e'); @@ -205,10 +266,12 @@ class TranslationController extends GetxController { // 处理识别事件 void _handleRecognitionEvent(RecognitionEvent event) { - if (event.type == RecognitionEventType.finalResult && event.text.isNotEmpty) { + if (event.type == RecognitionEventType.finalResult && + event.text.isNotEmpty) { detectedLanguageCode = event.detectedLanguage; handleFinalResult(event.text); - } else if (event.type == RecognitionEventType.intermediateResult && event.text.isNotEmpty) { + } else if (event.type == RecognitionEventType.intermediateResult && + event.text.isNotEmpty) { if (event.detectedLanguage.isNotEmpty) { detectedLanguageCode = event.detectedLanguage; } @@ -231,21 +294,25 @@ class TranslationController extends GetxController { _recognitionSubscription?.cancel(); _recognitionSubscription = null; isRecognizing.value = false; - + // 如果当前有中间结果，将其作为最终结果处理 - if (translationHistory.isNotEmpty && translationHistory.last.isIntermediate && translationHistory.last.sourceText.isNotEmpty) { + if (translationHistory.isNotEmpty && + translationHistory.last.isIntermediate && + translationHistory.last.sourceText.isNotEmpty) { // 标记为最终结果 translationHistory.last.isIntermediate = false; translationHistory.refresh(); - + // 如果还没有翻译结果，尝试翻译 if (translationHistory.last.translatedText.isEmpty) { - await translateText(translationHistory.last.sourceText, translationHistory.last.timestamp, isFinal: true); + await translateText(translationHistory.last.sourceText, + translationHistory.last.timestamp, + isFinal: true); } - + saveTranslationHistory(); } - + currentSessionId = null; } catch (e) { Logger.error('停止语音识别失败: $e'); @@ -265,10 +332,11 @@ class TranslationController extends GetxController { targetLanguageCode: targetLanguageCode.value, timestamp: DateTime.now(), sessionId: currentSessionId!, - isFirstInSession: translationHistory.isEmpty || translationHistory.last.sessionId != currentSessionId, + isFirstInSession: translationHistory.isEmpty || + translationHistory.last.sessionId != currentSessionId, isIntermediate: true, ); - + translationHistory.add(newItem); // 添加新项后滚动到底部 _scrollToBottom(); @@ -283,8 +351,12 @@ class TranslationController extends GetxController { // 防抖处理翻译请求 _translationDebounceTimer?.cancel(); _translationDebounceTimer = Timer(const Duration(milliseconds: 300), () { - if (translationHistory.isNotEmpty && translationHistory.last.isIntermediate && translationHistory.last.sourceText.isNotEmpty) { - translateText(translationHistory.last.sourceText, translationHistory.last.timestamp, isFinal: false); + if (translationHistory.isNotEmpty && + translationHistory.last.isIntermediate && + translationHistory.last.sourceText.isNotEmpty) { + translateText(translationHistory.last.sourceText, + translationHistory.last.timestamp, + isFinal: false); } }); } @@ -301,7 +373,7 @@ class TranslationController extends GetxController { // 更新后滚动到底部 _scrollToBottom(); saveTranslationHistory(); - + // 使用项目的时间戳进行翻译 translateText(text, translationHistory.last.timestamp, isFinal: true); } else { @@ -312,16 +384,17 @@ class TranslationController extends GetxController { sourceLanguageCode: sourceLanguageCode.value, targetLanguageCode: targetLanguageCode.value, timestamp: DateTime.now(), - sessionId: currentSessionId ?? DateTime.now().millisecondsSinceEpoch.toString(), - isFirstInSession: translationHistory.isEmpty || translationHistory.last.sessionId != currentSessionId, + sessionId: currentSessionId ?? + DateTime.now().millisecondsSinceEpoch.toString(), + isFirstInSession: translationHistory.isEmpty || + translationHistory.last.sessionId != currentSessionId, isIntermediate: false, ); - + translationHistory.add(newItem); translationHistory.refresh(); _scrollToBottom(); saveTranslationHistory(); - // 使用新项目的时间戳进行翻译 translateText(text, newItem.timestamp, isFinal: true); @@ -329,7 +402,8 @@ class TranslationController extends GetxController { } // 翻译文本 - Future translateText(String sourceText, DateTime timestamp, {bool isFinal = false}) async { + Future translateText(String sourceText, DateTime timestamp, + {bool isFinal = false}) async { if (sourceText.isEmpty) return null; isTranslating.value = true; @@ -338,8 +412,10 @@ class TranslationController extends GetxController { try { // 根据检测到的语言确定源语言和目标语言 final bool shouldSwap = detectedLanguageCode == sourceLanguageCode.value; - final detectedSourceLanguageCode = shouldSwap ? sourceLanguageCode.value : targetLanguageCode.value; - final detectedTargetLanguageCode = shouldSwap ? targetLanguageCode.value : sourceLanguageCode.value; + final detectedSourceLanguageCode = + shouldSwap ? sourceLanguageCode.value : targetLanguageCode.value; + final detectedTargetLanguageCode = + shouldSwap ? targetLanguageCode.value : sourceLanguageCode.value; // 调用翻译服务 translationResult = await _translationService.translateText( @@ -352,25 +428,28 @@ class TranslationController extends GetxController { // 查找匹配的项目 - 优先使用timestamp查找 int index = -1; - index = translationHistory.indexWhere((item) => - item.timestamp.isAtSameMomentAs(timestamp)); + index = translationHistory + .indexWhere((item) => item.timestamp.isAtSameMomentAs(timestamp)); if (index != -1) { // 更新历史项 translationHistory[index].translatedText = translationResult; - translationHistory[index].sourceLanguageCode = detectedSourceLanguageCode; - translationHistory[index].targetLanguageCode = detectedTargetLanguageCode; + translationHistory[index].sourceLanguageCode = + detectedSourceLanguageCode; + translationHistory[index].targetLanguageCode = + detectedTargetLanguageCode; translationHistory.refresh(); saveTranslationHistory(); // 更新后滚动到底部 _scrollToBottom(); - + // 播放TTS - 只在最终结果时播放 if (isTtsEnabled.value && isFinal) { // Logger.info('播放翻译文本: ${translationResult}'); await playTranslatedText(translationResult); } } else { - Logger.info('未找到匹配的历史项 ${sourceText} ${timestamp.toIso8601String() ?? "无时间戳"}'); + Logger.info( + '未找到匹配的历史项 ${sourceText} ${timestamp.toIso8601String() ?? "无时间戳"}'); } } } catch (e) { @@ -388,9 +467,12 @@ class TranslationController extends GetxController { try { final bool shouldSwap = detectedLanguageCode == targetLanguageCode.value; - final detectedTargetLanguageCode = shouldSwap ? sourceLanguageCode.value : targetLanguageCode.value; - final voiceName = _languageManager.getTtsVoiceNameByAsrCode(detectedTargetLanguageCode) ?? 'en-US-AriaNeural'; - + final detectedTargetLanguageCode = + shouldSwap ? sourceLanguageCode.value : targetLanguageCode.value; + final voiceName = _languageManager + .getTtsVoiceNameByAsrCode(detectedTargetLanguageCode) ?? + 'en-US-AriaNeural'; + await _ttsService.setVoice(voiceName); await _ttsService.speakOnce(text); } catch (e) { @@ -404,7 +486,7 @@ class TranslationController extends GetxController { if (isRecognizing.value) { await stopRecognition(); } - + _translationDebounceTimer?.cancel(); await _ttsService.stop(); } catch (e) { @@ -415,9 +497,11 @@ class TranslationController extends GetxController { // 播放特定翻译项 Future playTranslationItem(TranslationItem item) async { if (item.translatedText.isEmpty) return; - + try { - final voiceName = _languageManager.getTtsVoiceNameByAsrCode(item.targetLanguageCode) ?? 'en-US-AriaNeural'; + final voiceName = + _languageManager.getTtsVoiceNameByAsrCode(item.targetLanguageCode) ?? + 'en-US-AriaNeural'; await _ttsService.setVoice(voiceName); await _ttsService.speakOnce(item.translatedText); } catch (e) { @@ -428,12 +512,15 @@ class TranslationController extends GetxController { // 播放文本 Future playText(String text, {String? languageCode}) async { if (text.isEmpty) return; - + try { - final voiceName = languageCode != null - ? _languageManager.getTtsVoiceNameByAsrCode(languageCode) ?? 'en-US-AriaNeural' - : _languageManager.getTtsVoiceNameByAsrCode(targetLanguageCode.value) ?? 'en-US-AriaNeural'; - + final voiceName = languageCode != null + ? _languageManager.getTtsVoiceNameByAsrCode(languageCode) ?? + 'en-US-AriaNeural' + : _languageManager + .getTtsVoiceNameByAsrCode(targetLanguageCode.value) ?? + 'en-US-AriaNeural'; + await _ttsService.setVoice(voiceName); await _ttsService.speakOnce(text); } catch (e) { @@ -443,7 +530,8 @@ class TranslationController extends GetxController { // 滚动到底部 void _scrollToBottom() { - if (scrollController.hasClients && (translationHistory.isNotEmpty || currentSessionId != null)) { + if (scrollController.hasClients && + (translationHistory.isNotEmpty || currentSessionId != null)) { try { scrollController.animateTo( scrollController.position.maxScrollExtent, @@ -460,33 +548,33 @@ class TranslationController extends GetxController { void setupScrollListener() { // 简化后的空方法，保留接口兼容性 } - + // 底部控制栏高度 double _bottomBarHeight = 0.0; - + // 设置底部控制栏高度 void setBottomBarHeight(double height) { _bottomBarHeight = height; } - + // 获取底部控制栏高度 double get bottomBarHeight => _bottomBarHeight; // 切换源语言和目标语言 void swapLanguages() { stopAll(); - + final tempName = sourceLanguage.value; sourceLanguage.value = targetLanguage.value; targetLanguage.value = tempName; - + final tempCode = sourceLanguageCode.value; sourceLanguageCode.value = targetLanguageCode.value; targetLanguageCode.value = tempCode; - + // 保存语言设置 _saveLanguageSettings(); - + // 更新ASR支持的语言并重新初始化 _reinitializeAsrService(); } @@ -495,17 +583,17 @@ class TranslationController extends GetxController { void setSourceLanguage(String language) { if (sourceLanguage.value != language) { stopAll(); - + sourceLanguage.value = language; - + final asrCode = _languageManager.getAsrCodeByChineseName(language); if (asrCode != null) { sourceLanguageCode.value = asrCode; } - + // 保存语言设置 _saveLanguageSettings(); - + // 更新ASR支持的语言并重新初始化 _reinitializeAsrService(); } @@ -515,35 +603,38 @@ class TranslationController extends GetxController { void setTargetLanguage(String language) { if (targetLanguage.value != language) { stopAll(); - + targetLanguage.value = language; - + final asrCode = _languageManager.getAsrCodeByChineseName(language); if (asrCode != null) { targetLanguageCode.value = asrCode; } - + // 保存语言设置 _saveLanguageSettings(); - + // 更新ASR支持的语言并重新初始化 _reinitializeAsrService(); } } - + // 重新初始化ASR服务以支持新的语言 Future _reinitializeAsrService() async { try { // 更新ASR支持的语言 - final List asrSupportedLanguages = [sourceLanguageCode.value, targetLanguageCode.value]; - + final List asrSupportedLanguages = [ + sourceLanguageCode.value, + targetLanguageCode.value + ]; + Logger.info('重新初始化ASR服务，支持语言: $asrSupportedLanguages'); - + // 重新初始化ASR服务 await _asrService.initialize( supportedLanguages: asrSupportedLanguages, ); - + Logger.info('ASR服务重新初始化成功'); } catch (e) { Logger.error('重新初始化ASR服务失败: $e'); @@ -556,15 +647,17 @@ class TranslationController extends GetxController { if (historyJson != null) { try { final List historyList = jsonDecode(historyJson); - final List allHistory = historyList.map((item) => TranslationItem.fromJson(item)).toList(); - + final List allHistory = + historyList.map((item) => TranslationItem.fromJson(item)).toList(); + // 只保留最新的10条记录 if (allHistory.length > _maxHistoryItems) { - translationHistory.value = allHistory.sublist(allHistory.length - _maxHistoryItems); + translationHistory.value = + allHistory.sublist(allHistory.length - _maxHistoryItems); } else { translationHistory.value = allHistory; } - + // 使用微任务确保在视图完全渲染后滚动到底部 WidgetsBinding.instance.addPostFrameCallback((_) { _scrollToBottom(); @@ -579,11 +672,13 @@ class TranslationController extends GetxController { // 保存翻译历史 void saveTranslationHistory() { if (translationHistory.length > _maxHistoryItems) { - translationHistory.removeRange(0, translationHistory.length - _maxHistoryItems); + translationHistory.removeRange( + 0, translationHistory.length - _maxHistoryItems); } try { - final List> historyJson = translationHistory.map((item) => item.toJson()).toList(); + final List> historyJson = + translationHistory.map((item) => item.toJson()).toList(); _storage.write(_historyKey, jsonEncode(historyJson)); } catch (e) { Logger.error('保存翻译历史失败: $e'); @@ -644,7 +739,8 @@ class TranslationItem { sourceLanguageCode: json['sourceLanguageCode'] ?? 'zh-CN', targetLanguageCode: json['targetLanguageCode'] ?? 'en-US', timestamp: timestamp, - sessionId: json['sessionId'] ?? DateTime.now().millisecondsSinceEpoch.toString(), + sessionId: + json['sessionId'] ?? DateTime.now().millisecondsSinceEpoch.toString(), isFirstInSession: json['isFirstInSession'] ?? false, isIntermediate: false, // 从存储加载的都是最终结果 ); @@ -662,16 +758,18 @@ class TranslationItem { // 不保存isIntermediate，因为存储的都是最终结果 }; } - + // 获取源语言的中文名称 String get sourceLanguageName { final languageManager = Get.find(); - return languageManager.getChineseNameByAsrCode(sourceLanguageCode) ?? '未知语言'; + return languageManager.getChineseNameByAsrCode(sourceLanguageCode) ?? + '未知语言'; } - + // 获取目标语言的中文名称 String get targetLanguageName { final languageManager = Get.find(); - return languageManager.getChineseNameByAsrCode(targetLanguageCode) ?? '未知语言'; + return languageManager.getChineseNameByAsrCode(targetLanguageCode) ?? + '未知语言'; } } diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt index 62ad5685c..26595d9cd 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt @@ -28,7 +28,7 @@ class AzureAsrHelper(private val context: Context) { private var supportedLanguages = arrayOf("zh-CN") // 是否使用回音消除 - 内部控制常量 - private val useEchoCancellation = false + private val useEchoCancellation = true // 自定义音频处理相关 private var customAudioProcessor: CustomAudioProcessor? = null @@ -36,6 +36,7 @@ class AzureAsrHelper(private val context: Context) { private var audioConfig: AudioConfig? = null // 公共方法：允许外部写入音频数据 fun pushAudioData(data: ByteArray) { + FileLogger.d(TAG, "pushAudioData,data=${data},pushStream=${pushStream}") pushStream?.write(data) // 使用 SDK 的正确方法（如 push） } @@ -85,6 +86,7 @@ class AzureAsrHelper(private val context: Context) { // 创建识别器 try { if (useEchoCancellation) { + FileLogger.d(TAG, "如果使用回音消除，创建自定义音频输入流") // 如果使用回音消除，创建自定义音频输入流 setupCustomAudioProcessing() @@ -95,13 +97,13 @@ class AzureAsrHelper(private val context: Context) { recognizer = SpeechRecognizer(speechConfig, audioConfig) } } else { - // 使用默认麦克风输入 - if (isAutoDetectLanguage) { - val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) - recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) - } else { - recognizer = SpeechRecognizer(speechConfig) - } + // // 使用默认麦克风输入 + // if (isAutoDetectLanguage) { + // val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) + // recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) + // } else { + // recognizer = SpeechRecognizer(speechConfig) + // } } FileLogger.d(TAG, "Azure 语音服务初始化成功") @@ -140,13 +142,13 @@ class AzureAsrHelper(private val context: Context) { recognizer = SpeechRecognizer(speechConfig, audioConfig) } } else { - // 使用默认麦克风输入 - if (isAutoDetectLanguage) { - val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) - recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) - } else { - recognizer = SpeechRecognizer(speechConfig) - } + // // 使用默认麦克风输入 + // if (isAutoDetectLanguage) { + // val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) + // recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) + // } else { + // recognizer = SpeechRecognizer(speechConfig) + // } } return true } else { diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt index 090a72efc..e809403ea 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt @@ -4,6 +4,7 @@ import android.content.Context import android.app.Activity import android.os.Handler import android.os.Looper +import android.util.Log import androidx.annotation.NonNull import com.yunqiinnovation.azure_speech.utils.FileLogger @@ -91,6 +92,21 @@ class AzureSpeechPlugin: FlutterPlugin { } catch (e: Exception) { result.error("INITIALIZATION_ERROR", e.message, null) } + } + "pushAudioData" -> { + + val audioData = call.argument("AudioData") + +try { + Log.d("","AsrMethodHandler,pushAudioData${audioData}") + if (audioData != null) { + + val success = azureAsrHelper.pushAudioData(audioData) + result.success(success) + } +} catch (e: Exception) { + result.error("pushAudioData_ERROR", e.message, null) +} } "recognizeOnce" -> { azureAsrHelper.recognizeOnce(object : AzureAsrHelper.RecognizeCallback { diff --git a/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt b/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt index e2f474564..f09c69f50 100644 --- a/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt +++ b/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt @@ -92,7 +92,7 @@ class JieliPlugin: FlutterPlugin, MethodCallHandler, ActivityAware, EventChannel override fun onAudioDataReceived(data: ByteArray) { pluginScope.launch(Dispatchers.Main) { - Log.d("","数据流onAudioDataReceived${data}") + // Log.d("","数据流onAudioDataReceived${data}") sendEvent(EVENT_TYPE_AUDIO_DATA, data) } } diff --git a/local_plugins/jieli/lib/jieli.dart b/local_plugins/jieli/lib/jieli.dart index 50e7d99c2..d0d318627 100644 --- a/local_plugins/jieli/lib/jieli.dart +++ b/local_plugins/jieli/lib/jieli.dart @@ -127,7 +127,7 @@ class Jieli { break; case EventType.audioData: if (data is Uint8List) { - print('收到ble通道消息: $data'); + //print('收到ble通道消息: $data'); _audioController.add(data); } break; diff --git a/local_plugins/jl_opus/android/src/main/kotlin/com/yunqiinnovation/jl_opus/JlOpusPlugin.kt b/local_plugins/jl_opus/android/src/main/kotlin/com/yunqiinnovation/jl_opus/JlOpusPlugin.kt index bf47b2917..e8c6db4c9 100644 --- a/local_plugins/jl_opus/android/src/main/kotlin/com/yunqiinnovation/jl_opus/JlOpusPlugin.kt +++ b/local_plugins/jl_opus/android/src/main/kotlin/com/yunqiinnovation/jl_opus/JlOpusPlugin.kt @@ -324,7 +324,7 @@ class JlOpusPlugin: FlutterPlugin, MethodCallHandler { opusManager?.startDecodeStream(option, object : OnDecodeStreamCallback { override fun onDecodeStream(data: ByteArray?) { if (data != null) { - Log.d(TAG, "onDecodeStream${data}") + //Log.d(TAG, "onDecodeStream${data}") callback.onAudioDataReceived1(data) sendEventOnMainThread(mapOf( "event" to "onDecodeStream", From 1a9cfe3fcfe62a4cf6657aeca9c3f160c3eb9e51 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Thu, 24 Apr 2025 11:07:44 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../deepsound/VoiceInteractionHandler.kt | 12 ++++++++++++ .../buildOutputCleanup/buildOutputCleanup.lock | Bin 17 -> 17 bytes .../.gradle/buildOutputCleanup/cache.properties | 4 ++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt index f0a7d6d86..c31e6c319 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt @@ -13,6 +13,7 @@ import com.yunqiinnovation.azure_speech.AzureTtsHelper import com.yunqiinnovation.open_ai_service.OpenAIService import com.yunqiinnovation.open_ai_service.SystemFunctionHandler import de.minimalme.spotify_sdk.SpotifySdkHelper +import kotlinx.serialization.json.JsonObject /** @@ -274,6 +275,17 @@ class VoiceInteractionHandler( override fun onFunctionCall(call: JSONObject) { FileLogger.d(TAG, "processWithOpenAI 收到函数调用请求: ${call.getString("name")}") +// Log.e("OpenAIServicePlugin", "context:${call.toString()}") +// val context = call["context"] +// if (call.has("meta")) { +// val meta = call.get("meta") +// val metaJsonObject: JsonObject = meta as JsonObject +// Log.e("OpenAIServicePlugin", "工具调用结果 context:${context} meta:${meta.toString()}") +// if (metaJsonObject.containsKey("card_spotify")) { +// val card_spotify:JsonObject = metaJsonObject.get("card_spotify") as JsonObject +// playSpotifyMusic(card_spotify.get("name") as String) +// } +// } } } ) diff --git a/local_plugins/opus/.gradle/buildOutputCleanup/buildOutputCleanup.lock b/local_plugins/opus/.gradle/buildOutputCleanup/buildOutputCleanup.lock index fd40a1b74af7568dfb82c03afb8da2f181bd8ea4..a2d8b75c89b15fd04945a85eb2b33d6443b8fcc1 100644 GIT binary patch literal 17 TcmZQ}Dp$CADo{Lz0Ror-DFy>W literal 17 TcmZQ}Dp$CADo{Lz0Ror+DFp*U diff --git a/local_plugins/opus/.gradle/buildOutputCleanup/cache.properties b/local_plugins/opus/.gradle/buildOutputCleanup/cache.properties index 8a9b74646..816b483f3 100644 --- a/local_plugins/opus/.gradle/buildOutputCleanup/cache.properties +++ b/local_plugins/opus/.gradle/buildOutputCleanup/cache.properties @@ -1,2 +1,2 @@ -#Tue Apr 15 09:36:50 IST 2025 -gradle.version=8.10 +#Thu Apr 24 10:52:53 CST 2025 +gradle.version=8.9