From c8db38692b18963f9fed3f0a3153738645d73d39 Mon Sep 17 00:00:00 2001 From: wolfplus Date: Tue, 25 Feb 2025 10:57:35 +0000 Subject: [PATCH] add --- android/app/build.gradle.kts | 4 + android/app/src/main/AndroidManifest.xml | 6 +- .../com/example/deep_voice/MainActivity.kt | 19 +- .../example/deep_voice/VolcSpeechBridge.kt | 293 ++++++++++++++++++ android/settings.gradle.kts | 12 + lib/core/bindings/initial_binding.dart | 2 + lib/core/routes/app_pages.dart | 5 + lib/core/routes/app_routes.dart | 1 + .../services/voice_recognition_service.dart | 264 ++++++++++++++++ lib/modules/speech_demo/speech_demo_page.dart | 137 ++++++++ 10 files changed, 740 insertions(+), 3 deletions(-) create mode 100644 android/app/src/main/kotlin/com/example/deep_voice/VolcSpeechBridge.kt create mode 100644 lib/data/services/voice_recognition_service.dart create mode 100644 lib/modules/speech_demo/speech_demo_page.dart diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 7e5007083..f28f0f64a 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -1,3 +1,5 @@ + + plugins { id("com.android.application") id("kotlin-android") @@ -43,6 +45,8 @@ android { dependencies { coreLibraryDesugaring("com.android.tools:desugar_jdk_libs:2.0.4") + // 火山语音识别 SDK 依赖 + implementation("com.bytedance.speechengine:speechengine_asr_tob:1.1.6") } flutter { diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index e9093900d..f46ae0c52 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -1,4 +1,5 @@ - + @@ -12,7 +13,8 @@ + android:icon="@mipmap/ic_launcher" + tools:replace="android:label"> + when (call.method) { + "initialize" -> { + val appKey = call.argument("appKey") ?: "" + val accessKey = call.argument("accessKey") ?: "" + initialize(appKey, accessKey, result) + } + "startRecognition" -> { + startRecognition(result) + } + "stopRecognition" -> { + stopRecognition(result) + } + "isInitialized" -> { + result.success(isInitialized) + } + else -> { + result.notImplemented() + } + } + } + + // 设置 EventChannel 处理事件流 + EventChannel(messenger, EVENT_CHANNEL_NAME).setStreamHandler(object : EventChannel.StreamHandler { + override fun onListen(arguments: Any?, events: EventChannel.EventSink?) { + eventSink = events + } + + override fun onCancel(arguments: Any?) { + eventSink = null + } + }) + } + + /** + * 初始化火山语音识别引擎 + */ + fun initialize(appKey: String, accessKey: String, result: Result) { + methodResult = result + try { + Log.i(TAG, "Initializing Volcengine Speech Engine") + + // 创建引擎实例 + engine = SpeechEngineGenerator.getInstance() + engine?.createEngine() + + // 设置监听器 + engine?.setListener(this) + + // 设置应用 ID 和访问密钥 + engine?.setOptionString("app_id", appKey) + engine?.setOptionString("access_key", accessKey) + + // 设置语音识别参数 + engine?.setOptionBoolean("enable_punctuation", true) // 启用标点符号 + engine?.setOptionBoolean("enable_intermediate_result", true) // 启用中间结果 + engine?.setOptionBoolean("enable_vad", true) // 启用语音活动检测 + + // 设置语言（普通话） + engine?.setOptionString("language", "zh") + + // 初始化引擎 + val ret = engine?.initEngine() + if (ret == SpeechEngineDefines.ERR_NO_ERROR) { + isInitialized = true + Log.i(TAG, "Volcengine Speech Engine initialized successfully") + result.success("Engine initialized") + + // 发送状态更新到 Flutter + sendEvent("status", "initialized") + } else { + Log.e(TAG, "Engine initialization failed with error code: $ret") + result.error("INIT_ERROR", "Engine initialization failed with error code: $ret", null) + } + } catch (e: Exception) { + Log.e(TAG, "Initialization failed", e) + result.error("INIT_ERROR", e.message, null) + } + } + + /** + * 启动语音识别 + */ + fun startRecognition(result: Result) { + try { + if (!isInitialized) { + result.error("NOT_INITIALIZED", "Speech engine is not initialized", null) + return + } + + Log.i(TAG, "Starting speech recognition") + val ret = engine?.sendDirective(DIRECTIVE_START_RECOGNITION, "") + + if (ret == SpeechEngineDefines.ERR_NO_ERROR) { + result.success("Recognition started") + // 发送状态更新到 Flutter + sendEvent("status", "listening") + } else { + Log.e(TAG, "Start recognition failed with error code: $ret") + result.error("START_ERROR", "Start recognition failed with error code: $ret", null) + } + } catch (e: Exception) { + Log.e(TAG, "Start recognition failed", e) + result.error("START_ERROR", e.message, null) + } + } + + /** + * 停止语音识别 + */ + fun stopRecognition(result: Result) { + try { + if (!isInitialized) { + result.error("NOT_INITIALIZED", "Speech engine is not initialized", null) + return + } + + Log.i(TAG, "Stopping speech recognition") + val ret = engine?.sendDirective(DIRECTIVE_STOP_RECOGNITION, "") + + if (ret == SpeechEngineDefines.ERR_NO_ERROR) { + result.success("Recognition stopped") + // 发送状态更新到 Flutter + sendEvent("status", "idle") + } else { + Log.e(TAG, "Stop recognition failed with error code: $ret") + result.error("STOP_ERROR", "Stop recognition failed with error code: $ret", null) + } + } catch (e: Exception) { + Log.e(TAG, "Stop recognition failed", e) + result.error("STOP_ERROR", e.message, null) + } + } + + /** + * 释放资源 + */ + fun dispose() { + try { + engine?.destroyEngine() + engine = null + isInitialized = false + Log.i(TAG, "Volcengine Speech Engine released") + } catch (e: Exception) { + Log.e(TAG, "Error releasing engine", e) + } + } + + // ------------------ + // 发送事件到 Flutter + // ------------------ + + private fun sendEvent(eventName: String, data: Any) { + try { + val eventData = JSONObject().apply { + put("event", eventName) + put("data", data) + } + + eventSink?.success(eventData.toString()) + } catch (e: Exception) { + Log.e(TAG, "Error sending event", e) + } + } + + // ------------------ + // SpeechListener 接口回调 + // ------------------ + + /** + * 处理语音消息回调 + * 这是 SpeechListener 接口的必须实现方法 + */ + override fun onSpeechMessage(messageType: Int, messageData: ByteArray?, messageLength: Int) { + try { + when (messageType) { + // 初始化完成消息 + 1 -> { + Log.i(TAG, "Engine initialized successfully") + isInitialized = true + + // 通知 Flutter 初始化完成 + methodResult?.success("Engine initialized") + methodResult = null + + // 发送状态更新到 Flutter + sendEvent("status", "initialized") + } + + // 错误消息 + 2 -> { + val errorCode = messageData?.let { String(it, 0, messageLength).toIntOrNull() } ?: -1 + Log.e(TAG, "Engine error: $errorCode") + + // 通知 Flutter 识别出错 + val errorData = JSONObject().apply { + put("code", errorCode) + put("message", "Error code: $errorCode") + } + + sendEvent("error", errorData.toString()) + } + + // VAD 状态变化消息 + 3 -> { + val state = messageData?.let { String(it, 0, messageLength).toIntOrNull() } ?: -1 + Log.i(TAG, "VAD state changed: $state") + + val vadState = when (state) { + 0 -> "silence" + 1 -> "speech" + else -> "unknown" + } + + sendEvent("vad", vadState) + } + + // 识别结果消息 + 4 -> { + messageData?.let { + val resultJson = String(it, 0, messageLength) + try { + val jsonObject = JSONObject(resultJson) + val text = jsonObject.optString("text", "") + val isFinal = jsonObject.optBoolean("is_final", false) + + Log.i(TAG, "Recognition result: $text, isFinal: $isFinal") + + // 发送识别结果到 Flutter + val resultData = JSONObject().apply { + put("text", text) + put("isFinal", isFinal) + } + + sendEvent("result", resultData.toString()) + } catch (e: Exception) { + Log.e(TAG, "Error parsing result JSON", e) + } + } + } + + else -> { + Log.d(TAG, "Received unknown message type: $messageType") + } + } + } catch (e: Exception) { + Log.e(TAG, "Error processing speech message", e) + } + } +} \ No newline at end of file diff --git a/android/settings.gradle.kts b/android/settings.gradle.kts index a439442c2..c1c2fa423 100644 --- a/android/settings.gradle.kts +++ b/android/settings.gradle.kts @@ -12,6 +12,9 @@ pluginManagement { repositories { google() mavenCentral() + maven { + url = uri("https://artifact.bytedance.com/repository/Volcengine/") + } gradlePluginPortal() } } @@ -22,4 +25,13 @@ plugins { id("org.jetbrains.kotlin.android") version "1.8.22" apply false } +dependencyResolutionManagement { + repositories { + google() + mavenCentral() + maven { url = uri("https://artifact.bytedance.com/repository/Volcengine/") } + gradlePluginPortal() + } +} + include(":app") diff --git a/lib/core/bindings/initial_binding.dart b/lib/core/bindings/initial_binding.dart index 89e29b78f..296bb43f0 100644 --- a/lib/core/bindings/initial_binding.dart +++ b/lib/core/bindings/initial_binding.dart @@ -4,6 +4,7 @@ import '../../data/services/volcano_tts_service.dart'; import '../../data/services/audio_service.dart'; import '../../data/services/notification_service.dart'; import '../../data/services/volcano_ai_service.dart'; +import '../../data/services/voice_recognition_service.dart'; /// 初始绑定，用于管理全局依赖 class InitialBinding extends Bindings { @@ -14,6 +15,7 @@ class InitialBinding extends Bindings { final volcanoTtsService = Get.put(VolcanoTtsService(), permanent: true); final audioService = Get.put(AudioServiceManager(), permanent: true); final volcanoAiService = Get.put(VolcanoAIService(), permanent: true); + final voiceRecognitionService = Get.put(VoiceRecognitionService(), permanent: true); // 只注册全局控制器 Get.put(PermissionController(), permanent: true); diff --git a/lib/core/routes/app_pages.dart b/lib/core/routes/app_pages.dart index f2200a5da..43c181749 100644 --- a/lib/core/routes/app_pages.dart +++ b/lib/core/routes/app_pages.dart @@ -7,6 +7,7 @@ import '../../modules/profile/views/profile_view.dart'; import '../../modules/profile/bindings/profile_binding.dart'; import '../../modules/chat/bindings/chat_binding.dart'; import '../../modules/chat/views/chat_view.dart'; +import '../../modules/speech_demo/speech_demo_page.dart'; import './app_routes.dart'; abstract class AppPages { @@ -31,5 +32,9 @@ abstract class AppPages { page: () => const ProfileView(), binding: ProfileBinding(), ), + GetPage( + name: Routes.speechDemo, + page: () => const SpeechDemoPage(), + ), ]; } \ No newline at end of file diff --git a/lib/core/routes/app_routes.dart b/lib/core/routes/app_routes.dart index 1616e7dd1..c224a2cbe 100644 --- a/lib/core/routes/app_routes.dart +++ b/lib/core/routes/app_routes.dart @@ -4,4 +4,5 @@ abstract class Routes { static const explore = '/explore'; static const chat = '/chat'; static const profile = '/profile'; + static const speechDemo = '/speech_demo'; } \ No newline at end of file diff --git a/lib/data/services/voice_recognition_service.dart b/lib/data/services/voice_recognition_service.dart new file mode 100644 index 000000000..35024f5fb --- /dev/null +++ b/lib/data/services/voice_recognition_service.dart @@ -0,0 +1,264 @@ +import 'dart:async'; +import 'dart:convert'; +import 'package:flutter/services.dart'; +import 'package:get/get.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_dotenv/flutter_dotenv.dart'; + +/// 火山语音识别服务 +/// 负责与原生层的火山语音识别引擎交互 +class VoiceRecognitionService extends GetxService { + // 方法通道，用于调用原生方法 + static const MethodChannel _methodChannel = + MethodChannel('com.example.deep_voice/speech_recognition'); + + // 事件通道，用于接收原生层的事件 + static const EventChannel _eventChannel = + EventChannel('com.example.deep_voice/speech_recognition_events'); + + // 状态变量 + final isInitialized = false.obs; + final isListening = false.obs; + final isConnecting = true.obs; + + // 识别结果 + final recognizedText = ''.obs; + + // 回调函数 + Function(String)? onRecognitionResult; + Function(String)? onError; + Function()? onRecognitionComplete; + Function(String)? onConnectionStatusChanged; + Function(String)? onSentenceComplete; + + // 事件流订阅 + StreamSubscription? _eventSubscription; + + // 火山语音识别的 AppKey 和 AccessKey + late final String _appKey; + late final String _accessKey; + + @override + void onInit() { + super.onInit(); + _loadCredentials(); + _setupEventListener(); + } + + /// 加载火山语音识别的凭证 + void _loadCredentials() { + try { + _appKey = dotenv.env['VOLC_SPEECH_APP_KEY'] ?? ''; + _accessKey = dotenv.env['VOLC_SPEECH_ACCESS_KEY'] ?? ''; + + if (_appKey.isEmpty || _accessKey.isEmpty) { + debugPrint('警告: 火山语音识别凭证未设置，请在 .env 文件中设置 VOLC_SPEECH_APP_KEY 和 VOLC_SPEECH_ACCESS_KEY'); + } + } catch (e) { + debugPrint('加载火山语音识别凭证失败: $e'); + } + } + + /// 设置事件监听器 + void _setupEventListener() { + try { + _eventSubscription = _eventChannel.receiveBroadcastStream().listen( + (dynamic event) { + _handleEvent(event.toString()); + }, + onError: (dynamic error) { + debugPrint('事件通道错误: $error'); + if (onError != null) { + onError!('事件通道错误: $error'); + } + }, + ); + } catch (e) { + debugPrint('设置事件监听器失败: $e'); + } + } + + /// 处理来自原生层的事件 + void _handleEvent(String eventJson) { + try { + final eventData = jsonDecode(eventJson); + final eventName = eventData['event']; + final data = eventData['data']; + + switch (eventName) { + case 'status': + _handleStatusEvent(data.toString()); + break; + case 'result': + _handleResultEvent(data.toString()); + break; + case 'error': + _handleErrorEvent(data.toString()); + break; + case 'vad': + _handleVadEvent(data.toString()); + break; + } + } catch (e) { + debugPrint('处理事件失败: $e'); + } + } + + /// 处理状态事件 + void _handleStatusEvent(String status) { + switch (status) { + case 'initialized': + isInitialized.value = true; + isConnecting.value = false; + break; + case 'listening': + isListening.value = true; + break; + case 'idle': + isListening.value = false; + break; + } + + if (onConnectionStatusChanged != null) { + onConnectionStatusChanged!(status); + } + } + + /// 处理识别结果事件 + void _handleResultEvent(String resultJson) { + try { + final resultData = jsonDecode(resultJson); + final text = resultData['text']; + final isFinal = resultData['isFinal']; + + // 更新识别文本 + recognizedText.value = text; + + // 调用回调函数 + if (onRecognitionResult != null) { + onRecognitionResult!(text); + } + + // 如果是最终结果，调用完成回调 + if (isFinal && text.isNotEmpty) { + if (onSentenceComplete != null) { + onSentenceComplete!(text); + } + + if (onRecognitionComplete != null) { + onRecognitionComplete!(); + } + } + } catch (e) { + debugPrint('处理识别结果失败: $e'); + } + } + + /// 处理错误事件 + void _handleErrorEvent(String errorJson) { + try { + final errorData = jsonDecode(errorJson); + final errorMessage = errorData['message']; + + debugPrint('语音识别错误: $errorMessage'); + + if (onError != null) { + onError!(errorMessage); + } + } catch (e) { + debugPrint('处理错误事件失败: $e'); + } + } + + /// 处理 VAD 事件 + void _handleVadEvent(String vadState) { + // 可以根据需要处理 VAD 状态变化 + debugPrint('VAD 状态: $vadState'); + } + + /// 初始化语音识别引擎 + Future initialize() async { + try { + isConnecting.value = true; + + // 检查凭证 + if (_appKey.isEmpty || _accessKey.isEmpty) { + throw '火山语音识别凭证未设置'; + } + + // 调用原生方法初始化引擎 + final result = await _methodChannel.invokeMethod( + 'initialize', + { + 'appKey': _appKey, + 'accessKey': _accessKey, + }, + ); + + debugPrint('初始化结果: $result'); + + // 初始化成功后，状态会通过事件通道更新 + } catch (e) { + isConnecting.value = false; + debugPrint('初始化语音识别引擎失败: $e'); + if (onError != null) { + onError!('初始化失败: $e'); + } + rethrow; + } + } + + /// 开始语音识别 + Future startRecognition() async { + try { + // 检查是否已初始化 + if (!isInitialized.value) { + await initialize(); + } + + // 调用原生方法开始识别 + final result = await _methodChannel.invokeMethod('startRecognition'); + debugPrint('开始识别结果: $result'); + + // 状态会通过事件通道更新 + } catch (e) { + debugPrint('开始语音识别失败: $e'); + if (onError != null) { + onError!('开始识别失败: $e'); + } + rethrow; + } + } + + /// 停止语音识别 + Future stopRecognition() async { + try { + // 检查是否已初始化 + if (!isInitialized.value) { + return; + } + + // 调用原生方法停止识别 + final result = await _methodChannel.invokeMethod('stopRecognition'); + debugPrint('停止识别结果: $result'); + + // 状态会通过事件通道更新 + } catch (e) { + debugPrint('停止语音识别失败: $e'); + if (onError != null) { + onError!('停止识别失败: $e'); + } + } + } + + @override + void onClose() { + // 取消事件订阅 + _eventSubscription?.cancel(); + + // 停止识别 + stopRecognition(); + + super.onClose(); + } +} \ No newline at end of file diff --git a/lib/modules/speech_demo/speech_demo_page.dart b/lib/modules/speech_demo/speech_demo_page.dart new file mode 100644 index 000000000..473265d81 --- /dev/null +++ b/lib/modules/speech_demo/speech_demo_page.dart @@ -0,0 +1,137 @@ +import 'package:flutter/material.dart'; +import 'package:get/get.dart'; +import '../../data/services/voice_recognition_service.dart'; + +class SpeechDemoPage extends StatefulWidget { + const SpeechDemoPage({Key? key}) : super(key: key); + + @override + _SpeechDemoPageState createState() => _SpeechDemoPageState(); +} + +class _SpeechDemoPageState extends State { + final VoiceRecognitionService _recognitionService = Get.find(); + + @override + void initState() { + super.initState(); + _initializeEngine(); + } + + Future _initializeEngine() async { + try { + await _recognitionService.initialize(); + } catch (e) { + if (mounted) { + ScaffoldMessenger.of(context).showSnackBar( + SnackBar(content: Text('初始化失败: $e')), + ); + } + } + } + + Future _startRecognition() async { + try { + await _recognitionService.startRecognition(); + } catch (e) { + if (mounted) { + ScaffoldMessenger.of(context).showSnackBar( + SnackBar(content: Text('启动识别失败: $e')), + ); + } + } + } + + Future _stopRecognition() async { + try { + await _recognitionService.stopRecognition(); + } catch (e) { + if (mounted) { + ScaffoldMessenger.of(context).showSnackBar( + SnackBar(content: Text('停止识别失败: $e')), + ); + } + } + } + + @override + Widget build(BuildContext context) { + return Scaffold( + appBar: AppBar( + title: const Text('火山语音识别演示'), + ), + body: Center( + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + // 状态显示 + Obx(() { + String status = '准备中'; + if (_recognitionService.isConnecting.value) { + status = '连接中...'; + } else if (_recognitionService.isInitialized.value) { + status = _recognitionService.isListening.value ? '正在听...' : '就绪'; + } + return Text('状态: $status', style: const TextStyle(fontSize: 18)); + }), + + const SizedBox(height: 20), + + // 识别结果显示 + Padding( + padding: const EdgeInsets.all(16.0), + child: Container( + width: double.infinity, + height: 200, + decoration: BoxDecoration( + border: Border.all(color: Colors.grey), + borderRadius: BorderRadius.circular(8), + ), + padding: const EdgeInsets.all(16), + child: Obx(() { + return SingleChildScrollView( + child: Text( + _recognitionService.recognizedText.value, + style: const TextStyle(fontSize: 16), + ), + ); + }), + ), + ), + + const SizedBox(height: 20), + + // 控制按钮 + Row( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Obx(() { + return ElevatedButton( + onPressed: _recognitionService.isInitialized.value && !_recognitionService.isListening.value + ? _startRecognition + : null, + child: const Text('开始识别'), + ); + }), + + const SizedBox(width: 20), + + Obx(() { + return ElevatedButton( + onPressed: _recognitionService.isListening.value + ? _stopRecognition + : null, + style: ElevatedButton.styleFrom( + backgroundColor: Colors.red, + ), + child: const Text('停止识别'), + ); + }), + ], + ), + ], + ), + ), + ); + } +} \ No newline at end of file