From 4f18a6426d7d9ab61c2c8fc929e731467e206eb5 Mon Sep 17 00:00:00 2001 From: wolfplus Date: Thu, 20 Mar 2025 15:23:53 +0000 Subject: [PATCH] deplugin --- ios/Podfile.lock | 7 - ios/Runner.xcodeproj/project.pbxproj | 8 + ios/Runner/AppDelegate.swift | 370 +++++++++++++-- ios/Runner/AzureAsrHelper.swift | 408 +++++++++++++++++ ios/Runner/AzureTtsHelper.swift | 427 ++++++++++++++++++ ios/Runner/BluetoothMediaButtonHelper.swift | 15 +- lib/data/models/agent.dart | 4 + lib/data/providers/agent_provider.dart | 3 +- .../bluetooth_media_button_service.dart | 96 ++-- .../services/classic_bluetooth_service.dart | 7 +- .../speech_impl/azure_asr_service.dart | 38 +- .../chat/controllers/chat_controller.dart | 85 +++- .../home/controllers/home_controller.dart | 64 +-- .../controllers/pairing_controller.dart | 17 +- pubspec.yaml | 2 - 15 files changed, 1342 insertions(+), 209 deletions(-) create mode 100644 ios/Runner/AzureAsrHelper.swift create mode 100644 ios/Runner/AzureTtsHelper.swift diff --git a/ios/Podfile.lock b/ios/Podfile.lock index 8296de8a7..7ee0cf268 100644 --- a/ios/Podfile.lock +++ b/ios/Podfile.lock @@ -8,9 +8,6 @@ PODS: - Flutter - audio_waveforms (0.0.1): - Flutter - - azure_speech_recognition (0.1.0): - - Flutter - - MicrosoftCognitiveServicesSpeech-iOS (~> 1.34.0) - device_info_plus (0.0.1): - Flutter - Flutter (1.0.0) @@ -50,7 +47,6 @@ DEPENDENCIES: - audio_service (from `.symlinks/plugins/audio_service/darwin`) - audio_session (from `.symlinks/plugins/audio_session/ios`) - audio_waveforms (from `.symlinks/plugins/audio_waveforms/ios`) - - azure_speech_recognition (from `.symlinks/plugins/azure_speech_recognition/ios`) - device_info_plus (from `.symlinks/plugins/device_info_plus/ios`) - Flutter (from `Flutter`) - flutter_blue_plus_darwin (from `.symlinks/plugins/flutter_blue_plus_darwin/darwin`) @@ -80,8 +76,6 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/audio_session/ios" audio_waveforms: :path: ".symlinks/plugins/audio_waveforms/ios" - azure_speech_recognition: - :path: ".symlinks/plugins/azure_speech_recognition/ios" device_info_plus: :path: ".symlinks/plugins/device_info_plus/ios" Flutter: @@ -114,7 +108,6 @@ SPEC CHECKSUMS: audio_service: aa99a6ba2ae7565996015322b0bb024e1d25c6fd audio_session: 9bb7f6c970f21241b19f5a3658097ae459681ba0 audio_waveforms: a6dde7fe7c0ea05f06ffbdb0f7c1b2b2ba6cedcf - azure_speech_recognition: 063bf4f294dbabaf2741f529f76132b2c38b744b device_info_plus: 71ffc6ab7634ade6267c7a93088ed7e4f74e5896 Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 flutter_blue_plus_darwin: 09444a26fb6bdef523e55b68fc1c59af5a877ea6 diff --git a/ios/Runner.xcodeproj/project.pbxproj b/ios/Runner.xcodeproj/project.pbxproj index 88a2148e0..626e05a1b 100644 --- a/ios/Runner.xcodeproj/project.pbxproj +++ b/ios/Runner.xcodeproj/project.pbxproj @@ -11,6 +11,8 @@ 0A5EDC042D8B544100215E26 /* BluetoothMediaButtonHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0A5EDC032D8B544100215E26 /* BluetoothMediaButtonHelper.swift */; }; 0ADE29692D8C25660032BFCB /* silence.mp3 in Resources */ = {isa = PBXBuildFile; fileRef = 0ADE29682D8C25660032BFCB /* silence.mp3 */; }; 0ADE296A2D8C25660032BFCB /* sample.mp3 in Resources */ = {isa = PBXBuildFile; fileRef = 0ADE29672D8C25660032BFCB /* sample.mp3 */; }; + 0ADE296D2D8C66D60032BFCB /* AzureTtsHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0ADE296C2D8C66D60032BFCB /* AzureTtsHelper.swift */; }; + 0ADE296E2D8C66D60032BFCB /* AzureAsrHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0ADE296B2D8C66D60032BFCB /* AzureAsrHelper.swift */; }; 1498D2341E8E89220040F4C2 /* GeneratedPluginRegistrant.m in Sources */ = {isa = PBXBuildFile; fileRef = 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */; }; 331C808B294A63AB00263BE5 /* RunnerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 331C807B294A618700263BE5 /* RunnerTests.swift */; }; 3406320CAE3314A73EF78043 /* Pods_RunnerTests.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5BBE10AD7954527B6BB12CFB /* Pods_RunnerTests.framework */; }; @@ -52,6 +54,8 @@ 0A5EDC032D8B544100215E26 /* BluetoothMediaButtonHelper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BluetoothMediaButtonHelper.swift; sourceTree = ""; }; 0ADE29672D8C25660032BFCB /* sample.mp3 */ = {isa = PBXFileReference; lastKnownFileType = audio.mp3; path = sample.mp3; sourceTree = ""; }; 0ADE29682D8C25660032BFCB /* silence.mp3 */ = {isa = PBXFileReference; lastKnownFileType = audio.mp3; path = silence.mp3; sourceTree = ""; }; + 0ADE296B2D8C66D60032BFCB /* AzureAsrHelper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AzureAsrHelper.swift; sourceTree = ""; }; + 0ADE296C2D8C66D60032BFCB /* AzureTtsHelper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AzureTtsHelper.swift; sourceTree = ""; }; 1498D2321E8E86230040F4C2 /* GeneratedPluginRegistrant.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GeneratedPluginRegistrant.h; sourceTree = ""; }; 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GeneratedPluginRegistrant.m; sourceTree = ""; }; 331C807B294A618700263BE5 /* RunnerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RunnerTests.swift; sourceTree = ""; }; @@ -151,6 +155,8 @@ 97C146F01CF9000F007C117D /* Runner */ = { isa = PBXGroup; children = ( + 0ADE296B2D8C66D60032BFCB /* AzureAsrHelper.swift */, + 0ADE296C2D8C66D60032BFCB /* AzureTtsHelper.swift */, 0ADE29672D8C25660032BFCB /* sample.mp3 */, 0ADE29682D8C25660032BFCB /* silence.mp3 */, 0A5EDC032D8B544100215E26 /* BluetoothMediaButtonHelper.swift */, @@ -408,6 +414,8 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 0ADE296D2D8C66D60032BFCB /* AzureTtsHelper.swift in Sources */, + 0ADE296E2D8C66D60032BFCB /* AzureAsrHelper.swift in Sources */, 0A5EDBFC2D89C4FD00215E26 /* ClassicBluetoothHelper.swift in Sources */, 74858FAF1ED2DC5600515810 /* AppDelegate.swift in Sources */, 0A5EDC042D8B544100215E26 /* BluetoothMediaButtonHelper.swift in Sources */, diff --git a/ios/Runner/AppDelegate.swift b/ios/Runner/AppDelegate.swift index 70e9d28f1..6e837111b 100644 --- a/ios/Runner/AppDelegate.swift +++ b/ios/Runner/AppDelegate.swift @@ -5,6 +5,9 @@ import UIKit @objc class AppDelegate: FlutterAppDelegate { private var classicBluetoothHelper: ClassicBluetoothHelper? private var mediaButtonHelper: Any? + // 添加Azure语音服务辅助类 + private var azureAsrHelper: Any? + private var azureTtsHelper: Any? override func application( _ application: UIApplication, @@ -16,9 +19,10 @@ import UIKit // 立即初始化蓝牙，不要等待Flutter调用 classicBluetoothHelper?.initBluetooth() - // 仅在iOS 13.0及以上版本初始化媒体按钮辅助类 + // 仅在iOS 13.0及以上版本初始化媒体按钮辅助类和Azure语音服务辅助类 if #available(iOS 13.0, *) { mediaButtonHelper = BluetoothMediaButtonHelper() + // 稍后在方法通道注册时初始化Azure语音服务辅助类 } // 建立Flutter方法通道 @@ -81,44 +85,266 @@ import UIKit let bluetoothEventHandler = BluetoothStreamHandler(bluetoothHelper: self.classicBluetoothHelper!) classicEventChannel.setStreamHandler(bluetoothEventHandler) - // 仅在iOS 13.0及以上版本注册媒体按钮通道 - if #available(iOS 13.0, *), let mediaButtonHelper = mediaButtonHelper as? BluetoothMediaButtonHelper { + // 仅在iOS 13.0及以上版本注册媒体按钮通道和Azure语音服务通道 + if #available(iOS 13.0, *) { // 注册媒体按钮方法通道 - let mediaButtonChannel = FlutterMethodChannel( - name: "com.deep_voice.bluetooth_media_button", - binaryMessenger: controller.binaryMessenger - ) + registerMediaButtonChannels(controller: controller) + + // 注册Azure ASR方法通道 + registerAzureAsrChannels(controller: controller) - mediaButtonChannel.setMethodCallHandler { [weak self, weak mediaButtonHelper] (call, result) in - guard let _ = self, let mediaButtonHelper = mediaButtonHelper else { - result(FlutterError(code: "UNAVAILABLE", message: "媒体按钮辅助实例不可用", details: nil)) + // 注册Azure TTS方法通道 + registerAzureTtsChannels(controller: controller) + } + } + + // 注册媒体按钮通道 + @available(iOS 13.0, *) + private func registerMediaButtonChannels(controller: FlutterViewController) { + guard let mediaButtonHelper = mediaButtonHelper as? BluetoothMediaButtonHelper else { + return + } + + // 注册媒体按钮方法通道 + let mediaButtonChannel = FlutterMethodChannel( + name: "com.deep_voice.bluetooth_media_button", + binaryMessenger: controller.binaryMessenger + ) + + mediaButtonChannel.setMethodCallHandler { [weak self, weak mediaButtonHelper] (call, result) in + guard let _ = self, let mediaButtonHelper = mediaButtonHelper else { + result(FlutterError(code: "UNAVAILABLE", message: "媒体按钮辅助实例不可用", details: nil)) + return + } + + switch call.method { + case "startButtonListening": + let success = mediaButtonHelper.startButtonListening() + result(success) + + case "stopButtonListening": + let success = mediaButtonHelper.stopButtonListening() + result(success) + + default: + result(FlutterMethodNotImplemented) + } + } + + // 注册媒体按钮事件通道 + let mediaButtonEventChannel = FlutterEventChannel( + name: "com.deep_voice.bluetooth_media_button_events", + binaryMessenger: controller.binaryMessenger + ) + + // 创建媒体按钮事件处理器 + let mediaButtonEventHandler = MediaButtonStreamHandler(mediaButtonHelper: mediaButtonHelper) + mediaButtonEventChannel.setStreamHandler(mediaButtonEventHandler) + } + + // 注册Azure ASR通道 + @available(iOS 13.0, *) + private func registerAzureAsrChannels(controller: FlutterViewController) { + // 注册Azure ASR方法通道 + let azureAsrChannel = FlutterMethodChannel( + name: "com.deep_voice.azure_asr", + binaryMessenger: controller.binaryMessenger + ) + + // 初始化Azure ASR辅助类 + azureAsrHelper = AzureAsrHelper { [weak self] eventName, eventData in + // 推送事件到Flutter + self?.sendAsrEvent(eventName: eventName, eventData: eventData) + } + + // 设置ASR方法处理器 + azureAsrChannel.setMethodCallHandler { [weak self] (call, result) in + guard let self = self, + let asrHelper = self.azureAsrHelper as? AzureAsrHelper else { + result(FlutterError(code: "UNAVAILABLE", message: "Azure ASR辅助实例不可用", details: nil)) + return + } + + switch call.method { + case "initialize": + guard let args = call.arguments as? [String: Any], + let key = args["subscriptionKey"] as? String, + let region = args["region"] as? String else { + result(FlutterError(code: "INVALID_ARGS", message: "初始化参数无效", details: nil)) return } - switch call.method { - case "startButtonListening": - let success = mediaButtonHelper.startButtonListening() - result(success) - - case "stopButtonListening": - let success = mediaButtonHelper.stopButtonListening() - result(success) - - default: - result(FlutterMethodNotImplemented) - } + let languages = args["supportedLanguages"] as? [String] + let success = asrHelper.initialize(speechSubscriptionKey: key, serviceRegion: region, supportedLanguages: languages) + result(success) + + case "recognizeOnce": + let success = asrHelper.recognizeOnce() + result(success) + + case "startContinuousRecognition": + let success = asrHelper.startContinuousRecognition() + result(success) + + case "stopContinuousRecognition": + let success = asrHelper.stopContinuousRecognition() + result(success) + + case "isContinuousRecognitionActive": + let isActive = asrHelper.isContinuousRecognitionActive() + result(isActive) + + case "dispose": + asrHelper.dispose() + result(true) + + default: + result(FlutterMethodNotImplemented) + } + } + + // 注册Azure ASR事件通道 + let azureAsrEventChannel = FlutterEventChannel( + name: "com.deep_voice.azure_asr_events", + binaryMessenger: controller.binaryMessenger + ) + + // 创建ASR事件处理器 + let asrStreamHandler = AsrStreamHandler() + azureAsrEventChannel.setStreamHandler(asrStreamHandler) + } + + // 注册Azure TTS通道 + @available(iOS 13.0, *) + private func registerAzureTtsChannels(controller: FlutterViewController) { + // 注册Azure TTS方法通道 + let azureTtsChannel = FlutterMethodChannel( + name: "com.deep_voice.azure_tts", + binaryMessenger: controller.binaryMessenger + ) + + // 初始化Azure TTS辅助类 + azureTtsHelper = AzureTtsHelper { [weak self] eventName, eventData in + // 推送事件到Flutter + self?.sendTtsEvent(eventName: eventName, eventData: eventData) + } + + // 设置TTS方法处理器 + azureTtsChannel.setMethodCallHandler { [weak self] (call, result) in + guard let self = self, + let ttsHelper = self.azureTtsHelper as? AzureTtsHelper else { + result(FlutterError(code: "UNAVAILABLE", message: "Azure TTS辅助实例不可用", details: nil)) + return } - // 注册媒体按钮事件通道 - let mediaButtonEventChannel = FlutterEventChannel( - name: "com.deep_voice.bluetooth_media_button_events", - binaryMessenger: controller.binaryMessenger - ) - - // 创建媒体按钮事件处理器 - let mediaButtonEventHandler = MediaButtonStreamHandler(mediaButtonHelper: mediaButtonHelper) - mediaButtonEventChannel.setStreamHandler(mediaButtonEventHandler) + switch call.method { + case "initialize": + guard let args = call.arguments as? [String: Any], + let key = args["subscriptionKey"] as? String, + let region = args["region"] as? String else { + result(FlutterError(code: "INVALID_ARGS", message: "初始化参数无效", details: nil)) + return + } + + let language = args["language"] as? String ?? "zh-CN" + let success = ttsHelper.initialize(speechSubscriptionKey: key, serviceRegion: region, language: language) + result(success) + + case "setVoice": + guard let args = call.arguments as? [String: Any], + let voiceName = args["voiceName"] as? String else { + result(FlutterError(code: "INVALID_ARGS", message: "语音名称参数无效", details: nil)) + return + } + + let success = ttsHelper.setVoice(voiceName: voiceName) + result(success) + + case "setSpeechParams": + guard let args = call.arguments as? [String: Any] else { + result(FlutterError(code: "INVALID_ARGS", message: "语音参数无效", details: nil)) + return + } + + let rate = args["rate"] as? Int ?? 0 + let pitch = args["pitch"] as? Int ?? 0 + let volume = args["volume"] as? Int ?? 100 + + let success = ttsHelper.setSpeechParams(rate: rate, pitch: pitch, volume: volume) + result(success) + + case "speakText": + guard let args = call.arguments as? [String: Any], + let text = args["text"] as? String else { + result(FlutterError(code: "INVALID_ARGS", message: "文本参数无效", details: nil)) + return + } + + let success = ttsHelper.speakText(text: text) + result(success) + + case "stopSpeaking": + let success = ttsHelper.stopSpeaking() + result(success) + + case "isSpeaking": + let isSpeaking = ttsHelper.isSpeaking() + result(isSpeaking) + + case "dispose": + ttsHelper.dispose() + result(true) + + default: + result(FlutterMethodNotImplemented) + } } + + // 注册Azure TTS事件通道 + let azureTtsEventChannel = FlutterEventChannel( + name: "com.deep_voice.azure_tts_events", + binaryMessenger: controller.binaryMessenger + ) + + // 创建TTS事件处理器 + let ttsStreamHandler = TtsStreamHandler() + azureTtsEventChannel.setStreamHandler(ttsStreamHandler) + } + + // 存储ASR事件 + private var _cachedAsrEvent: [String: Any]? + + // 发送ASR事件 + @available(iOS 13.0, *) + private func sendAsrEvent(eventName: String, eventData: [String: Any]) { + // 调整格式：将event改为type，并将data的内容提升到顶层 + var eventMap = eventData + eventMap["type"] = eventName + + _cachedAsrEvent = eventMap + NotificationCenter.default.post( + name: Notification.Name("AzureAsrEvent"), + object: nil, + userInfo: eventMap + ) + } + + // 存储TTS事件 + private var _cachedTtsEvent: [String: Any]? + + // 发送TTS事件 + @available(iOS 13.0, *) + private func sendTtsEvent(eventName: String, eventData: [String: Any]) { + // 调整格式：将event改为type，并将data的内容提升到顶层 + var eventMap = eventData + eventMap["type"] = eventName + + _cachedTtsEvent = eventMap + NotificationCenter.default.post( + name: Notification.Name("AzureTtsEvent"), + object: nil, + userInfo: eventMap + ) } } @@ -190,3 +416,85 @@ class MediaButtonStreamHandler: NSObject, FlutterStreamHandler, MediaButtonEvent } } } + +// ASR事件流处理器 +@available(iOS 13.0, *) +class AsrStreamHandler: NSObject, FlutterStreamHandler { + private var eventSink: FlutterEventSink? + private var notificationObserver: Any? + + deinit { + if let observer = notificationObserver { + NotificationCenter.default.removeObserver(observer) + } + } + + func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { + self.eventSink = events + + // 注册通知观察者 + notificationObserver = NotificationCenter.default.addObserver( + forName: Notification.Name("AzureAsrEvent"), + object: nil, + queue: .main) { [weak self] notification in + if let userInfo = notification.userInfo as? [String: Any] { + self?.eventSink?(userInfo) + } + } + + return nil + } + + func onCancel(withArguments arguments: Any?) -> FlutterError? { + eventSink = nil + + // 移除通知观察者 + if let observer = notificationObserver { + NotificationCenter.default.removeObserver(observer) + notificationObserver = nil + } + + return nil + } +} + +// TTS事件流处理器 +@available(iOS 13.0, *) +class TtsStreamHandler: NSObject, FlutterStreamHandler { + private var eventSink: FlutterEventSink? + private var notificationObserver: Any? + + deinit { + if let observer = notificationObserver { + NotificationCenter.default.removeObserver(observer) + } + } + + func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { + self.eventSink = events + + // 注册通知观察者 + notificationObserver = NotificationCenter.default.addObserver( + forName: Notification.Name("AzureTtsEvent"), + object: nil, + queue: .main) { [weak self] notification in + if let userInfo = notification.userInfo as? [String: Any] { + self?.eventSink?(userInfo) + } + } + + return nil + } + + func onCancel(withArguments arguments: Any?) -> FlutterError? { + eventSink = nil + + // 移除通知观察者 + if let observer = notificationObserver { + NotificationCenter.default.removeObserver(observer) + notificationObserver = nil + } + + return nil + } +} diff --git a/ios/Runner/AzureAsrHelper.swift b/ios/Runner/AzureAsrHelper.swift new file mode 100644 index 000000000..4bd02368f --- /dev/null +++ b/ios/Runner/AzureAsrHelper.swift @@ -0,0 +1,408 @@ +import Foundation +import MicrosoftCognitiveServicesSpeech +import AVFoundation + +/// Azure ASR工具类，负责实现语音识别服务接口 +@available(iOS 13.0, *) +class AzureAsrHelper: NSObject { + // MARK: - 属性 + + /// 事件处理回调 + private var eventHandler: (String, [String: Any]) -> Void + + /// 语音配置信息 + private var speechSubscriptionKey: String = "" + private var serviceRegion: String = "" + + /// 语音识别相关 + private var speechConfig: SPXSpeechConfiguration? + private var recognizer: SPXSpeechRecognizer? + private var audioConfig: SPXAudioConfiguration? + + /// 状态标志 + private var isInitialized = false + private var _isContinuousRecognitionActive = false + + /// 当前语言和支持的语言 + private var currentLanguage = "zh-CN" + private var supportedLanguages: [String] = ["zh-CN", "en-US"] + private var isAutoDetectLanguage = false + + // MARK: - 初始化 + + init(eventHandler: @escaping (String, [String: Any]) -> Void) { + self.eventHandler = eventHandler + super.init() + } + + deinit { + dispose() + } + + // MARK: - ASR Service 接口实现 + + /// 初始化语音识别服务 + /// - Parameters: + /// - speechSubscriptionKey: Azure 语音服务订阅密钥 + /// - serviceRegion: Azure 服务区域 (如 eastasia) + /// - supportedLanguages: 支持的语言代码数组 (可选) + /// - Returns: 初始化是否成功 + func initialize(speechSubscriptionKey: String, serviceRegion: String, supportedLanguages: [String]? = nil) -> Bool { + print("[AzureAsrHelper] 初始化 Azure 语音服务") + + // 检查配置是否为空 + if speechSubscriptionKey.isEmpty || serviceRegion.isEmpty { + print("[AzureAsrHelper] 错误: Azure 配置信息不完整") + eventHandler("error", ["message": "Azure 配置信息不完整"]) + return false + } + + // 释放之前的资源 + dispose() + + // 记录配置信息 + self.speechSubscriptionKey = speechSubscriptionKey + self.serviceRegion = serviceRegion + + // 设置语言 + if let languages = supportedLanguages, !languages.isEmpty { + self.supportedLanguages = languages + } + + // 根据支持的语言数量决定是否启用自动语言检测 + isAutoDetectLanguage = self.supportedLanguages.count >= 2 + + // 如果只有一种语言，设置为当前语言 + if !isAutoDetectLanguage && !self.supportedLanguages.isEmpty { + currentLanguage = self.supportedLanguages[0] + } + + // 创建识别器和设置回调 + if !createRecognizerAndSetupCallbacks() { + return false + } + + print("[AzureAsrHelper] Azure 语音服务初始化成功") + isInitialized = true + return true + } + + /// 创建识别器并设置回调 + private func createRecognizerAndSetupCallbacks() -> Bool { + // 释放之前的 recognizer + recognizer = nil + audioConfig = nil + + do { + // 创建语音配置 + speechConfig = try SPXSpeechConfiguration(subscription: speechSubscriptionKey, region: serviceRegion) + + // 设置音频输入参数 + try setupAudioSession() + + // 直接使用麦克风音频配置 + audioConfig = try SPXAudioConfiguration() + + // 设置语言配置 + if isAutoDetectLanguage { + // 设置自动语言检测 + speechConfig?.setPropertyTo("Continuous", by: SPXPropertyId.speechServiceConnectionLanguageIdMode) + + // 创建自动语言检测配置 + let autoDetectSourceLanguageConfig = try SPXAutoDetectSourceLanguageConfiguration(supportedLanguages) + + // 创建识别器 + recognizer = try SPXSpeechRecognizer( + speechConfiguration: speechConfig!, + autoDetectSourceLanguageConfiguration: autoDetectSourceLanguageConfig, + audioConfiguration: audioConfig! + ) + } else { + // 设置指定的识别语言 + speechConfig?.speechRecognitionLanguage = currentLanguage + + // 创建识别器 + recognizer = try SPXSpeechRecognizer(speechConfiguration: speechConfig!, audioConfiguration: audioConfig!) + } + + // 设置所有回调 + setupAllCallbacks() + + return true + } catch { + print("[AzureAsrHelper] 错误: 创建识别器失败: \(error.localizedDescription)") + eventHandler("error", ["message": "创建识别器失败: \(error.localizedDescription)"]) + return false + } + } + + /// 设置音频会话 + private func setupAudioSession() throws { + let audioSession = AVAudioSession.sharedInstance() + + // 使用playAndRecord类别允许同时录音和播放 + try audioSession.setCategory(.playAndRecord, + mode: .voiceChat, // 使用voiceChat模式能够更好地支持回音消除 + options: [.allowBluetooth, .defaultToSpeaker, .allowAirPlay]) + + // 设置首选的输入和输出 + let currentRoute = audioSession.currentRoute + + // 获取当前是否连接了耳机或外部麦克风 + let hasHeadphones = currentRoute.outputs.contains { + $0.portType == .headphones || $0.portType == .bluetoothA2DP || $0.portType == .bluetoothHFP + } + + // 如果没有耳机，明确启用内置麦克风和扬声器的回音消除 + if !hasHeadphones { + try audioSession.setMode(.voiceChat) // 语音聊天模式有更强的回音消除 + + // 启用回音消除和噪声抑制 + try audioSession.setInputGain(0.8) // 适当降低输入增益以减少扬声器音频被麦克风捕获的可能性 + } else { + // 耳机模式，可以使用不同的设置 + try audioSession.setMode(.voiceChat) + try audioSession.setInputGain(1.0) + } + + // 设置合适的采样率 + try audioSession.setPreferredSampleRate(16000.0) // Azure语音识别推荐的采样率 + try audioSession.setPreferredIOBufferDuration(0.01) // 较小的缓冲区大小以减少延迟 + + // 激活音频会话 + try audioSession.setActive(true, options: .notifyOthersOnDeactivation) + + print("[AzureAsrHelper] 音频会话配置成功，已启用回音消除") + } + + /// 设置所有回调 + private func setupAllCallbacks() { + guard let recognizer = recognizer else { return } + + // 最终识别结果 + recognizer.addRecognizedEventHandler { [weak self] _, event in + guard let self = self else { return } + + if event.result.reason == SPXResultReason.recognizedSpeech { + let detectedLanguage = self.getDetectedLanguage(from: event.result) + print("[AzureAsrHelper] 识别结果: \(event.result.text ?? ""), 语言: \(detectedLanguage)") + self.eventHandler("result", [ + "text": event.result.text ?? "", + "detectedLanguage": detectedLanguage + ]) + } + } + + // 识别中事件 + recognizer.addRecognizingEventHandler { [weak self] _, event in + guard let self = self else { return } + + if event.result.reason == SPXResultReason.recognizingSpeech { + let detectedLanguage = self.getDetectedLanguage(from: event.result) + // print("[AzureAsrHelper] 识别中: \(event.result.text ?? ""), 语言: \(detectedLanguage)") + self.eventHandler("recognizing", [ + "text": event.result.text ?? "", + "detectedLanguage": detectedLanguage + ]) + } + } + + // 会话事件 + recognizer.addSessionStartedEventHandler { [weak self] _, _ in + guard let self = self else { return } + + print("[AzureAsrHelper] 识别会话已开始") + self._isContinuousRecognitionActive = true + self.eventHandler("sessionStarted", [:]) + } + + recognizer.addSessionStoppedEventHandler { [weak self] _, _ in + guard let self = self else { return } + + print("[AzureAsrHelper] 识别会话已结束") + self._isContinuousRecognitionActive = false + self.eventHandler("sessionStopped", [:]) + } + + // 取消事件 + recognizer.addCanceledEventHandler { [weak self] _, event in + guard let self = self else { return } + + let reason = event.reason.rawValue + let errorDetails = event.errorDetails ?? "未知错误" + + print("[AzureAsrHelper] 识别取消: \(errorDetails)") + + self.eventHandler("canceled", [ + "reason": reason, + "errorDetails": errorDetails + ]) + + self._isContinuousRecognitionActive = false + } + } + + /// 执行一次性语音识别 + /// - Returns: 是否成功启动识别 + func recognizeOnce() -> Bool { + if !isInitialized { + print("[AzureAsrHelper] 错误: 语音服务未初始化") + eventHandler("error", ["message": "语音服务未初始化"]) + return false + } + + // 如果正在连续识别，先停止 + if _isContinuousRecognitionActive { + stopContinuousRecognition() + } + + // 确保识别器已创建 + if recognizer == nil && !createRecognizerAndSetupCallbacks() { + return false + } + + do { + // 通知会话开始 + eventHandler("sessionStarted", [:]) + + // 执行识别 + try recognizer?.recognizeOnceAsync { [weak self] result in + guard let self = self else { return } + + if result.reason == SPXResultReason.recognizedSpeech { + let detectedLanguage = self.getDetectedLanguage(from: result) + self.eventHandler("result", [ + "text": result.text ?? "", + "detectedLanguage": detectedLanguage + ]) + } else if result.reason == SPXResultReason.noMatch { + print("[AzureAsrHelper] 无匹配结果") + self.eventHandler("noMatch", [:]) + } else if result.reason == SPXResultReason.canceled { + do { + let details = try SPXCancellationDetails(fromCanceledRecognitionResult: result) + let errorDetails = details.errorDetails ?? "未知错误" + self.eventHandler("error", ["message": "识别取消: \(errorDetails)"]) + } catch { + print("[AzureAsrHelper] 错误: 获取取消详情失败: \(error.localizedDescription)") + self.eventHandler("error", ["message": "识别取消，无法获取详细原因"]) + } + } + } + + return true + } catch { + print("[AzureAsrHelper] 错误: 识别异常: \(error.localizedDescription)") + eventHandler("error", ["message": "识别异常: \(error.localizedDescription)"]) + return false + } + } + + /// 开始连续语音识别 + /// - Returns: 是否成功启动识别 + func startContinuousRecognition() -> Bool { + if !isInitialized { + print("[AzureAsrHelper] 错误: 语音服务未初始化") + eventHandler("error", ["message": "语音服务未初始化"]) + return false + } + + // 如果已经在进行连续识别，先停止 + if _isContinuousRecognitionActive { + stopContinuousRecognition() + } + + // 确保识别器已创建 + if recognizer == nil && !createRecognizerAndSetupCallbacks() { + return false + } + + // 重新确保音频设置正确 + do { + try setupAudioSession() + } catch { + print("[AzureAsrHelper] 警告: 设置音频会话失败: \(error.localizedDescription)") + } + + do { + // 启动连续识别 + try recognizer?.startContinuousRecognition() + _isContinuousRecognitionActive = true + + print("[AzureAsrHelper] 连续识别开始") + return true + } catch { + print("[AzureAsrHelper] 错误: 开始连续识别失败: \(error.localizedDescription)") + eventHandler("error", ["message": "开始连续识别失败: \(error.localizedDescription)"]) + _isContinuousRecognitionActive = false + return false + } + } + + /// 停止连续语音识别 + /// - Returns: 是否成功停止识别 + func stopContinuousRecognition() -> Bool { + if !_isContinuousRecognitionActive || recognizer == nil { + return true + } + + do { + try recognizer?.stopContinuousRecognition() + _isContinuousRecognitionActive = false + print("[AzureAsrHelper] 连续识别已停止") + return true + } catch { + print("[AzureAsrHelper] 错误: 停止连续识别失败: \(error.localizedDescription)") + eventHandler("error", ["message": "停止连续识别失败: \(error.localizedDescription)"]) + _isContinuousRecognitionActive = false + return false + } + } + + /// 检查连续识别是否活跃 + /// - Returns: 连续识别是否处于活跃状态 + func isContinuousRecognitionActive() -> Bool { + return _isContinuousRecognitionActive + } + + /// 释放资源 + func dispose() { + print("[AzureAsrHelper] 释放资源") + + // 停止连续识别 + if _isContinuousRecognitionActive { + stopContinuousRecognition() + } + + // 释放音频会话 + do { + try AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + } catch { + print("[AzureAsrHelper] 警告: 释放音频会话失败: \(error.localizedDescription)") + } + + // 释放资源 + recognizer = nil + speechConfig = nil + audioConfig = nil + + // 重置状态 + _isContinuousRecognitionActive = false + isInitialized = false + } + + /// 从结果中获取检测到的语言 + private func getDetectedLanguage(from result: SPXSpeechRecognitionResult) -> String { + if isAutoDetectLanguage { + do { + let langResult = try SPXAutoDetectSourceLanguageResult(result) + return langResult.language ?? currentLanguage + } catch { + print("[AzureAsrHelper] 错误: 获取检测到的语言失败: \(error.localizedDescription)") + return currentLanguage + } + } else { + return currentLanguage + } + } +} \ No newline at end of file diff --git a/ios/Runner/AzureTtsHelper.swift b/ios/Runner/AzureTtsHelper.swift new file mode 100644 index 000000000..589df617c --- /dev/null +++ b/ios/Runner/AzureTtsHelper.swift @@ -0,0 +1,427 @@ +import Foundation +import MicrosoftCognitiveServicesSpeech +import AVFoundation + +/// Azure TTS工具类，负责实现TTS服务接口 +@available(iOS 13.0, *) +class AzureTtsHelper: NSObject { + // MARK: - 属性 + + /// 事件处理回调 + private var eventHandler: (String, [String: Any]) -> Void + + /// 语音配置信息 + private var speechSubscriptionKey: String = "" + private var serviceRegion: String = "" + + /// 语音合成配置 + private var speechConfig: SPXSpeechConfiguration? + + /// 语音合成器 + private var synthesizer: SPXSpeechSynthesizer? + + /// 是否初始化成功 + private var isInitialized = false + + /// 当前是否正在播放 + private var _isSpeaking = false + + /// 音频会话配置 + private var isAudioSessionConfigured = false + + // MARK: - 语音设置 + + /// 当前语音 + private var currentVoice = "zh-CN-XiaoxiaoNeural" + + /// 支持的语音映射 + private var voiceMap: [String: String] = [ + "zh-CN": "zh-CN-XiaoxiaoNeural", + "en-US": "en-US-JennyNeural", + "ja-JP": "ja-JP-NanamiNeural", + "ko-KR": "ko-KR-SunHiNeural", + "zh-TW": "zh-TW-HsiaoChenNeural", + "zh-HK": "zh-HK-HiuMaanNeural" + ] + + /// 当前语音合成参数 + private var currentSpeechRate = "0%" + private var currentPitch = "0%" + private var currentVolume = "100%" + + // MARK: - 初始化 + + init(eventHandler: @escaping (String, [String: Any]) -> Void) { + self.eventHandler = eventHandler + super.init() + } + + deinit { + dispose() + } + + // MARK: - TTS 接口实现 + + /// 初始化语音合成服务 + /// - Parameters: + /// - speechSubscriptionKey: Azure 语音服务订阅密钥 + /// - serviceRegion: Azure 服务区域 (如 eastasia) + /// - language: 语言代码 (默认 zh-CN) + /// - Returns: 初始化是否成功 + func initialize(speechSubscriptionKey: String, serviceRegion: String, language: String = "zh-CN") -> Bool { + print("[AzureTtsHelper] 初始化语音合成服务") + + // 检查配置是否为空 + if speechSubscriptionKey.isEmpty || serviceRegion.isEmpty { + print("[AzureTtsHelper] 错误: Azure 配置信息不完整") + eventHandler("error", ["error": "Azure 配置信息不完整"]) + return false + } + + // 释放之前的资源 + dispose() + + // 记录配置信息 + self.speechSubscriptionKey = speechSubscriptionKey + self.serviceRegion = serviceRegion + + // 配置音频会话 + if !configureAudioSession() { + print("[AzureTtsHelper] 警告: 音频会话配置失败，将尝试继续初始化") + } + + do { + // 创建语音配置 + speechConfig = try SPXSpeechConfiguration(subscription: speechSubscriptionKey, region: serviceRegion) + + // 设置默认语音 + let defaultVoice = getDefaultVoiceForLanguage(language) + currentVoice = defaultVoice + speechConfig?.speechSynthesisVoiceName = defaultVoice + + // 创建语音合成器 + synthesizer = try SPXSpeechSynthesizer(speechConfig!) + + // 设置事件处理器 + setupSynthesizerEvents() + + isInitialized = true + print("[AzureTtsHelper] TTS 引擎初始化成功") + + return true + } catch { + print("[AzureTtsHelper] 错误: 初始化语音合成服务失败: \(error.localizedDescription)") + eventHandler("error", ["error": "初始化语音合成服务失败: \(error.localizedDescription)"]) + return false + } + } + + /// 配置音频会话 + private func configureAudioSession() -> Bool { + let audioSession = AVAudioSession.sharedInstance() + do { + // 使用playback类别，但支持混合和空中播放 + try audioSession.setCategory(.playback, + mode: .spokenAudio, + options: [.mixWithOthers, .allowAirPlay, .duckOthers]) + + // 根据设备类型选择最佳配置 + let currentRoute = audioSession.currentRoute + let hasHeadphones = currentRoute.outputs.contains { + $0.portType == .headphones || $0.portType == .bluetoothA2DP || $0.portType == .bluetoothHFP + } + + // 优化音频路由 + if hasHeadphones { + // 耳机模式，使用默认设置 + try audioSession.setPreferredIOBufferDuration(0.005) // 较小的缓冲区大小以减少延迟 + } else { + // 扬声器模式 + try audioSession.setPreferredIOBufferDuration(0.005) + } + + // 避免完全激活音频会话，因为ASR可能已经激活 + // 这里使用setActive(false)是为了不与ASR冲突 + if !audioSession.isOtherAudioPlaying { + try audioSession.setActive(true, options: .notifyOthersOnDeactivation) + } + + isAudioSessionConfigured = true + print("[AzureTtsHelper] 音频会话配置成功") + return true + } catch { + print("[AzureTtsHelper] 警告: 音频会话配置失败: \(error.localizedDescription)") + isAudioSessionConfigured = false + return false + } + } + + /// 设置语音 + /// - Parameter voiceName: 语音名称 (如 "zh-CN-XiaoxiaoNeural") + /// - Returns: 设置是否成功 + func setVoice(voiceName: String) -> Bool { + if !isInitialized { + print("[AzureTtsHelper] 错误: TTS 引擎尚未初始化") + eventHandler("error", ["error": "TTS 引擎尚未初始化"]) + return false + } + + if voiceName.isEmpty { + print("[AzureTtsHelper] 错误: 声音名称为空") + eventHandler("error", ["error": "声音名称不能为空"]) + return false + } + + if voiceName == currentVoice { + print("[AzureTtsHelper] 已设置语音: \(voiceName)") + return true + } + + print("[AzureTtsHelper] 设置声音: \(voiceName)") + currentVoice = voiceName + + // 更新语音配置 + if let speechConfig = speechConfig { + speechConfig.speechSynthesisVoiceName = voiceName + return true + } + + return false + } + + /// 设置语音合成参数 + /// - Parameters: + /// - rate: 语速，范围 -100 到 100，默认为 0 + /// - pitch: 音调，范围 -100 到 100，默认为 0 + /// - volume: 音量，范围 0 到 100，默认为 100 + /// - Returns: 是否设置成功 + func setSpeechParams(rate: Int = 0, pitch: Int = 0, volume: Int = 100) -> Bool { + if !isInitialized { + print("[AzureTtsHelper] 错误: TTS 引擎尚未初始化") + eventHandler("error", ["error": "TTS 引擎尚未初始化"]) + return false + } + + // 转换参数格式 + currentSpeechRate = formatRateParam(rate) + currentPitch = formatPitchParam(pitch) + currentVolume = formatVolumeParam(volume) + + print("[AzureTtsHelper] 已设置语音参数: 语速=\(currentSpeechRate), 音调=\(currentPitch), 音量=\(currentVolume)") + return true + } + + /// 合成文本为语音并播放 + /// - Parameter text: 要合成的文本 + /// - Returns: 操作是否成功启动 + func speakText(text: String) -> Bool { + if !isInitialized { + print("[AzureTtsHelper] 错误: TTS 引擎尚未初始化") + eventHandler("error", ["error": "TTS 引擎尚未初始化"]) + return false + } + + if text.isEmpty { + print("[AzureTtsHelper] 警告: 要播放的文本为空") + return true + } + + // 确保音频会话已配置 + if !isAudioSessionConfigured { + _ = configureAudioSession() + } + + print("[AzureTtsHelper] 开始语音合成: \(text.prefix(50))...") + + // 生成SSML + let ssml = generateSsml(text: text) + + // 直接进行SSML合成 + return speakSsmlInternal(text: ssml) + } + + /// 内部SSML合成和播放 + private func speakSsmlInternal(text: String) -> Bool { + guard let synthesizer = synthesizer else { + print("[AzureTtsHelper] 错误: 合成器未初始化") + eventHandler("error", ["error": "合成器未初始化"]) + return false + } + + _isSpeaking = true + eventHandler("started", [:]) + + Task { + do { + // 使用异步方法进行合成并直接播放 + _ = try await synthesizer.startSpeakingSsml(text) + + } catch { + print("[AzureTtsHelper] 错误: 语音合成失败: \(error.localizedDescription)") + DispatchQueue.main.async { + self._isSpeaking = false + self.eventHandler("error", ["error": "语音合成失败: \(error.localizedDescription)"]) + } + } + } + + return true + } + + /// 停止当前语音合成 + /// - Returns: 操作是否成功 + func stopSpeaking() -> Bool { + if !isInitialized || !_isSpeaking { + return true + } + + // 停止合成 + do { + try synthesizer?.stopSpeaking() + _isSpeaking = false + eventHandler("canceled", [:]) + print("[AzureTtsHelper] 已停止语音合成") + return true + } catch { + print("[AzureTtsHelper] 错误: 停止语音合成失败: \(error.localizedDescription)") + eventHandler("error", ["error": "停止语音合成失败: \(error.localizedDescription)"]) + return false + } + } + + /// 检查是否正在播放 + /// - Returns: 当前是否正在播放语音 + func isSpeaking() -> Bool { + return _isSpeaking + } + + /// 释放资源 + func dispose() { + try? stopSpeaking() + + // 释放合成器和配置 + synthesizer = nil + speechConfig = nil + + isInitialized = false + _isSpeaking = false + isAudioSessionConfigured = false + print("[AzureTtsHelper] TTS 引擎已释放") + } + + // MARK: - 私有辅助方法 + + /// 设置合成器事件处理 + private func setupSynthesizerEvents() { + guard let synthesizer = synthesizer else { return } + + // 添加书签到达事件处理 + synthesizer.addBookmarkReachedEventHandler { _, e in + print("[AzureTtsHelper] 书签事件: 音频偏移: \((e.audioOffset + 5000) / 10000)ms, 文本: \"\(e.text)\"") + } + + // 合成完成事件 + synthesizer.addSynthesisCompletedEventHandler { [weak self] _, e in + guard let self = self else { return } + print("[AzureTtsHelper] 语音合成完成: 音频持续时间: \(e.result.audioDuration)") + DispatchQueue.main.async { + self._isSpeaking = false + self.eventHandler("completed", [:]) + } + } + + // 合成取消事件 + synthesizer.addSynthesisCanceledEventHandler { [weak self] _, e in + guard let self = self else { return } + + let result = e.result + do { + let cancellationDetails = try SPXSpeechSynthesisCancellationDetails(fromCanceledSynthesisResult: result) + print("[AzureTtsHelper] 语音合成取消: 原因: \(cancellationDetails.reason)") + + if cancellationDetails.reason == SPXCancellationReason.error { + print("[AzureTtsHelper] 错误代码: \(cancellationDetails.errorCode)") + print("[AzureTtsHelper] 错误详情: \(cancellationDetails.errorDetails ?? "未知")") + } + + DispatchQueue.main.async { + self._isSpeaking = false + self.eventHandler("error", ["error": "语音合成取消: \(cancellationDetails.errorDetails ?? "未知错误")"]) + } + } catch { + print("[AzureTtsHelper] 获取取消详情时出错: \(error)") + + DispatchQueue.main.async { + self._isSpeaking = false + self.eventHandler("error", ["error": "语音合成被取消"]) + } + } + } + + // 合成开始事件 + synthesizer.addSynthesisStartedEventHandler { _, _ in + // print("[AzureTtsHelper] 语音合成开始") + } + + // 合成中事件 + synthesizer.addSynthesizingEventHandler { _, _ in + // print("[AzureTtsHelper] 语音合成中") + } + } + + /// 生成 SSML 文本 + private func generateSsml(text: String) -> String { + return """ + + + + \(text) + + + + """ + } + + /// 格式化语速参数 + private func formatRateParam(_ rate: Int) -> String { + let clampedRate = rate.clamp(min: -100, max: 100) + if clampedRate == 0 { + return "0%" + } else if clampedRate < 0 { + return "\(Int(Double(clampedRate) * 0.9))%" + } else { + return "+\(clampedRate)%" + } + } + + /// 格式化音调参数 + private func formatPitchParam(_ pitch: Int) -> String { + let clampedPitch = pitch.clamp(min: -100, max: 100) + if clampedPitch == 0 { + return "0%" + } else { + return "\(Int(Double(clampedPitch) * 0.5))%" + } + } + + /// 格式化音量参数 + private func formatVolumeParam(_ volume: Int) -> String { + let clampedVolume = volume.clamp(min: 0, max: 100) + return "\(clampedVolume)%" + } + + /// 获取指定语言的默认语音 + private func getDefaultVoiceForLanguage(_ language: String) -> String { + return voiceMap[language] ?? "zh-CN-XiaoxiaoNeural" + } +} + +// MARK: - 扩展 + +extension Int { + func clamp(min: Int, max: Int) -> Int { + if self < min { return min } + if self > max { return max } + return self + } +} \ No newline at end of file diff --git a/ios/Runner/BluetoothMediaButtonHelper.swift b/ios/Runner/BluetoothMediaButtonHelper.swift index a7495693e..d385567ca 100644 --- a/ios/Runner/BluetoothMediaButtonHelper.swift +++ b/ios/Runner/BluetoothMediaButtonHelper.swift @@ -82,12 +82,19 @@ class BluetoothMediaButtonHelper: NSObject, AVAudioPlayerDelegate { isListening = true // 确保NowPlaying信息可用 - 这也不依赖于活跃的音频会话 - setupNowPlaying(isPlaying: false) + setupNowPlaying(isPlaying: true) // 尝试启用静音音频播放（但即使失败，远程命令仍然可用） do { try passivelySetupAudioSession() - // startSilentAudio() + startSilentAudio() + + //延时1秒 + DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { + self.setupNowPlaying(isPlaying: false) + self.stopSilentAudio() + } + // startUpdatingPlaybackInfo() } catch { NSLog("\(TAG): 音频设置或播放失败: \(error.localizedDescription)") @@ -136,7 +143,7 @@ class BluetoothMediaButtonHelper: NSObject, AVAudioPlayerDelegate { silentAudioPlayer = try AVAudioPlayer(contentsOf: audioUrl) silentAudioPlayer?.numberOfLoops = -1 // 无限循环播放 - silentAudioPlayer?.volume = 1.0 // 设置为全音量 + silentAudioPlayer?.volume = 0.0 // 设置为全音量 silentAudioPlayer?.prepareToPlay() // 预加载 silentAudioPlayer?.delegate = self @@ -337,6 +344,8 @@ class BluetoothMediaButtonHelper: NSObject, AVAudioPlayerDelegate { // 发送事件到处理器 eventHandler?.sendEvent(eventData) + + // 检查是否有其他音频播放 // let audioSession = AVAudioSession.sharedInstance() // let isOtherAudioPlaying = audioSession.isOtherAudioPlaying diff --git a/lib/data/models/agent.dart b/lib/data/models/agent.dart index 00e59c5d2..0bdbd278a 100644 --- a/lib/data/models/agent.dart +++ b/lib/data/models/agent.dart @@ -8,6 +8,7 @@ class Agent { final bool isOnline; final String voice; final String welcomeMessage; + final String? voiceWelcomeMessage; final String systemPrompt; const Agent({ @@ -19,6 +20,7 @@ class Agent { required this.isOnline, required this.voice, required this.welcomeMessage, + this.voiceWelcomeMessage, required this.systemPrompt, }); @@ -49,6 +51,7 @@ class Agent { 'voice': voice, 'welcomeMessage': welcomeMessage, 'systemPrompt': systemPrompt, + 'voiceWelcomeMessage': voiceWelcomeMessage, }; } @@ -74,6 +77,7 @@ class Agent { voice: voice ?? this.voice, welcomeMessage: welcomeMessage ?? this.welcomeMessage, systemPrompt: systemPrompt ?? this.systemPrompt, + voiceWelcomeMessage: voiceWelcomeMessage ?? this.voiceWelcomeMessage, ); } } \ No newline at end of file diff --git a/lib/data/providers/agent_provider.dart b/lib/data/providers/agent_provider.dart index 6294f7a5d..405baf58a 100644 --- a/lib/data/providers/agent_provider.dart +++ b/lib/data/providers/agent_provider.dart @@ -11,6 +11,7 @@ class AgentProvider { isOnline: true, voice: "zh-CN-XiaoxiaoNeural", welcomeMessage: "你好!我是 Emma,很高兴成为你的私人助理。我可以帮你处理各种任务,比如时间管理、目标规划、信息整理等。有什么我可以帮你的吗?", + voiceWelcomeMessage: "我在呢!", systemPrompt: ''' 你是一位名叫 Emma 的全能型个人助理,擅长处理各类工作和生活事务。 @@ -52,7 +53,7 @@ class AgentProvider { - 及时承认不足并寻求改进 -请始终以专业、高效、周到的态度,协助用户更好地管理工作和生活, 不要输出表情符号。 +请始终以专业、高效、周到的态度,协助用户更好地管理工作和生活, 不要输出表情符号或非可读的符号。 '''), Agent( diff --git a/lib/data/services/bluetooth_media_button_service.dart b/lib/data/services/bluetooth_media_button_service.dart index 8a76089fd..984e9140d 100644 --- a/lib/data/services/bluetooth_media_button_service.dart +++ b/lib/data/services/bluetooth_media_button_service.dart @@ -6,26 +6,8 @@ import '../../core/utils/logger.dart'; /// 蓝牙媒体按钮事件类型 enum MediaButtonType { - /// 播放按钮 - play, - - /// 暂停按钮 - pause, - - /// 下一曲按钮 - next, - - /// 上一曲按钮 - previous, - - /// 快进按钮 - seekForward, - - /// 快退按钮 - seekBackward, - - /// 播放/暂停切换按钮 - togglePlayPause, + /// 播放/暂停按钮 + playPause, /// 未知按钮 unknown @@ -48,29 +30,19 @@ class MediaButtonEvent { MediaButtonType buttonType; switch (map['buttonType']) { - case 'play': - buttonType = MediaButtonType.play; - break; - case 'pause': - buttonType = MediaButtonType.pause; - break; - case 'next': - buttonType = MediaButtonType.next; - break; - case 'previous': - buttonType = MediaButtonType.previous; - break; - case 'seekForward': - buttonType = MediaButtonType.seekForward; - break; - case 'seekBackward': - buttonType = MediaButtonType.seekBackward; - break; - case 'togglePlayPause': - buttonType = MediaButtonType.togglePlayPause; + case 'playPause': + buttonType = MediaButtonType.playPause; break; default: - buttonType = MediaButtonType.unknown; + // 将所有其他事件类型映射为 playPause + if (map['buttonType'] == 'play' || + map['buttonType'] == 'pause' || + map['buttonType'] == 'togglePlayPause') { + buttonType = MediaButtonType.playPause; + } else { + // 忽略其他类型（next, previous, seekForward, seekBackward） + buttonType = MediaButtonType.unknown; + } } return MediaButtonEvent( @@ -82,19 +54,7 @@ class MediaButtonEvent { @override String toString() { switch (buttonType) { - case MediaButtonType.play: - return '蓝牙媒体按钮: 播放'; - case MediaButtonType.pause: - return '蓝牙媒体按钮: 暂停'; - case MediaButtonType.next: - return '蓝牙媒体按钮: 下一曲'; - case MediaButtonType.previous: - return '蓝牙媒体按钮: 上一曲'; - case MediaButtonType.seekForward: - return '蓝牙媒体按钮: 快进'; - case MediaButtonType.seekBackward: - return '蓝牙媒体按钮: 快退'; - case MediaButtonType.togglePlayPause: + case MediaButtonType.playPause: return '蓝牙媒体按钮: 播放/暂停'; case MediaButtonType.unknown: return '蓝牙媒体按钮: 未知'; @@ -219,12 +179,30 @@ class BluetoothMediaButtonService extends GetxService { try { if (eventMap['type'] == 'mediaButtonEvent') { - final buttonEvent = MediaButtonEvent.fromMap(eventMap); + // 获取按钮类型 + final String buttonType = eventMap['buttonType'] as String? ?? 'unknown'; - // 广播事件 - _buttonEventController.add(buttonEvent); - - Logger.info('成功处理蓝牙媒体按钮事件: $buttonEvent'); + // 判断是否为playPause相关事件 + if (buttonType == 'playPause' || + buttonType == 'play' || + buttonType == 'pause' || + buttonType == 'togglePlayPause') { + + // 创建playPause事件并广播 + final buttonEvent = MediaButtonEvent( + buttonType: MediaButtonType.playPause, + timestamp: eventMap['timestamp'] as int? ?? DateTime.now().millisecondsSinceEpoch, + ); + + + + // 广播事件 + _buttonEventController.add(buttonEvent); + Logger.info('发送playPause按钮事件: $buttonEvent'); + } else { + // 忽略其他类型的按钮事件 + Logger.info('忽略非playPause按钮事件: $buttonType'); + } } else { Logger.warning('收到未知类型的事件: ${eventMap['type']}'); } diff --git a/lib/data/services/classic_bluetooth_service.dart b/lib/data/services/classic_bluetooth_service.dart index dc3324134..a73ad5d6a 100644 --- a/lib/data/services/classic_bluetooth_service.dart +++ b/lib/data/services/classic_bluetooth_service.dart @@ -38,7 +38,7 @@ class ClassicBluetoothService extends GetxService { StreamSubscription? _bluetoothStateListener; /// 蓝牙开关状态 - final bluetoothState = 'on'.obs; + final bluetoothState = 'unknown'.obs; /// 蓝牙是否可用 bool get isBluetoothOn => bluetoothState.value == 'on'; @@ -56,6 +56,11 @@ class ClassicBluetoothService extends GetxService { try { Logger.info('正在初始化蓝牙服务...'); + // 获取当前蓝牙状态 + final bool isEnabled = await _channel.invokeMethod('isBluetoothEnabled') ?? false; + bluetoothState.value = isEnabled ? 'on' : 'off'; + Logger.info('初始蓝牙状态: ${bluetoothState.value}'); + // 启动蓝牙设备监控 - 这将调用原生代码中的initBluetooth await _channel.invokeMethod('startBluetoothDeviceMonitoring'); diff --git a/lib/data/services/speech_impl/azure_asr_service.dart b/lib/data/services/speech_impl/azure_asr_service.dart index 95f01bddd..41e9fd3f7 100644 --- a/lib/data/services/speech_impl/azure_asr_service.dart +++ b/lib/data/services/speech_impl/azure_asr_service.dart @@ -26,10 +26,6 @@ class AzureAsrService extends GetxService implements AsrService { StreamController? _eventStreamController; StreamSubscription? _eventSubscription; - // 事件通道状态 - bool _isEventChannelReady = false; - Completer? _eventChannelReadyCompleter; - // 最新的识别结果 String _latestRecognizedText = ''; String get latestRecognizedText => _latestRecognizedText; @@ -55,41 +51,15 @@ class AzureAsrService extends GetxService implements AsrService { /// 设置事件通道 void _setupEventChannel() { - _eventChannelReadyCompleter = Completer(); - _eventSubscription = _eventChannel .receiveBroadcastStream() .listen((event) { if (event is Map) { - final String eventType = event['type'] as String? ?? ''; - - // 处理通道准备好的事件 - if (eventType == 'channelReady') { - _isEventChannelReady = true; - if (!_eventChannelReadyCompleter!.isCompleted) { - _eventChannelReadyCompleter!.complete(); - } - return; - } - _handleRecognitionEvent(event); } }, onError: _handleRecognitionError); } - /// 等待事件通道准备好 - Future _waitForEventChannel({Duration timeout = const Duration(seconds: 5)}) async { - if (_isEventChannelReady) return true; - - try { - await _eventChannelReadyCompleter!.future.timeout(timeout); - return true; - } on TimeoutException { - Logger.error('等待事件通道准备好超时'); - return false; - } - } - @override Future initialize({ List? supportedLanguages, @@ -159,11 +129,6 @@ class AzureAsrService extends GetxService implements AsrService { await initialize(); } - // 确保事件通道已准备好 - if (!await _waitForEventChannel()) { - throw Exception('事件通道未准备好，无法开始连续识别'); - } - if (_isContinuousRecognitionActive) { await stopContinuousRecognition(); } @@ -217,6 +182,9 @@ class AzureAsrService extends GetxService implements AsrService { final Map eventMap = event; final String eventType = eventMap['type'] as String? ?? ''; + // 添加日志帮助调试 + Logger.debug('收到语音识别事件: $eventType, 数据: $eventMap'); + switch (eventType) { case 'result': final String text = eventMap['text'] as String? ?? ''; diff --git a/lib/modules/chat/controllers/chat_controller.dart b/lib/modules/chat/controllers/chat_controller.dart index 152e8d8f5..b9e75b7fa 100644 --- a/lib/modules/chat/controllers/chat_controller.dart +++ b/lib/modules/chat/controllers/chat_controller.dart @@ -10,6 +10,7 @@ import '../../../core/utils/logger.dart'; import '../../../data/services/chat_history_service.dart'; import '../../../data/services/voice_interaction_service.dart'; import '../../../data/services/asr_service.dart'; + class ChatController extends GetxController { // 服务 final VolcanoAIService _aiService = Get.find(); @@ -35,6 +36,9 @@ class ChatController extends GetxController { final RxBool isSending = false.obs; final RxBool isTtsEnabled = true.obs; final RxBool isVoiceRecognitionActive = false.obs; + + // 语音交互模式 + bool _isVoiceInteractionMode = false; // 语音识别状态 final RxString recognizedText = ''.obs; @@ -51,6 +55,12 @@ class ChatController extends GetxController { // 取消标志 bool _shouldCancelAiResponse = false; + + // 自动停止语音识别计时器 + Timer? _autoStopTimer; + + // 记录最后一次活动时间 + DateTime _lastActivityTime = DateTime.now(); @override Future onInit() async { @@ -60,7 +70,7 @@ class ChatController extends GetxController { // 从路由参数获取Agent信息 final Map args = Get.arguments ?? {}; - final String agentId = args['agentId'] ?? 'personal_assistant'; + final String agentId = args['agent'] ?? args['agentId'] ?? 'personal_assistant'; // 获取Agent final Agent? foundAgent = AgentProvider.getAgentById(agentId); @@ -74,6 +84,9 @@ class ChatController extends GetxController { _ttsService = Get.find(); _asrService = Get.find(); + + // 设置语音交互模式 + _isVoiceInteractionMode = args['isVoiceInteraction'] == true; // 设置TTS服务 await _ttsService.setVoice(agent.voice); @@ -94,9 +107,11 @@ class ChatController extends GetxController { // } } - // 如果playVoiceOnEnter为true，自动激活语音识别 - if (args['playVoiceOnEnter'] == true) { - startVoiceRecognition(); + // 如果isVoiceInteraction为true，自动激活语音识别 + if (args['isVoiceInteraction'] == true) { + _ttsService.speakOnce(agent.voiceWelcomeMessage ?? agent.welcomeMessage); + startVoiceRecognition(); + // 自动停止计时器会在startVoiceRecognition中启动 } // 订阅VoiceInteractionService事件 @@ -105,6 +120,9 @@ class ChatController extends GetxController { // 使用微任务在布局完成后自动滚动到底部 WidgetsBinding.instance.addPostFrameCallback((_) { _scrollToBottom(); + Future.delayed(const Duration(milliseconds: 500), () { + _scrollToBottom(); + }); }); } @@ -203,6 +221,7 @@ class ChatController extends GetxController { _aiResponseSubscription?.cancel(); _recognitionSubscription?.cancel(); _voiceInteractionSubscription?.cancel(); + _autoStopTimer?.cancel(); // 停止TTS _ttsService.stop(); @@ -422,7 +441,7 @@ class ChatController extends GetxController { _voiceInteractionService.pauseVoiceInteraction(); try { - // 启动连续语音识别 + // 启动连续语音识别 // 注意：在UI更新后，使用模拟波形动画替代实际麦克风输入的波形数据 final recognitionStream = await _asrService.startContinuousRecognition(); @@ -432,9 +451,10 @@ class ChatController extends GetxController { // 重置临时消息索引 _tempMessageIndex = -1; - - + // 启动自动停止计时器 + _startAutoStopTimer(); + // 监听识别事件 _recognitionSubscription = recognitionStream.listen( _handleRecognitionEvent, @@ -449,6 +469,9 @@ class ChatController extends GetxController { // 处理语音识别事件 void _handleRecognitionEvent(RecognitionEvent event) { + // 更新活动时间 + _updateActivityTime(); + switch (event.type) { case RecognitionEventType.intermediateResult: _handleRecognizing(event.text); @@ -465,6 +488,42 @@ class ChatController extends GetxController { } } + // 更新最后活动时间 + void _updateActivityTime() { + _lastActivityTime = DateTime.now(); + } + + // 启动自动停止计时器 + void _startAutoStopTimer() { + // 取消现有计时器 + _autoStopTimer?.cancel(); + + // 初始化最后活动时间 + _updateActivityTime(); + + // 根据模式设置超时时间 + final int timeoutSeconds = _isVoiceInteractionMode ? 8 : 15; + + // 创建新的计时器，每秒检查一次 + _autoStopTimer = Timer.periodic(Duration(seconds: 1), (timer) { + // 检查是否有活动 + bool isActive = _ttsService.isSpeaking || isSpeechDetected.value; + + // 如果有活动，更新时间并继续等待 + if (isActive) { + _updateActivityTime(); + return; + } + + // 检查空闲时间 + int idleSeconds = DateTime.now().difference(_lastActivityTime).inSeconds; + if (idleSeconds >= timeoutSeconds) { + timer.cancel(); + _stopVoiceRecognitionAutomatically(); + } + }); + } + // 处理语音识别中的状态 void _handleRecognizing(String text) { if (text.isEmpty) return; @@ -476,6 +535,9 @@ class ChatController extends GetxController { isSpeechDetected.value = true; } + // 更新活动时间 + _updateActivityTime(); + // 用户开始讲话时，立即停止当前TTS和中断AI响应 if (isSending.value || _aiResponseSubscription != null) { _interruptCurrentResponse("检测到用户开始讲话，中断当前响应"); @@ -509,6 +571,9 @@ class ChatController extends GetxController { Logger.info('处理语音识别最终结果: $text'); recognizedText.value = text; + + // 更新活动时间 + _updateActivityTime(); // 查找并移除临时消息 if (_tempMessageIndex >= 0 && _tempMessageIndex < messages.length) { @@ -627,4 +692,10 @@ class ChatController extends GetxController { _saveChatHistory(); _scrollToBottom(); } + + void _stopVoiceRecognitionAutomatically() { + final int timeoutSeconds = _isVoiceInteractionMode ? 8 : 15; + Logger.info('${timeoutSeconds}秒无活动，自动停止语音识别'); + stopVoiceRecognition(); + } } diff --git a/lib/modules/home/controllers/home_controller.dart b/lib/modules/home/controllers/home_controller.dart index 13151d58a..cfe397256 100644 --- a/lib/modules/home/controllers/home_controller.dart +++ b/lib/modules/home/controllers/home_controller.dart @@ -8,6 +8,7 @@ import 'package:flutter_blue_plus/flutter_blue_plus.dart' hide BluetoothDevice; import 'package:flutter/material.dart'; import '../../../data/services/voice_interaction_service.dart'; import 'package:permission_handler/permission_handler.dart'; +import '../../../data/providers/agent_provider.dart'; // 连接状态枚举 enum ConnectionStatus { disconnected, // 未连接 @@ -329,29 +330,13 @@ class HomeController extends GetxController { // 根据按钮类型执行相应操作 switch (event.buttonType) { - case MediaButtonType.play: - // 处理播放按钮事件 - _handlePlayButtonPressed(); - break; - - case MediaButtonType.pause: - // 处理暂停按钮事件 - _handlePauseButtonPressed(); - break; - - case MediaButtonType.togglePlayPause: - // 处理播放/暂停切换按钮事件 - _handleTogglePlayPause(); - break; - - case MediaButtonType.next: - // 处理下一曲按钮事件 - _handleNextButtonPressed(); - break; - - case MediaButtonType.previous: - // 处理上一曲按钮事件 - _handlePreviousButtonPressed(); + case MediaButtonType.playPause: + // 跳转到chat页面，使用个人助手模式并开启语音交互 + // Get.toNamed('/chat', arguments: { + // 'isVoiceInteraction': true, + // 'agent': 'personal_assistant' + // }); + Logger.info('通过蓝牙按钮启动个人助手语音交互'); break; default: @@ -360,39 +345,6 @@ class HomeController extends GetxController { } } - // 处理播放按钮事件 - void _handlePlayButtonPressed() { - Logger.info('处理播放按钮事件'); - // 在这里添加播放功能的实现 - // 例如：启动语音助手 - } - - // 处理暂停按钮事件 - void _handlePauseButtonPressed() { - Logger.info('处理暂停按钮事件'); - // 在这里添加暂停功能的实现 - // 例如：暂停语音交互 - } - - // 处理播放/暂停切换按钮事件 - void _handleTogglePlayPause() { - Logger.info('处理播放/暂停切换按钮事件'); - // 在这里添加播放/暂停切换功能的实现 - } - - // 处理下一曲按钮事件 - void _handleNextButtonPressed() { - Logger.info('处理下一曲按钮事件'); - // 在这里添加下一曲功能的实现 - // 例如：跳到下一个问题或调整音量 - } - - // 处理上一曲按钮事件 - void _handlePreviousButtonPressed() { - Logger.info('处理上一曲按钮事件'); - // 在这里添加上一曲功能的实现 - // 例如：回到上一个问题或调整音量 - } @override void onClose() { diff --git a/lib/modules/pairing/controllers/pairing_controller.dart b/lib/modules/pairing/controllers/pairing_controller.dart index e0b970ed7..437beaef4 100644 --- a/lib/modules/pairing/controllers/pairing_controller.dart +++ b/lib/modules/pairing/controllers/pairing_controller.dart @@ -8,6 +8,7 @@ import 'package:flutter/material.dart'; import 'package:permission_handler/permission_handler.dart'; import 'package:app_settings/app_settings.dart'; import 'package:flutter/services.dart'; +import 'dart:io'; /// 简化的配对控制器，仅用于显示已连接经典蓝牙设备和处理基本配对流程 @@ -129,13 +130,15 @@ class PairingController extends GetxController { } _setupStatusListeners(); // 检查权限 - // statusDescription.value = '正在检查权限...'; - // final hasPermission = await _bluetoothService.checkPermissions(); - // if (!hasPermission) { - // statusDescription.value = '缺少蓝牙权限'; - // isLoading.value = false; - // return; - // } + if (Platform.isAndroid) { + statusDescription.value = '正在检查权限...'; + final hasPermission = await _bluetoothService.checkPermissions(); + if (!hasPermission) { + statusDescription.value = '缺少蓝牙权限'; + isLoading.value = false; + return; + } + } // 刷新设备列表 statusDescription.value = '正在搜索设备...'; diff --git a/pubspec.yaml b/pubspec.yaml index 3d1b8cc58..c47019e21 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -29,8 +29,6 @@ dependencies: logger: ^2.0.2+1 flutter_blue_plus: ^1.31.16 just_audio: ^0.9.36 - azure_speech_recognition: - path: ./azure window_size: