From d33330eda0c46372a233e07e4144ff816bbe3841 Mon Sep 17 00:00:00 2001 From: fdp <1286779656@qq.com> Date: Wed, 6 Aug 2025 20:18:53 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dai=E5=A3=B0=E9=9F=B3?= =?UTF-8?q?=E6=97=B6=E5=A4=A7=E6=97=B6=E5=B0=8F=E7=9A=84=E9=97=AE=E9=A2=98?= =?UTF-8?q?=EF=BC=8C=E5=8A=A0=E5=85=A5=E8=AF=86=E5=88=AB=E8=B6=85=E6=97=B6?= =?UTF-8?q?=E6=9C=BA=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agent/controllers/agent_controller.dart | 20 ++++++++--- .../controllers/translation_controller.dart | 9 ++++- .../agent_service/AgentService.kt | 10 +++--- .../agent_service/AgentServiceImpl.swift | 22 +++++++++--- .../agent_service/AgentServicePlugin.swift | 9 ++++- .../agent_service/lib/agent_service.dart | 29 +++++++++++++-- .../azure_speech/AzureAsrHelper.kt | 2 +- .../Sources/azure_speech/AzureAsrHelper.swift | 36 ++++++++++++------- .../azure_speech/AzureSpeechPlugin.swift | 15 ++++++-- 9 files changed, 117 insertions(+), 35 deletions(-) diff --git a/lib/modules/agent/controllers/agent_controller.dart b/lib/modules/agent/controllers/agent_controller.dart index 5330abd4c..8debed827 100644 --- a/lib/modules/agent/controllers/agent_controller.dart +++ b/lib/modules/agent/controllers/agent_controller.dart @@ -3,6 +3,8 @@ import 'dart:convert'; import 'dart:ffi'; import 'dart:io'; import 'package:amap_service/amap_service.dart'; +import '../../../data/services/ble_manager.dart'; +import '../../../data/services/bluetooth_manager.dart'; import '../../../data/services/novice_guide_service.dart'; import '/core/utils/upload_oss.dart'; import '/data/models/user_Info.dart'; @@ -41,7 +43,10 @@ class AgentController extends GetxController { final ScrollController scrollController = ScrollController(); final noviceGuideService = Get.find(); - + // 确保服务已初始化 + // ========== 依赖服务 ========== + final BluetoothManager _bluetoothManager = Get.find(); + final BleManager _bleManager = Get.find(); //音乐播放管理 final _musiceManager = Get.find(); @@ -558,16 +563,16 @@ class AgentController extends GetxController { break; case AgentServiceEventType.ttsStarted: - // isSpeaking.value = true; - // Logger.i(TAG, 'TTS开始播放'); + isSpeaking.value = true; + print('TTS开始播放'); break; case AgentServiceEventType.ttsCompleted: case AgentServiceEventType.ttsStopped: case AgentServiceEventType.ttsCanceled: - // isSpeaking.value = false; + isSpeaking.value = false; - // Logger.i(TAG, 'TTS停止播放, $event.type'); + Logger.i(TAG, 'TTS停止播放, $event.type'); break; case AgentServiceEventType.playbackStarted: @@ -1035,6 +1040,11 @@ class AgentController extends GetxController { try { await AgentService.recognizeCallback(); + if (_bluetoothManager.currentDeviceRx.value != null) { + await AgentService.restoreOriginalAudioState(); + } else { + await AgentService.disableBluetoothAudio(); + } } catch (e) { Logger.e(TAG, '启动识别器失败: $e'); } diff --git a/lib/modules/translation/controllers/translation_controller.dart b/lib/modules/translation/controllers/translation_controller.dart index 94cf4b25c..ae649a20e 100644 --- a/lib/modules/translation/controllers/translation_controller.dart +++ b/lib/modules/translation/controllers/translation_controller.dart @@ -9,6 +9,7 @@ import 'package:get_storage/get_storage.dart'; import 'package:intl/intl.dart'; import 'package:path_provider/path_provider.dart'; import 'package:permission_handler/permission_handler.dart'; +import '../../../data/services/bluetooth_manager.dart'; import '../../../data/services/music_manager.dart'; import '../../../data/services/volcano_translation_service.dart'; import '../../../data/services/tts_service.dart'; @@ -30,6 +31,8 @@ class TranslationController extends GetxController { static const String TAG = 'TranslationController'; // 服务 + final BluetoothManager _bluetoothManager = Get.find(); + final BleManager _bleManager = Get.find(); final AsrService _asrService = Get.find(); final VolcanoTranslationService _translationService = Get.find(); @@ -483,7 +486,11 @@ class TranslationController extends GetxController { if (currentMode.value == 'faceToFace') { _asrService.stopContinuousRecognition(); } else { - await _ttsService.setAudioOutputDevice(0); + if (_bluetoothManager.currentDeviceRx.value != null) { + restoreOriginalAudioState(); + } else { + disableBluetoothAudio(); + } await _asrService.startContinuousRecognition(_audioSourceType); // 开始ASR活跃时长计时 _startAsrActiveTracking(); diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index 76f1fc5dc..cd5762b9d 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -420,7 +420,7 @@ object AgentService : CoroutineScope { try { delay(maxIdleSeconds * 1000L) - if (isActive && isRecognitionActive && !hasSpeechDetected && !isTtsSpeaking && !isAiStreaming) { + if (isActive && isRecognitionActive && !isTtsSpeaking && !isAiStreaming) { stopRecognition() sendEvent( "auto_stop", @@ -522,12 +522,12 @@ object AgentService : CoroutineScope { override fun onRecognizing(recognizing: String, detectedLanguage: String) { if (recognizing.isNotEmpty()) { // 检测到语音,更新状态 - val previousHasSpeech = hasSpeechDetected - _hasSpeechDetected.set(true) + val previousHasSpeech = hasSpeechDetected + _hasSpeechDetected.set(true) if (!previousHasSpeech) { - restartIdleCheck() - } + restartIdleCheck() + } sendEvent( "recognizing", mapOf( diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index e0e0ece65..65fe56f08 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -291,8 +291,8 @@ class AgentServiceImpl: NSObject { // 使用更安全的方式检查self是否存在 DispatchQueue.main.async { [weak self] in guard let self = self else { return } - - if self.isRecognizing && !self.hasSpeechDetected && !self.isSpeaking && !self.isAiStreaming { + + if self.isRecognizing && !self.isSpeaking && !self.isAiStreaming { self.stopRecognition() self.sendEvent(name: "auto_stop", data: [ "reason": "idle_timeout", @@ -349,8 +349,14 @@ print("ai启动语音=\(audioSourceType)=\(useBle)") return true } - func recognizeCallback(useBle: Bool = false) -> Bool { - + func recognizeCallback() -> Bool { + print("重新设置识别回调=") + guard let asrsetupSuccess = azureAsrHelper?.setupEventListeners( + callback: self + ), asrsetupSuccess else { + sendError("初始化语音识别服务失败", code: "ASR_INIT_ERROR") + return false + } return true } @@ -365,7 +371,12 @@ print("ai启动语音=\(audioSourceType)=\(useBle)") return true } - + func disableBluetoothAudio() { + azureAsrHelper?.disableBluetoothAudio() + } + func restoreOriginalAudioState() { + azureAsrHelper?.restoreOriginalAudioState() + } func pushAudioData(_ audioData: Data) -> Bool { if !isRecognizing || !isInitialized { return false @@ -1572,6 +1583,7 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { hasSpeechDetected = true if !previousHasSpeech { + print("onRecognizing restartIdleCheck") restartIdleCheck() } diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift index 0c0ef5703..fb8baa1f8 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift @@ -51,7 +51,14 @@ public class AgentServicePlugin: NSObject, FlutterPlugin { case "stopConversation": impl.stopRecognition() result(true) - + + case "disableBluetoothAudio": + impl.disableBluetoothAudio(); + return + + case "restoreOriginalAudioState": + impl.restoreOriginalAudioState(); + return case "processTextInput": guard let arguments = call.arguments as? [String: Any], let text = arguments["text"] as? String else { diff --git a/local_plugins/agent_service/lib/agent_service.dart b/local_plugins/agent_service/lib/agent_service.dart index e3e2eb1c4..f8294ef2a 100644 --- a/local_plugins/agent_service/lib/agent_service.dart +++ b/local_plugins/agent_service/lib/agent_service.dart @@ -261,6 +261,33 @@ class AgentService { } } + /// 恢复原始音频状态 + /// + /// 恢复原始音频状态 + /// 返回是否成功恢复原始音频状态 + static Future restoreOriginalAudioState() async { + try { + final bool result = + await _channel.invokeMethod('restoreOriginalAudioState'); + return result; + } on PlatformException catch (e) { + throw AgentServiceException(e.code, e.message ?? '恢复原始音频状态失败', e.details); + } + } + + /// 禁用蓝牙音频 + /// + /// 禁用蓝牙音频 + /// 返回是否成功禁用蓝牙音频 + static Future disableBluetoothAudio() async { + try { + final bool result = await _channel.invokeMethod('disableBluetoothAudio'); + return result; + } on PlatformException catch (e) { + throw AgentServiceException(e.code, e.message ?? '禁用蓝牙音频失败', e.details); + } + } + /// 停止对话 /// /// 停止语音识别 @@ -332,8 +359,6 @@ class AgentService { } } - - /// 停止语音合成 static Future stopAiSteam() async { try { diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt index 02d909d4a..0d00c776b 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt @@ -745,7 +745,7 @@ class AzureAsrHelper(private val context: Context) { */ fun saveAudioDataTo(buffer: ByteArray) { - if (audioSourceType == AudioSourceType.MICROPHONE) return + if (audioSourceType == AudioSourceType.MICROPHONE||!isContinuousRecognitionActive) return // 放入队列,由写线程写入 writeQueue.offer(buffer.copyOf()) diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift index 92b6b40c0..40846707f 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift @@ -131,7 +131,7 @@ public class AzureAsrHelper: NSObject { */ private func preInitializeAudioComponents() { if audioStream == nil { - audioStream = AudioStream() + audioStream = AudioStream(parentHelper: self) audioStream?.initAudioRecord() audioStream?.onAudioData = { [weak self] data in self?.recordfile?.saveAudioDataToWav(data) @@ -354,7 +354,7 @@ public class AzureAsrHelper: NSObject { do { // 【优化】检查是否已经初始化,避免重复创建 if audioStream == nil { - audioStream = AudioStream() + audioStream = AudioStream(parentHelper: self) audioStream?.initAudioRecord() audioStream?.onAudioData = { [weak self] data in self?.recordfile?.saveAudioDataToWav(data) @@ -548,6 +548,7 @@ public class AzureAsrHelper: NSObject { public func disableBluetoothAudio() { do { + print("禁用蓝牙音频功能,切换回正常音频模式") audioStream?.setAudioOutputRoute(.speaker) } catch { print("disableBluetoothAudio") @@ -559,6 +560,7 @@ public class AzureAsrHelper: NSObject { public func restoreOriginalAudioState() { do { + print("恢复原始音频设备状态(通常是重新启用蓝牙)") audioStream?.setAudioOutputRoute(.receiver) } catch { print("restoreOriginalAudioState") @@ -632,9 +634,18 @@ public class AzureAsrHelper: NSObject { private var isRunning = false public var isWriting = false private let bufferSize: Int = 4096 - private var writeThread: DispatchQueue? - private var currentRoute: AudioOutputRoute? - public var onAudioData: ((Data) -> Void)? + private var writeThread: DispatchQueue? + private var currentRoute: AudioOutputRoute? + public var onAudioData: ((Data) -> Void)? + + // 添加对外部类的弱引用 + private weak var parentHelper: AzureAsrHelper? + + // 添加初始化方法,接收外部类引用 + init(parentHelper: AzureAsrHelper) { + self.parentHelper = parentHelper + super.init() + } /** * 初始化 */ @@ -654,7 +665,7 @@ public class AzureAsrHelper: NSObject { // 创建新的写线程 writeThread = DispatchQueue(label: "audio.stream.writer") startWriteThread() - self.setAudioOutputRoute(.receiver) + //self.setAudioOutputRoute(.receiver) } /// 获取最佳音频格式 (iOS 通常支持标准采样率) @@ -717,7 +728,8 @@ public class AzureAsrHelper: NSObject { * @param data 音频数据字节数组 */ public func saveAudioDataTo(data: Data) { - if audioSourceType != .external { + // 通过父类引用调用方法 + if audioSourceType != .external || !(parentHelper?.isContinuousRecognitionActive() ?? false) { return } // print("外部data=\(data)") @@ -879,8 +891,7 @@ private func runMicrophoneCapture() { case bluetooth } public func setAudioOutputRoute(_ route: AudioOutputRoute) { - guard currentRoute != route else { return } - print("setAudioOutputRoute") + do { // if self.isWriting { @@ -890,14 +901,15 @@ private func runMicrophoneCapture() { try audioSession.setActive(false) } - + print("setAudioOutputRoutecurrent,route=\(route)") switch route { case .speaker: + print("setAudioOutputRoutecurrent,speaker") // 使用扬声器时必须用videoChat模式 try audioSession.setCategory( .playAndRecord, mode: .videoChat, - options: [.allowBluetooth, .mixWithOthers, .defaultToSpeaker] + options: [.defaultToSpeaker] ) try audioSession.overrideOutputAudioPort(.speaker) @@ -906,7 +918,7 @@ private func runMicrophoneCapture() { try audioSession.setCategory( .playAndRecord, mode: .voiceChat, - options: [.allowBluetooth, .mixWithOthers, .defaultToSpeaker] + options: [.allowBluetooth] ) try audioSession.overrideOutputAudioPort(.none) diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift index f07e56855..84c96f361 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift @@ -184,6 +184,15 @@ import os.log callback: currentAsrCallback! ) result(success) + + case "disableBluetoothAudio": + azureAsrHelper.disableBluetoothAudio(); + return + + case "restoreOriginalAudioState": + azureAsrHelper.restoreOriginalAudioState(); + return + case "stopContinuousRecognition": let success = azureAsrHelper.stopContinuousRecognition() @@ -224,7 +233,7 @@ import os.log return } do { - print("音频文件名称为: \(filePath)") + print("音频文件名称为: (filePath)") try azureAsrHelper.renameFile(filePath: filePath, newName: newName) result(true) } catch { @@ -239,7 +248,7 @@ import os.log return } do { - print("音频文件名称为: \(sourcePath)") + print("音频文件名称为: (sourcePath)") try azureAsrHelper.moveFile(sourcePath: sourcePath, destPath: destPath) result(true) } catch { @@ -253,7 +262,7 @@ import os.log return } do { - print("音频文件名称为: \(filePath)") + print("音频文件名称为: (filePath)") try azureAsrHelper.enableRecord(filePath: filePath) result(true) } catch { From fcce6427ff18ac8405e85399fd2f1cfb64fd3cc7 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Wed, 6 Aug 2025 20:59:04 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../chat_api/ios/chat_api/Sources/chat_api/MCPClient.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local_plugins/chat_api/ios/chat_api/Sources/chat_api/MCPClient.swift b/local_plugins/chat_api/ios/chat_api/Sources/chat_api/MCPClient.swift index 7e497bfbb..35a9df702 100644 --- a/local_plugins/chat_api/ios/chat_api/Sources/chat_api/MCPClient.swift +++ b/local_plugins/chat_api/ios/chat_api/Sources/chat_api/MCPClient.swift @@ -486,7 +486,7 @@ public class MCPClient { if let function = toolMap["function"] as? [String: Any], let name = function["name"] as? String, let description = function["description"] as? String { - // print("[MCPClient] \(index + 1). \(name) - \(description)") + print("[MCPClient] \(index + 1). \(name) - \(description)") } } From 29e77ac606a79ca9b617548c3f6d2663757845cd Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Thu, 7 Aug 2025 11:49:40 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=9D=90=E6=A0=87?= =?UTF-8?q?=E8=AF=BB=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Sources/azure_speech/AzureTtsHelper.swift | 2 +- .../Sources/chat_api/SystemFunctionHandler.swift | 11 ++++------- .../Sources/location_service/LocationData.swift | 4 +--- .../Sources/location_service/LocationService.swift | 12 +++++------- pubspec.yaml | 2 +- 5 files changed, 12 insertions(+), 19 deletions(-) diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index 3d6c25e73..86ac66bb9 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -166,7 +166,7 @@ public class AzureTtsHelper: NSObject, ITtsService { let currentText = streamBuffer // 标点符号列表 (与Android一致) - let punctuationMarks: [Character] = [".", "。", "!", "!", "?", "?", ";", ";", ",", ",", ":", ":", "\n"] + let punctuationMarks: [Character] = [".", "。", "!", "!","*", "?", "?", ";", ";", ",", ",", ":", ":", "\n"] // 从后往前查找最后一个标点符号 var lastPunctuationIndex = -1 diff --git a/local_plugins/chat_api/ios/chat_api/Sources/chat_api/SystemFunctionHandler.swift b/local_plugins/chat_api/ios/chat_api/Sources/chat_api/SystemFunctionHandler.swift index bcacf4eb5..e55b3c15f 100644 --- a/local_plugins/chat_api/ios/chat_api/Sources/chat_api/SystemFunctionHandler.swift +++ b/local_plugins/chat_api/ios/chat_api/Sources/chat_api/SystemFunctionHandler.swift @@ -462,8 +462,7 @@ private class GetCurrentLocationHandler: FunctionHandler { guard let latStr = locationInfo["user_lat"] as? String, let lngStr = locationInfo["user_lng"] as? String, let latitude = Double(latStr), - let longitude = Double(lngStr), - let address = locationInfo["user_location"] as? String else { + let longitude = Double(lngStr) else { print("[GetCurrentLocationHandler] 位置数据格式错误") continuation.resume(returning: self.createErrorResponse(message: "位置数据格式错误,请重试")) return @@ -471,8 +470,8 @@ private class GetCurrentLocationHandler: FunctionHandler { print("[GetCurrentLocationHandler] 位置获取成功: lat=\(latitude), lng=\(longitude)") - let message = "当前位置:\(address)(纬度 \(latitude),经度 \(longitude))" - let successResponse = self.createSuccessResponse(message: message, latitude: latitude, longitude: longitude, address: address) + // let message = "当前位置:\(address)(纬度 \(latitude),经度 \(longitude))" + let successResponse = self.createSuccessResponse(latitude: latitude, longitude: longitude) print("[GetCurrentLocationHandler] 返回位置结果") continuation.resume(returning: successResponse) @@ -486,14 +485,12 @@ private class GetCurrentLocationHandler: FunctionHandler { } } - private func createSuccessResponse(message: String, latitude: Double, longitude: Double, address: String) -> String { + private func createSuccessResponse(latitude: Double, longitude: Double) -> String { let result: [String: Any] = [ "success": true, - "message": message, "data": [ "latitude": latitude, "longitude": longitude, - "address": address, "timestamp": Date().timeIntervalSince1970 * 1000 ] ] diff --git a/local_plugins/location_service/ios/location_service/Sources/location_service/LocationData.swift b/local_plugins/location_service/ios/location_service/Sources/location_service/LocationData.swift index af7f9827c..4701388b1 100644 --- a/local_plugins/location_service/ios/location_service/Sources/location_service/LocationData.swift +++ b/local_plugins/location_service/ios/location_service/Sources/location_service/LocationData.swift @@ -4,13 +4,11 @@ import Foundation public struct LocationData: Codable { public let latitude: Double public let longitude: Double - public let address: String public let timestamp: TimeInterval - public init(latitude: Double, longitude: Double, address: String, timestamp: TimeInterval) { + public init(latitude: Double, longitude: Double,timestamp: TimeInterval) { self.latitude = latitude self.longitude = longitude - self.address = address self.timestamp = timestamp } } diff --git a/local_plugins/location_service/ios/location_service/Sources/location_service/LocationService.swift b/local_plugins/location_service/ios/location_service/Sources/location_service/LocationService.swift index 2312b1fc4..bc682d13e 100644 --- a/local_plugins/location_service/ios/location_service/Sources/location_service/LocationService.swift +++ b/local_plugins/location_service/ios/location_service/Sources/location_service/LocationService.swift @@ -190,11 +190,11 @@ public class LocationService: NSObject { } // 使用缓存的状态信息 - print("[LocationService] 开始位置请求") - print("[LocationService] 缓存的权限状态: \(self.authorizationStatus.rawValue)") - print("[LocationService] 缓存的位置服务状态: \(self.isLocationServicesEnabled)") - print("[LocationService] 当前线程: \(Thread.current)") - print("[LocationService] 是否主线程: \(Thread.isMainThread)") + // print("[LocationService] 开始位置请求") + // print("[LocationService] 缓存的权限状态: \(self.authorizationStatus.rawValue)") + // print("[LocationService] 缓存的位置服务状态: \(self.isLocationServicesEnabled)") + // print("[LocationService] 当前线程: \(Thread.current)") + // print("[LocationService] 是否主线程: \(Thread.isMainThread)") // 验证权限状态 if !self.checkLocationPermission() { @@ -238,7 +238,6 @@ public class LocationService: NSObject { let locationData = LocationData( latitude: latitude, longitude: longitude, - address: "", timestamp: Date().timeIntervalSince1970 * 1000 ) completion(.success(locationData)) @@ -273,7 +272,6 @@ public class LocationService: NSObject { return [ "user_lat": String(locationData.latitude), "user_lng": String(locationData.longitude), - "user_location": locationData.address, "timestamp": locationData.timestamp ] } diff --git a/pubspec.yaml b/pubspec.yaml index d59266ff8..ed31e55ad 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -55,7 +55,7 @@ dependencies: uuid: ^4.3.3 record: ^5.2.1 share_plus: ^11.0.0 - image_picker: ^1.1.2 + image_picker: ^1.1. audio_waveforms: ^1.3.0 font_awesome_flutter: ^10.8.0 lottie: ^3.3.1 From 96d82ee31777aa0778e46360a3faea351ec6d60e Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Thu, 7 Aug 2025 14:24:56 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E7=BB=8F=E7=BA=AC?= =?UTF-8?q?=E5=BA=A6=E6=95=B0=E6=8D=AE=E9=80=82=E9=85=8D=E9=AB=98=E5=BE=B7?= =?UTF-8?q?=E7=94=A8=E7=9A=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Sources/location_service/LocationData.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/local_plugins/location_service/ios/location_service/Sources/location_service/LocationData.swift b/local_plugins/location_service/ios/location_service/Sources/location_service/LocationData.swift index 4701388b1..5facd02e0 100644 --- a/local_plugins/location_service/ios/location_service/Sources/location_service/LocationData.swift +++ b/local_plugins/location_service/ios/location_service/Sources/location_service/LocationData.swift @@ -5,10 +5,12 @@ public struct LocationData: Codable { public let latitude: Double public let longitude: Double public let timestamp: TimeInterval + public let latitudeandlongitude: String public init(latitude: Double, longitude: Double,timestamp: TimeInterval) { self.latitude = latitude self.longitude = longitude self.timestamp = timestamp + self.latitudeandlongitude = "\(latitude),\(longitude)" } } From 3339fb84b68c1a7784dde1b07d2844d4ce82afc3 Mon Sep 17 00:00:00 2001 From: fdp <1286779656@qq.com> Date: Thu, 7 Aug 2025 17:13:51 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=80=80=E5=87=BA?= =?UTF-8?q?=E4=B8=8D=E6=89=93=E6=96=AD=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/modules/translation/controllers/translation_controller.dart | 2 +- .../azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift | 2 +- pubspec.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/modules/translation/controllers/translation_controller.dart b/lib/modules/translation/controllers/translation_controller.dart index ae649a20e..d3b3c1631 100644 --- a/lib/modules/translation/controllers/translation_controller.dart +++ b/lib/modules/translation/controllers/translation_controller.dart @@ -367,7 +367,7 @@ class TranslationController extends GetxController { @override void onClose() { - stopRecognition(); + stopAll(); _recognitionSubscription?.cancel(); _recognitionSubscription = null; _voiceInteractionSubscription?.cancel(); diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift index 84c96f361..5955ac7c8 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift @@ -359,7 +359,7 @@ import os.log let success = azureTtsHelper.flushStream() result(success) - case "stop": + case "stopSpeaking": let success = azureTtsHelper.stop() result(success) diff --git a/pubspec.yaml b/pubspec.yaml index ed31e55ad..d59266ff8 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -55,7 +55,7 @@ dependencies: uuid: ^4.3.3 record: ^5.2.1 share_plus: ^11.0.0 - image_picker: ^1.1. + image_picker: ^1.1.2 audio_waveforms: ^1.3.0 font_awesome_flutter: ^10.8.0 lottie: ^3.3.1 From b157782b7c8bdd7537d8aa6d2b10fc74df464bcf Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Thu, 7 Aug 2025 18:32:20 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E8=8B=B9=E6=9E=9C?= =?UTF-8?q?=E7=9A=84=E5=9B=BE=E7=89=87=E8=AF=86=E5=88=AB=E9=80=BB=E8=BE=91?= =?UTF-8?q?=E8=A1=A5=E5=85=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Sources/chat_api/ChatApiService.swift | 66 +++++++++++++++---- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/local_plugins/chat_api/ios/chat_api/Sources/chat_api/ChatApiService.swift b/local_plugins/chat_api/ios/chat_api/Sources/chat_api/ChatApiService.swift index 267f4ffff..524feda90 100644 --- a/local_plugins/chat_api/ios/chat_api/Sources/chat_api/ChatApiService.swift +++ b/local_plugins/chat_api/ios/chat_api/Sources/chat_api/ChatApiService.swift @@ -311,7 +311,7 @@ public func sendMessageStream(_ sessionId: String, messages: [[String: Any]]) { tools: chatTools ) - // print("[ChatApiService] 发起AI问答 问题:\(messages[0]) 工具数量: \(chatTools.count)") + print("[ChatApiService] 发起AI问答 问题:\(messages) 工具数量: \(chatTools.count)") } else { query = ChatQuery( messages: chatMessages, @@ -761,7 +761,44 @@ private func convertMapToJsonObject(_ map: [String: Any]) -> [String: Any] { } return false } - + // 解析并构建正确的图片消息 + func parseImageData(_ data: [String: Any]) -> ChatQuery.ChatCompletionMessageParam? { + // 1. 校验基础字段 + guard let type = data["type"] as? String, type == "image_url", + let imageUrlDict = data["image_url"] as? [String: Any], + let imageUrlString = imageUrlDict["url"] as? String else { + print("数据结构错误:缺少type或image_url.url") + return nil + } + + // 2. 补充默认的detail(原结构体必须,默认auto) + let detailString = imageUrlDict["detail"] as? String ?? "auto" + let imageDetail: ChatQuery.ChatCompletionMessageParam.UserMessageParam.Content.VisionContent.ChatCompletionContentPartImageParam.ImageURL.Detail + switch detailString.lowercased() { + case "low": + imageDetail = .low + case "high": + imageDetail = .high + default: + imageDetail = .auto + } + + // 3. 逐层构建结构体(按原结构体要求映射) + let imageUrlModel = ChatQuery.ChatCompletionMessageParam.UserMessageParam.Content.VisionContent.ChatCompletionContentPartImageParam.ImageURL( + url: imageUrlString, + detail: imageDetail + ) + + let imageContentPart = ChatQuery.ChatCompletionMessageParam.UserMessageParam.Content.VisionContent.ChatCompletionContentPartImageParam( + imageUrl: imageUrlModel + ) + + let visionContent = ChatQuery.ChatCompletionMessageParam.UserMessageParam.Content.VisionContent.chatCompletionContentPartImageParam(imageContentPart) + let userContent = ChatQuery.ChatCompletionMessageParam.UserMessageParam.Content.vision([visionContent]) + let userMessageParam = ChatQuery.ChatCompletionMessageParam.UserMessageParam(content: userContent) + + return .user(userMessageParam) + } /// 转换为OpenAI Message格式 private func convertToChatMessages(_ messages: [[String: Any]]) throws -> [ChatQuery.ChatCompletionMessageParam] { var chatMessages: [ChatQuery.ChatCompletionMessageParam] = [] @@ -851,17 +888,24 @@ private func convertMapToJsonObject(_ map: [String: Any]) -> [String: Any] { var textContent = "" for item in contentArray { - if let type = item["type"] as? String, type == "text", - let text = item["text"] as? String { - textContent += text + if let type = item["type"] as? String { + switch type { + case "text": + if let text = item["text"] as? String { + if let chatMessage = ChatQuery.ChatCompletionMessageParam(role: messageRole, content: text) { + chatMessages.append(chatMessage) + } + } + case "image_url": + if let imageMessage = parseImageData(item) { + chatMessages.append(imageMessage) + print("图片消息解析成功") + } + default: + break + } } } - - let finalContent = textContent.isEmpty ? "多媒体内容" : textContent - - if let chatMessage = ChatQuery.ChatCompletionMessageParam(role: messageRole, content: finalContent) { - chatMessages.append(chatMessage) - } } } From 5c5f5909ab5e69fb60c075b7693b7b3e790aa476 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Thu, 7 Aug 2025 21:33:02 +0800 Subject: [PATCH 7/7] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E8=81=8A=E5=A4=A9?= =?UTF-8?q?=E5=9B=BE=E7=89=87=E6=98=BE=E7=A4=BA=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/data/models/message_model.dart | 4 ++ .../agent/controllers/agent_controller.dart | 41 +++++++++++++------ lib/modules/agent/views/message_bubble.dart | 4 +- .../agent_service/AgentService.kt | 4 +- .../agent_service/AgentServiceImpl.swift | 17 +++++--- .../chat_storage/ChatStorageHelper.swift | 6 +-- 6 files changed, 50 insertions(+), 26 deletions(-) diff --git a/lib/data/models/message_model.dart b/lib/data/models/message_model.dart index aef99cbc9..09744a939 100644 --- a/lib/data/models/message_model.dart +++ b/lib/data/models/message_model.dart @@ -1,3 +1,7 @@ +import 'dart:io'; + +import 'package:path_provider/path_provider.dart'; + class Message { final bool isUser; //标识信息是否来自用户还是AI final String text; //信息文本内容 diff --git a/lib/modules/agent/controllers/agent_controller.dart b/lib/modules/agent/controllers/agent_controller.dart index 8debed827..a6a5730f7 100644 --- a/lib/modules/agent/controllers/agent_controller.dart +++ b/lib/modules/agent/controllers/agent_controller.dart @@ -126,7 +126,7 @@ class AgentController extends GetxController { String _currentResponseId = ''; final TAG = 'AgentController'; - + var AppDir = "".obs; // 会话ID final String sessionId = 'default_agent'; @@ -153,6 +153,7 @@ class AgentController extends GetxController { @override void onInit() { super.onInit(); + _initAppDir(); _loadTtsPreference(); //加载TTS偏好设置 _setupScrollListener(); //设置滚动监听器 _loadChatHistory(); //加载聊天历史记录 @@ -164,6 +165,10 @@ class AgentController extends GetxController { startVoiceInput(); //开启语音输入 } + Future _initAppDir() async { + AppDir.value = (await getApplicationDocumentsDirectory()).path; + } + // 初始化音乐播放状态 Future _initMusicStatus() async { try { @@ -291,7 +296,7 @@ class AgentController extends GetxController { page: 1, //从第一页开始 pageSize: pageSize, //每页20条记录 ); - + Logger.i(TAG, '加载聊天历史记录: $result'); if (result.messages.isNotEmpty) { // 转换并添加消息到列表 final historyMessages = _convertChatMessagesToMessages(result.messages); @@ -303,7 +308,8 @@ class AgentController extends GetxController { // 更新分页状态 currentPage.value = 1; hasMoreMessages.value = result.totalPages > 1; - + print( + 'liwei-------------读取历史记录 ${AppDir.value}${historyMessages[0].imagePath}'); Logger.i(TAG, '成功加载 ${historyMessages.length} 条聊天记录,总页数: ${result.totalPages}'); _logMessageDetails(historyMessages); @@ -421,7 +427,7 @@ class AgentController extends GetxController { 'meta: ${msg.meta != null ? "有" : "无"}' '}'); } - Logger.d(TAG, '===== 历史消息详情结束 ====='); + Logger.d(TAG, '===== 历史消息详情结束 ==lishi==='); } // 设置滚动监听器 @@ -903,10 +909,14 @@ class AgentController extends GetxController { } // 复制图片到应用永久存储目录 - final permanentImagePath = await _copyImageToPermanentStorage(imagePath); + // 复制图片到应用永久存储目录 + final imagePathResult = await _copyImageToPermanentStorage(imagePath); + final permanentImagePath = imagePathResult['fullPath']!; + final relativeImagePath = imagePathResult['relativePath']!; var ImageUrl = await UploadOss.upload( - filepath: imagePath, rootDir: 'User/${User.instance.uid}/ChatImages'); - Logger.i(TAG, '图片上传: $imagePath URL: $ImageUrl'); + filepath: permanentImagePath, + rootDir: 'User/${User.instance.uid}/ChatImages'); + Logger.i(TAG, '图片上传: $permanentImagePath URL: $ImageUrl'); // 显示的文本,如果没有提供则使用默认值 final displayText = text?.isNotEmpty == true ? text! : '[图片]'; @@ -915,7 +925,7 @@ class AgentController extends GetxController { isUser: true, text: displayText, hasImage: true, - imagePath: permanentImagePath, + imagePath: relativeImagePath, ); messages.add(message); @@ -935,7 +945,7 @@ class AgentController extends GetxController { // 调用Agent Service处理图片,考虑TTS状态 await AgentService.processImageInput( - permanentImagePath, + relativeImagePath, ImageUrl, text: text ?? '', speakResponse: isTtsEnabled.value, @@ -948,12 +958,16 @@ class AgentController extends GetxController { } // 将图片复制到应用的永久存储目录 - Future _copyImageToPermanentStorage(String sourcePath) async { + Future> _copyImageToPermanentStorage( + String sourcePath) async { try { final File sourceFile = File(sourcePath); if (!await sourceFile.exists()) { Logger.e(TAG, '源图片文件不存在: $sourcePath'); - return sourcePath; // 如果源文件不存在,返回原路径 + return { + 'fullPath': sourcePath, + 'relativePath': sourcePath + }; // 如果源文件不存在,返回原路径 } // 创建应用文档目录下的chatImages文件夹 @@ -967,15 +981,16 @@ class AgentController extends GetxController { final fileName = '${DateTime.now().millisecondsSinceEpoch}_${sourcePath.split('/').last}'; final targetPath = '${imagesDir.path}/$fileName'; + final relativePath = '/chatImages/$fileName'; // 复制文件 await sourceFile.copy(targetPath); Logger.i(TAG, '图片已复制到永久存储: $targetPath'); - return targetPath; + return {'fullPath': targetPath, 'relativePath': relativePath}; } catch (e) { Logger.e(TAG, '复制图片到永久存储失败: $e'); - return sourcePath; // 出错时返回原路径 + return {'fullPath': sourcePath, 'relativePath': sourcePath}; // 出错时返回原路径 } } diff --git a/lib/modules/agent/views/message_bubble.dart b/lib/modules/agent/views/message_bubble.dart index 6099dc1f0..660f8c7d4 100644 --- a/lib/modules/agent/views/message_bubble.dart +++ b/lib/modules/agent/views/message_bubble.dart @@ -169,7 +169,7 @@ class MessageBubble extends GetView { child: ClipRRect( borderRadius: BorderRadius.circular(12), child: Image.file( - File(imagePath!), + File('${controller.AppDir.value}$imagePath'), width: 200, height: 200, fit: BoxFit.cover, @@ -202,7 +202,7 @@ class MessageBubble extends GetView { child: ClipRRect( borderRadius: BorderRadius.circular(12), child: Image.file( - File(imagePath!), + File('${controller.AppDir.value}$imagePath'), width: 200, height: 200, fit: BoxFit.cover, diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index cd5762b9d..d084b19fc 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -526,8 +526,8 @@ object AgentService : CoroutineScope { _hasSpeechDetected.set(true) if (!previousHasSpeech) { - restartIdleCheck() - } + restartIdleCheck() + } sendEvent( "recognizing", mapOf( diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 65fe56f08..b6a154a9f 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -469,8 +469,8 @@ audioStream.saveAudioDataTo(data: audioData) processWithChatApiServiceInternal(userMessage: userMessage, displayText: text, speakResponse: speakResponse) } - private func processWithChatApiServiceInternal(userMessage: [String: Any], displayText: String, speakResponse: Bool = true, hasImage: Bool = false) { - os_log("开始处理ChatAPI请求: displayText=%{public}@, speakResponse=%{public}@, hasImage=%{public}@", + private func processWithChatApiServiceInternal(userMessage: [String: Any], displayText: String, speakResponse: Bool = true, hasImage: Bool = false,imagePath:String = "") { + os_log("开始处理ChatAPI请求: displayText=%{public}@, speakResponse=%{public}@, hasImage=%{public}@", log: logger, type: .info, displayText, speakResponse ? "是" : "否", hasImage ? "是" : "否") guard let chatApiService = chatApiService else { @@ -517,7 +517,8 @@ audioStream.saveAudioDataTo(data: audioData) agentService: self, speakResponse: speakResponse, displayText: displayText, - hasImage: hasImage + hasImage: hasImage, + imagePath: imagePath ) currsessionId = UUID().uuidString os_log("设置回调并调用sendMessageStream", log: logger, type: .info) @@ -574,7 +575,8 @@ audioStream.saveAudioDataTo(data: audioData) userMessage: userMessage, displayText: displayText, speakResponse: speakResponse, - hasImage: true + hasImage: true, + imagePath: imagePath ) } } @@ -1132,19 +1134,22 @@ class ChatApiStreamCallback: StreamCallback { private let speakResponse: Bool private let displayText: String private let hasImage: Bool + private let imagePath: String private var responseBuilder = "" private var metadata = "" private var userMetadata:[String: Any] = [:] private var reply = true; //是否回复 private var broadcast = true //是否播发 - init(agentService: AgentServiceImpl, speakResponse: Bool, displayText: String, hasImage: Bool) { + init(agentService: AgentServiceImpl, speakResponse: Bool, displayText: String, hasImage: Bool,imagePath:String) { self.agentService = agentService self.speakResponse = speakResponse self.displayText = displayText self.hasImage = hasImage + self.imagePath = imagePath + self.userMetadata["hasImage"] = hasImage + self.userMetadata["imagePath"] = imagePath - } func onUsage(_ sessionid: String,_ prompt_tokens: Int?,_ completion_tokens: Int?,_ total_tokens: Int?){ diff --git a/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift b/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift index eb3e38433..96fa34a8e 100644 --- a/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift +++ b/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift @@ -186,7 +186,7 @@ public class ChatStorageHelper { "totalCount": totalCount, "totalPages": totalPages ] - + os_log("getMessages: 读取历史记录: %{public}@", log: logger, type: .error, messagesArray) do { let jsonData = try JSONSerialization.data(withJSONObject: result, options: []) if let jsonString = String(data: jsonData, encoding: .utf8) { @@ -199,7 +199,7 @@ public class ChatStorageHelper { let errmsg = String(cString: sqlite3_errmsg(db)!) os_log("getMessages: SQL准备失败: %{public}@", log: logger, type: .error, errmsg) } - + // 返回空结果,但保持格式一致 let emptyResult: [String: Any] = [ "messages": [], @@ -370,4 +370,4 @@ public class ChatStorageHelper { return messages } -} \ No newline at end of file +}