liwei1dao
/
deepvoice_client
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

加入 音视频翻译，录音文件。优化ui

weicu
fdp 1 year ago
parent
7fb514178b
commit
3b4c8b5c81
12 changed files with 851 additions and 604 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      lib/data/services/asr_service.dart
  2. 26
      lib/data/services/speech_impl/azure_asr_service.dart
  3. 14
      lib/modules/ota_upgrade/controllers/ota_upgrade_controller.dart
  4. 20
      lib/modules/translation/controllers/translation_controller.dart
  5. 6
      lib/modules/translation/views/translation_view.dart
  6. 2
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt
  7. 1
      local_plugins/azure_speech/android/build.gradle.kts
  8. 477
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
  9. 833
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt
  10. 5
      local_plugins/azure_speech/pubspec.yaml
  11. 22
      local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt
  12. 43
      local_plugins/ota/android/src/main/kotlin/com/example/ota/OtaPlugin.kt

6
lib/data/services/asr_service.dart
View File

@ -29,6 +29,12 @@ abstract class AsrService {
/// 检查连续识别是否活跃
bool isContinuousRecognitionActive();
/// 开启录音
Future<bool> enableRecord();
/// 停止录音
Future<bool> stopRecord();
/// 释放资源
Future<void> dispose();
}

26
lib/data/services/speech_impl/azure_asr_service.dart
View File

@ -342,4 +342,28 @@ class AzureAsrService extends GetxService implements AsrService {
return false;
}
}
}
@override
Future<bool> enableRecord() async {
try {
final bool result = await _channel.invokeMethod('enableRecord');
return result;
} catch (e) {
Logger.error('开始录音: ${e.toString()}');
rethrow;
}
}
@override
Future<bool> stopRecord() async {
try {
final bool result = await _channel.invokeMethod('stopRecord');
return result;
} catch (e) {
Logger.error('停止录音: ${e.toString()}');
rethrow;
}
}
}

14
lib/modules/ota_upgrade/controllers/ota_upgrade_controller.dart
View File

@ -149,8 +149,20 @@ class OtaUpgradeController extends GetxController {
deviceInfo.firmwareProductId.isEmpty) {
throw "获取设备信息失败，请检查连接";
}
// 拆分 firmwareVersion 并判断两边是否一致
final firmwareParts = deviceInfo.firmwareVersion.split('.');
// 从第三个 `.` 开始分割
final firstPart = firmwareParts.sublist(0, 3).join('.');
final secondPart = firmwareParts.sublist(3).join('.');
print("firstPart${firstPart},secondPart=${secondPart}");
// 判断两部分是否一致
if (firstPart != secondPart) {
throw "固件版本格式错误：${deviceInfo.firmwareVersion}";
}
// 正常流程
currentVersion.value = deviceInfo.firmwareVersion;
currentVersion.value = firstPart;
print('getequipmentinfo,id: ${deviceInfo.firmwareProductId}');
// API请求
final response = await Api.getequipmentinfo({

20
lib/modules/translation/controllers/translation_controller.dart
View File

@ -33,7 +33,7 @@ class TranslationController extends GetxController {
final isRecognizing = false.obs;
final isTranslating = false.obs;
final isTtsEnabled = true.obs;
final isRecording = false.obs;
// 语言相关
final sourceLanguage = '中文(简体)'.obs;
final targetLanguage = '英语'.obs;
@ -193,6 +193,16 @@ class TranslationController extends GetxController {
// 切换TTS功能
void toggleTts() => isTtsEnabled.toggle();
// 切换录音功能
Future<void> toggleisRecord() async {
isRecording.toggle();
if (isRecording.value) {
await _asrService.enableRecord();
} else {
await _asrService.stopRecord();
}
}
// 开始语音识别
Future<void> startRecognition() async {
Logger.info('开始语音识别');
@ -233,15 +243,18 @@ class TranslationController extends GetxController {
_bleManager.openDecoder();
// 发送ble音乐或者通话远端声音
_audioSourceType = true;
isTtsEnabled.value = false; // 禁用TTS以避免干扰
Logger.info('发送ble音乐或者通话远端声音');
} else if (currentMode.value == "call") {
// 发送ble系统mic和dac（音乐或者通话远端）声音
_bleManager.openA2DPDecoder();
_audioSourceType = true;
isTtsEnabled.value = true;
Logger.info('发送ble系统mic和dac（音乐或者通话远端）声音');
} else {
// 开始连续语音识别
_audioSourceType = false;
isTtsEnabled.value = true;
}
final recognitionStream =
await _asrService.startContinuousRecognition(_audioSourceType);
@ -476,6 +489,11 @@ class TranslationController extends GetxController {
// 停止所有操作
Future<void> stopAll() async {
try {
if (currentMode.value == "audioVideo" || currentMode.value == "call") {
_bleManager.closeCodec();
// 发送ble音乐或者通话远端声音
Logger.info('发送ble音乐或者通话远端声音');
}
if (isRecognizing.value) {
await stopRecognition();
}

6
lib/modules/translation/views/translation_view.dart
View File

@ -87,13 +87,13 @@ class TranslationView extends GetView<TranslationController> {
// TTS toggle button
Obx(() => IconButton(
icon: Icon(
controller.isTtsEnabled.value ? Icons.mic : Icons.mic_off,
controller.isRecording.value ? Icons.mic : Icons.mic_off,
color: isDarkMode ? Colors.white : Colors.black,
size: 24.sp,
),
// onPressed: controller.toggleTts,默认播放
onPressed: controller.toggleTts,
tooltip: '自动播放翻译结果',
onPressed: controller.toggleisRecord,
tooltip: '是否开启录音',
)),
// 历史记录按钮

2
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt
View File

@ -1,8 +1,6 @@
package com.yunqiinnovation.agent_service
import android.content.Context
import com.yunqiinnovation.agent_service.AgentService
import com.yunqiinnovation.agent_service.AgentServiceListener
import android.util.Log
import com.yunqiinnovation.ble_service.BleService
import com.yunqiinnovation.ble_service.BleConst

1
local_plugins/azure_speech/android/build.gradle.kts
View File

@ -54,5 +54,6 @@ dependencies {
// 添加Microsoft语音SDK
implementation("com.microsoft.cognitiveservices.speech:client-sdk:1.43.0")
implementation(project(":speech"))
add("compileOnly", project(":ble_service"))
}

477
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
View File

@ -28,51 +28,56 @@ import java.util.concurrent.TimeUnit
import java.io.File
import java.io.FileOutputStream
import java.io.RandomAccessFile
/**
* Azure语音识别辅助类，支持麦克风和外部音频源的单次和连续语音识别
*
*
* 针对回音消除的场景，使用拉流模式 (PullAudioInputStreamCallback) 实现，避免复杂的协程处理
* 通过检测扬声器和耳机状态，自动决定是否启用回音消除
*/
class AzureAsrHelper(private val context: Context) {
private val tag = "AzureAsrHelper"
// 核心组件
private var speechConfig: SpeechConfig? = null
private var recognizer: SpeechRecognizer? = null
private var audioConfig: AudioConfig? = null
// 状态管理
private var isContinuousRecognitionActive = false
// 配置参数
private var currentLanguage = "zh-CN"
private var supportedLanguages = arrayOf("zh-CN")
private var isAutoDetectLanguage = false
private var subscriptionKey = ""
private var region = ""
// 音频源配置
private var audioSourceType = AudioSourceType.MICROPHONE
var audioSourceType = AudioSourceType.MICROPHONE
// 音频处理
private var microphoneStream: MicrophoneStream? = null
private var externalAudioStream: ExternalAudioPullStream? = null
// 录音文件处理
private var recordfile: RecordingFile? = null
private var isRecording = false
/**
* 音频来源类型
*/
enum class AudioSourceType {
/** 使用设备麦克风 */
MICROPHONE,
/** 使用外部提供的音频数据 */
EXTERNAL
}
/**
* 初始化Azure语音服务
*
*
* @param subscriptionKey Azure 订阅密钥
* @param region Azure 区域
* @param supportedLanguages 支持的语言数组
@ -91,28 +96,28 @@ class AzureAsrHelper(private val context: Context) {
Log.e(tag, "Azure 配置信息不完整")
return false
}
// 释放之前的资源
dispose()
// 保存配置
this.subscriptionKey = subscriptionKey
this.region = region
this.audioSourceType = audioSourceType
// 设置语言
if (supportedLanguages.isNotEmpty()) {
this.supportedLanguages = supportedLanguages
}
// 根据支持的语言数量决定是否启用自动语言检测
this.isAutoDetectLanguage = supportedLanguages.size >= 2
// 如果只有一种语言，设置为当前语言
if (!isAutoDetectLanguage && supportedLanguages.isNotEmpty()) {
this.currentLanguage = supportedLanguages[0]
}
// 创建语音配置
speechConfig = SpeechConfig.fromSubscription(subscriptionKey, region).apply {
if (isAutoDetectLanguage) {
@ -122,15 +127,16 @@ class AzureAsrHelper(private val context: Context) {
// 设置指定的识别语言
speechRecognitionLanguage = currentLanguage
}
// 设置静音超时时间（毫秒）
setProperty("SpeechServiceConnection_EndSilenceTimeoutMs", "800")
setProperty("Speech_SegmentationSilenceTimeoutMs", "800")
// 设置分段策略为时间模式
setProperty("Speech_SegmentationStrategy", "Time")
}
// 录音文件类
recordfile = RecordingFile()
// 创建识别器
return setupRecognizer()
} catch (e: Exception) {
@ -138,7 +144,7 @@ class AzureAsrHelper(private val context: Context) {
return false
}
}
/**
* 设置识别器
*/
@ -147,17 +153,18 @@ class AzureAsrHelper(private val context: Context) {
// 清理旧的识别器
recognizer?.close()
recognizer = null
// 设置音频配置
when (audioSourceType) {
AudioSourceType.MICROPHONE -> {
// 判断是否使用手机扬声器输出和耳机连接状态
val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
val audioManager =
context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
val isSpeakerphoneOn = audioManager.isSpeakerphoneOn
// 检查是否连接了有线耳机
val isWiredHeadsetOn = audioManager.isWiredHeadsetOn
// 检查是否连接了蓝牙设备(需要注意兼容性)
val isBluetoothConnected = try {
val method = AudioManager::class.java.getMethod("isBluetoothA2dpOn")
@ -165,12 +172,12 @@ class AzureAsrHelper(private val context: Context) {
} catch (e: Exception) {
false
}
val isHeadsetConnected = isWiredHeadsetOn || isBluetoothConnected
// 如果使用手机扬声器输出且没有连接耳机，可能会产生回音，需启动回音消除功能
val needEchoCancellation = !isHeadsetConnected
if (needEchoCancellation) {
// 使用拉流方式进行回音消除
setupMicrophoneStream()
@ -178,20 +185,22 @@ class AzureAsrHelper(private val context: Context) {
audioConfig = AudioConfig.fromDefaultMicrophoneInput()
}
}
AudioSourceType.EXTERNAL -> {
// 改用拉流方式处理外部音频
setupExternalAudioStream()
}
}
// 创建识别器
recognizer = if (isAutoDetectLanguage) {
val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList())
val autoDetectConfig =
AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList())
SpeechRecognizer(speechConfig, autoDetectConfig, audioConfig)
} else {
SpeechRecognizer(speechConfig, audioConfig)
}
return true
} catch (e: Exception) {
Log.e(tag, "创建识别器失败: ${e.message}")
@ -199,7 +208,7 @@ class AzureAsrHelper(private val context: Context) {
return false
}
}
/**
* 设置麦克风流 - 使用拉流方式
*/
@ -207,7 +216,7 @@ class AzureAsrHelper(private val context: Context) {
try {
// 创建麦克风流
microphoneStream = MicrophoneStream()
// 创建音频配置 - 正确使用fromStreamInput方法，只传入回调
audioConfig = AudioConfig.fromStreamInput(microphoneStream)
} catch (e: Exception) {
@ -215,7 +224,7 @@ class AzureAsrHelper(private val context: Context) {
microphoneStream = null
}
}
/**
* 设置外部音频流 - 使用拉流方式
*/
@ -223,7 +232,7 @@ class AzureAsrHelper(private val context: Context) {
try {
// 创建外部音频拉流对象
externalAudioStream = ExternalAudioPullStream()
// 创建音频配置
audioConfig = AudioConfig.fromStreamInput(externalAudioStream)
} catch (e: Exception) {
@ -231,56 +240,61 @@ class AzureAsrHelper(private val context: Context) {
externalAudioStream = null
}
}
/**
* 向音频流写入音频数据
* 仅当音频源设置为EXTERNAL时有效
*
*
* @param data 音频数据字节数组
*/
fun pushAudioData(data: ByteArray) {
if (audioSourceType != AudioSourceType.EXTERNAL) {
return
}
// 使用拉流模式，将数据推入队列
externalAudioStream?.pushAudio(data)
}
/**
* 执行一次性语音识别
*
*
* @param callback 识别结果回调
*/
fun recognizeOnce(callback: RecognizeCallback, audioSourceType: AudioSourceType = AudioSourceType.MICROPHONE) {
fun recognizeOnce(
callback: RecognizeCallback,
audioSourceType: AudioSourceType = AudioSourceType.MICROPHONE
) {
if (speechConfig == null) {
callback.onError("语音服务未初始化")
return
}
this.audioSourceType = audioSourceType
this.audioSourceType = audioSourceType
// 确保不在连续识别中
if (isContinuousRecognitionActive) {
stopContinuousRecognition()
}
// 重置识别器
if (!setupRecognizer()) {
callback.onError("重置识别器失败")
return
}
try {
// 启动音频处理
startAudioProcessing()
// 执行同步识别
val result = recognizer?.recognizeOnceAsync()?.get()
// 停止音频处理
stopAudioProcessing()
if (result != null && result.reason == ResultReason.RecognizedSpeech) {
val detectedLanguage = AutoDetectSourceLanguageResult.fromResult(result)?.language ?: supportedLanguages[0]
val detectedLanguage = AutoDetectSourceLanguageResult.fromResult(result)?.language
?: supportedLanguages[0]
callback.onResult(result.text, detectedLanguage)
} else {
callback.onError("未能识别语音")
@ -290,19 +304,22 @@ class AzureAsrHelper(private val context: Context) {
callback.onError("识别异常: ${e.message}")
}
}
/**
* 开始连续语音识别
*
*
* @param callback 连续识别结果回调
* @return 是否成功开始识别
*/
fun startContinuousRecognition(callback: ContinuousRecognizeCallback, audioSourceType: AudioSourceType = AudioSourceType.MICROPHONE): Boolean {
fun startContinuousRecognition(
callback: ContinuousRecognizeCallback,
audioSourceType: AudioSourceType = AudioSourceType.MICROPHONE
): Boolean {
if (speechConfig == null) {
callback.onError("语音服务未初始化")
return false
}
if (isContinuousRecognitionActive) {
return true
}
@ -312,18 +329,21 @@ class AzureAsrHelper(private val context: Context) {
callback.onError("重置识别器失败")
return false
}
try {
// 设置各种事件监听
setupEventListeners(callback)
// 启动音频处理
startAudioProcessing()
// 开始连续识别
recognizer?.startContinuousRecognitionAsync()
isContinuousRecognitionActive = true
Log.e(tag, "开始连续语音识别 是否录音${isRecording}")
if (isRecording) {
enableRecord()
}
return true
} catch (e: Exception) {
stopAudioProcessing()
@ -332,7 +352,7 @@ class AzureAsrHelper(private val context: Context) {
return false
}
}
/**
* 设置事件监听器
*/
@ -340,7 +360,8 @@ class AzureAsrHelper(private val context: Context) {
// 识别中事件
recognizer?.recognizing?.addEventListener(
EventHandler<SpeechRecognitionEventArgs> { _, event ->
val detectedLanguage = AutoDetectSourceLanguageResult.fromResult(event.result)?.language ?: ""
val detectedLanguage =
AutoDetectSourceLanguageResult.fromResult(event.result)?.language ?: ""
// 直接在当前线程调用回调
callback.onRecognizing(event.result.text, detectedLanguage)
}
@ -350,7 +371,8 @@ class AzureAsrHelper(private val context: Context) {
recognizer?.recognized?.addEventListener(
EventHandler<SpeechRecognitionEventArgs> { _, event ->
if (event.result.reason == ResultReason.RecognizedSpeech) {
val detectedLanguage = AutoDetectSourceLanguageResult.fromResult(event.result)?.language ?: ""
val detectedLanguage =
AutoDetectSourceLanguageResult.fromResult(event.result)?.language ?: ""
// 直接在当前线程调用回调
callback.onResult(event.result.text, detectedLanguage)
}
@ -385,7 +407,7 @@ class AzureAsrHelper(private val context: Context) {
)
}
/**
* 停止连续语音识别
*
@ -396,11 +418,11 @@ class AzureAsrHelper(private val context: Context) {
Log.e(tag, "语音服务未初始化")
return false
}
if (!isContinuousRecognitionActive) {
return true
}
try {
if (recognizer == null) {
Log.w(tag, "识别器为空，重置状态")
@ -410,23 +432,24 @@ class AzureAsrHelper(private val context: Context) {
if (audioSourceType == AudioSourceType.EXTERNAL) {
pushAudioData(ByteArray(0))
}
// 直接停止连续识别（SDK内部已是异步操作）
recognizer?.stopContinuousRecognitionAsync()?.get(1000, TimeUnit.MILLISECONDS)
// 停止音频处理
stopAudioProcessing()
// 停止录音
recordfile!!.closeFile()
// 会话结束事件会设置isContinuousRecognitionActive = false
return true
} catch (e: Exception) {
// 强制重置状态
isContinuousRecognitionActive = false
Log.e(tag, "停止连续识别失败: ${e.message}")
// 停止音频处理
stopAudioProcessing()
recordfile!!.closeFile()
// 尝试强制关闭识别器
try {
recognizer?.close()
@ -434,16 +457,16 @@ class AzureAsrHelper(private val context: Context) {
} catch (ex: Exception) {
Log.e(tag, "关闭识别器失败: ${ex.message}")
}
return false
}
}
/**
* 检查连续识别是否活跃
*/
fun isContinuousRecognitionActive(): Boolean = isContinuousRecognitionActive
/**
* 释放所有资源
*/
@ -455,22 +478,23 @@ class AzureAsrHelper(private val context: Context) {
recognizer?.stopContinuousRecognitionAsync()
isContinuousRecognitionActive = false
}
// 停止音频处理
stopAudioProcessing()
// 停止录音
recordfile!!.closeFile()
// 释放recognizer
recognizer?.close()
recognizer = null
// 释放speechConfig
speechConfig?.close()
speechConfig = null
// 释放音频配置
audioConfig?.close()
audioConfig = null
// 确保状态被重置
isContinuousRecognitionActive = false
microphoneStream = null
@ -488,9 +512,9 @@ class AzureAsrHelper(private val context: Context) {
speechConfig = null
}
}
// 音频处理相关方法
/**
* 启动音频处理
*/
@ -500,12 +524,13 @@ class AzureAsrHelper(private val context: Context) {
// 拉流模式不需要额外启动，SDK会自动拉取数据
// 无需执行任何操作
}
AudioSourceType.EXTERNAL -> {
// 外部音频数据模式下不需要启动处理，等待外部调用pushAudioData
}
}
}
/**
* 停止音频处理
*/
@ -519,7 +544,7 @@ class AzureAsrHelper(private val context: Context) {
e.printStackTrace()
}
}
externalAudioStream?.let {
try {
it.close()
@ -530,7 +555,31 @@ class AzureAsrHelper(private val context: Context) {
}
}
}
/**
* 开启录音
*/
fun enableRecord() {
Log.i(tag, "开启录音:")
isRecording = true
recordfile!!.closeFile()
if (isContinuousRecognitionActive) {
recordfile!!.creatingFiles()
}
}
/**
* 停止录音
*/
fun stopRecord() {
Log.i(tag, "停止连续录音:")
isRecording = false
recordfile!!.closeFile()
}
/**
* 麦克风流 - 拉流模式
* 实现PullAudioInputStreamCallback，为Azure SDK提供音频数据
@ -538,12 +587,7 @@ class AzureAsrHelper(private val context: Context) {
private inner class MicrophoneStream : PullAudioInputStreamCallback() {
private var audioRecord: AudioRecord? = null
private var echoCanceler: AcousticEchoCanceler? = null
private var currentAudioFile: File? = null
private var fos: FileOutputStream? = null
// 用于存储音频数据的缓冲区
private val dataBuffer = mutableListOf<ByteArray>()
private var totalBytesWritten = 0
// 音频配置
private val sampleRate = 16000
private val channelConfig = AudioFormat.CHANNEL_IN_MONO
@ -551,30 +595,16 @@ class AzureAsrHelper(private val context: Context) {
private val bufferSize = AudioRecord.getMinBufferSize(
sampleRate, channelConfig, audioFormat
).let { if (it < 0) 4096 else it * 2 }
init {
initMicrophone()
}
/**
* 初始化麦克风
*/
private fun initMicrophone() {
try {
// 创建新的音频文件
val timestamp = System.currentTimeMillis()
val filePath = context.getExternalFilesDir(null)?.absolutePath + "/recorded_audio_$timestamp.wav"
currentAudioFile = File(filePath)
currentAudioFile?.createNewFile()
// 仅写入初始文件头
fos = FileOutputStream(currentAudioFile).apply {
write(generateWavHeader(0)) // 初始0长度
close()
}
// 追加音频数据到文件
fos = FileOutputStream(currentAudioFile, true)
// 创建录音对象
if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.M) {
val format = AudioFormat.Builder()
@ -582,7 +612,7 @@ class AzureAsrHelper(private val context: Context) {
.setEncoding(audioFormat)
.setChannelMask(channelConfig)
.build()
audioRecord = AudioRecord.Builder()
.setAudioSource(MediaRecorder.AudioSource.VOICE_COMMUNICATION)
.setAudioFormat(format)
@ -597,18 +627,18 @@ class AzureAsrHelper(private val context: Context) {
bufferSize
)
}
// 检查初始化状态
if (audioRecord?.state != AudioRecord.STATE_INITIALIZED) {
throw IllegalStateException("AudioRecord初始化失败")
}
// 应用音频效果
applyAudioEffects()
// 开始录音
audioRecord?.startRecording()
// 检查录音状态
if (audioRecord?.recordingState != AudioRecord.RECORDSTATE_RECORDING) {
throw IllegalStateException("录音启动失败")
@ -617,23 +647,24 @@ class AzureAsrHelper(private val context: Context) {
e.printStackTrace()
}
}
/**
* 应用音频效果
*/
private fun applyAudioEffects() {
val sessionId = audioRecord?.audioSessionId ?: return
// 回音消除
echoCanceler = AcousticEchoCanceler.create(sessionId)
echoCanceler?.enabled = true;
}
/**
* 获取音频格式
*/
fun getFormat(): AudioStreamFormat = AudioStreamFormat.getWaveFormatPCM(16000.toLong(), 16.toShort(), 1.toShort())
fun getFormat(): AudioStreamFormat =
AudioStreamFormat.getWaveFormatPCM(16000.toLong(), 16.toShort(), 1.toShort())
/**
* Azure SDK 调用此方法获取音频数据
* 这是拉流模式的核心方法，由SDK调用以获取音频数据
@ -645,82 +676,25 @@ class AzureAsrHelper(private val context: Context) {
Log.e(tag, "读取音频数据失败: $result")
return 0
}
// 将音频数据保存成wav格式的音频文件
saveAudioDataToWav(buffer, result)
Log.i(tag, "麦克风:")
// 将音频数据保存成wav格式的音频文件
recordfile?.saveAudioDataToWav(buffer)
return result
} catch (e: Exception) {
Log.e(tag, "读取音频异常: ${e.message}")
return 0
}
}
/**
* 保存音频数据到 WAV 文件
*/
private fun saveAudioDataToWav(buffer: ByteArray, length: Int) {
// 复制数据到新数组
val copy = ByteArray(length)
System.arraycopy(buffer, 0, copy, 0, length)
// 添加到缓冲区
dataBuffer.add(copy)
totalBytesWritten += length
// 写入文件
try {
fos?.write(copy, 0, length)
} catch (e: Exception) {
Log.e(tag, "写入音频数据失败: ${e.message}")
}
}
// 更新文件头生成（修正RIFF长度计算）
private fun generateWavHeader(dataLength: Int): ByteArray {
val totalLength = 36 + dataLength // RIFF块总长度 = 头部36字节 + 音频数据
val byteRate = sampleRate * 2 * 1 // 采样率 * 字节/样本 * 通道数
return byteArrayOf(
'R'.code.toByte(), 'I'.code.toByte(), 'F'.code.toByte(), 'F'.code.toByte(),
(totalLength and 0xFF).toByte(), ((totalLength shr 8) and 0xFF).toByte(),
((totalLength shr 16) and 0xFF).toByte(), ((totalLength shr 24) and 0xFF).toByte(),
'W'.code.toByte(), 'A'.code.toByte(), 'V'.code.toByte(), 'E'.code.toByte(),
'f'.code.toByte(), 'm'.code.toByte(), 't'.code.toByte(), ' '.code.toByte(),
16, 0, 0, 0, // PCM头长度
1, 0, // PCM格式
1, 0, // 单声道
(sampleRate and 0xFF).toByte(), ((sampleRate shr 8) and 0xFF).toByte(), 0, 0, // 采样率
(byteRate and 0xFF).toByte(), ((byteRate shr 8) and 0xFF).toByte(), 0, 0, // 字节率
2, 0, // 块对齐 (通道数 * 样本位数/8)
16, 0, // 样本位数
'd'.code.toByte(), 'a'.code.toByte(), 't'.code.toByte(), 'a'.code.toByte(),
(dataLength and 0xFF).toByte(), ((dataLength shr 8) and 0xFF).toByte(),
((dataLength shr 16) and 0xFF).toByte(), ((dataLength shr 24) and 0xFF).toByte()
)
}
/**
* 关闭音频资源
*/
override fun close() {
releaseAudioResources()
try {
// 关闭文件流
fos?.close()
// 更新WAV文件头
currentAudioFile?.let { file ->
RandomAccessFile(file, "rw").use { raf ->
raf.seek(0)
raf.write(generateWavHeader(totalBytesWritten))
}
Log.d(tag, "音频文件保存完成: ${file.absolutePath}")
}
} catch (e: Exception) {
Log.e(tag, "更新WAV文件头失败: ${e.message}")
} finally {
fos = null
currentAudioFile = null
}
}
/**
* 释放音频资源
*/
@ -730,11 +704,11 @@ private fun saveAudioDataToWav(buffer: ByteArray, length: Int) {
if (audioRecord?.recordingState == AudioRecord.RECORDSTATE_RECORDING) {
audioRecord?.stop()
}
// 释放回音消除器
echoCanceler?.release()
echoCanceler = null
// 释放录音实例
audioRecord?.release()
audioRecord = null
@ -743,7 +717,7 @@ private fun saveAudioDataToWav(buffer: ByteArray, length: Int) {
}
}
}
/**
* 外部音频拉流
* 实现PullAudioInputStreamCallback，将外部推送的音频数据转换为SDK可拉取的形式
@ -751,17 +725,19 @@ private fun saveAudioDataToWav(buffer: ByteArray, length: Int) {
private inner class ExternalAudioPullStream : PullAudioInputStreamCallback() {
private val queue: BlockingQueue<ByteArray> = LinkedBlockingQueue()
private var closed = false
/**
* 外部调用：推送音频数据到队列
* @param data 音频数据
*/
fun pushAudio(data: ByteArray) {
// 将音频数据保存成wav格式的音频文件
recordfile?.saveAudioDataToWav(data)
if (!closed) {
queue.offer(data)
}
}
/**
* SDK调用：从队列中拉取数据
* @param buffer SDK提供的缓冲区
@ -769,10 +745,12 @@ private fun saveAudioDataToWav(buffer: ByteArray, length: Int) {
*/
override fun read(buffer: ByteArray): Int {
try {
// 阻塞等待下一块数据
val chunk = queue.take()
val toCopy = minOf(chunk.size, buffer.size)
System.arraycopy(chunk, 0, buffer, 0, toCopy)
Log.i(tag, "外部音频:")
return toCopy
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
@ -782,7 +760,7 @@ private fun saveAudioDataToWav(buffer: ByteArray, length: Int) {
return 0
}
}
/**
* SDK调用：关闭流
*/
@ -791,70 +769,177 @@ private fun saveAudioDataToWav(buffer: ByteArray, length: Int) {
queue.clear()
}
}
/**
* 一次性识别回调接口
*/
interface RecognizeCallback {
/**
* 返回识别结果
*
*
* @param text 识别的文本
* @param detectedLanguage 检测到的语言
*/
fun onResult(text: String, detectedLanguage: String)
/**
* 识别错误时调用
*
*
* @param error 错误信息
*/
fun onError(error: String)
}
/**
* 连续识别回调接口
*/
interface ContinuousRecognizeCallback {
/**
* 返回识别结果
*
*
* @param text 识别的文本
* @param detectedLanguage 检测到的语言
*/
fun onResult(text: String, detectedLanguage: String)
/**
* 识别进行中调用
*
*
* @param recognizing 正在识别的文本
* @param detectedLanguage 检测到的语言
*/
fun onRecognizing(recognizing: String, detectedLanguage: String)
/**
* 会话开始时调用
*/
fun onSessionStarted()
/**
* 会话结束时调用
*/
fun onSessionStopped()
/**
* 识别取消时调用
*
*
* @param reason 取消原因
* @param errorDetails 错误详情
*/
fun onCanceled(reason: String, errorDetails: String)
/**
* 识别出错时调用
*
*
* @param error 错误信息
*/
fun onError(error: String)
}
private inner class RecordingFile {
private var currentAudioFile: File? = null
private var fos: FileOutputStream? = null
// 用于存储音频数据的缓冲区
private val dataBuffer = mutableListOf<ByteArray>()
private var totalBytesWritten = 0
// 音频配置
private val sampleRate = 16000
internal fun creatingFiles() {
if (fos != null || currentAudioFile != null) {
return
}
// 创建新的音频文件
val timestamp = System.currentTimeMillis()
val filePath =
context.getExternalFilesDir(null)?.absolutePath + "/recorded_audio_$timestamp.wav"
currentAudioFile = File(filePath)
currentAudioFile?.createNewFile()
// 仅写入初始文件头
fos = FileOutputStream(currentAudioFile).apply {
write(generateWavHeader(0)) // 初始0长度
close()
}
// 追加音频数据到文件
fos = FileOutputStream(currentAudioFile, true)
}
/**
* 保存音频数据到 WAV 文件
*/
internal fun saveAudioDataToWav(buffer: ByteArray) {
if (fos == null || currentAudioFile == null) {
return
}
// 复制数据到新数组
val copy = ByteArray(buffer.size)
System.arraycopy(buffer, 0, copy, 0, buffer.size)
// 添加到缓冲区
dataBuffer.add(copy)
totalBytesWritten += buffer.size
// 写入文件
try {
fos?.write(copy, 0, buffer.size)
} catch (e: Exception) {
Log.e(tag, "写入音频数据失败: ${e.message}")
}
}
// 更新文件头生成（修正RIFF长度计算）
private fun generateWavHeader(dataLength: Int): ByteArray {
val totalLength = 36 + dataLength // RIFF块总长度 = 头部36字节 + 音频数据
val byteRate = sampleRate * 2 * 1 // 采样率 * 字节/样本 * 通道数
return byteArrayOf(
'R'.code.toByte(), 'I'.code.toByte(), 'F'.code.toByte(), 'F'.code.toByte(),
(totalLength and 0xFF).toByte(), ((totalLength shr 8) and 0xFF).toByte(),
((totalLength shr 16) and 0xFF).toByte(), ((totalLength shr 24) and 0xFF).toByte(),
'W'.code.toByte(), 'A'.code.toByte(), 'V'.code.toByte(), 'E'.code.toByte(),
'f'.code.toByte(), 'm'.code.toByte(), 't'.code.toByte(), ' '.code.toByte(),
16, 0, 0, 0, // PCM头长度
1, 0, // PCM格式
1, 0, // 单声道
(sampleRate and 0xFF).toByte(), ((sampleRate shr 8) and 0xFF).toByte(), 0, 0, // 采样率
(byteRate and 0xFF).toByte(), ((byteRate shr 8) and 0xFF).toByte(), 0, 0, // 字节率
2, 0, // 块对齐 (通道数 * 样本位数/8)
16, 0, // 样本位数
'd'.code.toByte(), 'a'.code.toByte(), 't'.code.toByte(), 'a'.code.toByte(),
(dataLength and 0xFF).toByte(), ((dataLength shr 8) and 0xFF).toByte(),
((dataLength shr 16) and 0xFF).toByte(), ((dataLength shr 24) and 0xFF).toByte()
)
}
internal fun closeFile() {
try {
if (fos == null || currentAudioFile == null) {
return
}
// 关闭文件流
fos?.close()
// 更新WAV文件头
currentAudioFile?.let { file ->
RandomAccessFile(file, "rw").use { raf ->
raf.seek(0)
raf.write(generateWavHeader(totalBytesWritten))
}
Log.d(tag, "音频文件保存完成: ${file.absolutePath}")
}
} catch (e: Exception) {
Log.e(tag, "更新WAV文件头失败: ${e.message}")
} finally {
fos = null
currentAudioFile = null
}
}
}
}

833
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt
View File

@ -15,384 +15,499 @@ import com.microsoft.cognitiveservices.speech.audio.*
import com.deep_voice.speech.tts.TtsEvent
import com.deep_voice.speech.tts.TtsEventListener
import com.deep_voice.speech.tts.TtsEventType
import com.yunqiinnovation.ble_service.BleService
/** AzureSpeechPlugin */
class AzureSpeechPlugin: FlutterPlugin {
private val tag = "AzureSpeechPlugin"
private lateinit var context: Context
private val mainHandler = Handler(Looper.getMainLooper())
// ASR相关
private lateinit var asrChannel : MethodChannel
private lateinit var asrEventChannel: EventChannel
private var asrEventSink: EventChannel.EventSink? = null
private lateinit var azureAsrHelper: AzureAsrHelper
// TTS相关
private lateinit var ttsChannel : MethodChannel
private lateinit var ttsEventChannel: EventChannel
private var ttsEventSink: EventChannel.EventSink? = null
private lateinit var azureTtsHelper: AzureTtsHelper
// 是否已添加TTS事件监听器
private var isTtsListenerAdded = false
// ASR 事件发送方法
private fun sendAsrEvent(event: Map<String, Any>) {
if (asrEventSink == null) {
FileLogger.w(tag, "无法发送ASR事件：事件通道未准备好")
return
}
class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
private val tag = "AzureSpeechPlugin"
private lateinit var context: Context
private val mainHandler = Handler(Looper.getMainLooper())
mainHandler.post {
try {
asrEventSink?.success(event)
} catch (e: Exception) {
FileLogger.e(tag, "发送ASR事件失败: ${e.message}")
}
}
}
// TTS 事件发送方法
private fun sendTtsEvent(event: Map<String, Any>) {
if (ttsEventSink == null) {
FileLogger.w(tag, "无法发送TTS事件：事件通道未准备好")
return
}
// 确保在主线程上发送事件
mainHandler.post {
try {
ttsEventSink?.success(event)
} catch (e: Exception) {
FileLogger.e(tag, "发送TTS事件失败: ${e.message}")
}
}
}
override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) {
context = flutterPluginBinding.applicationContext
// 初始化ASR通道
asrChannel = MethodChannel(flutterPluginBinding.binaryMessenger, "azure_speech/asr")
asrChannel.setMethodCallHandler(AsrMethodHandler())
// 初始化TTS通道
ttsChannel = MethodChannel(flutterPluginBinding.binaryMessenger, "azure_speech/tts")
ttsChannel.setMethodCallHandler(TtsMethodHandler())
// 初始化ASR事件通道
asrEventChannel = EventChannel(flutterPluginBinding.binaryMessenger, "azure_speech/asr_events")
asrEventChannel.setStreamHandler(object : EventChannel.StreamHandler {
override fun onListen(arguments: Any?, events: EventChannel.EventSink?) {
asrEventSink = events
}
override fun onCancel(arguments: Any?) {
asrEventSink = null
}
})
// 初始化TTS事件通道
ttsEventChannel = EventChannel(flutterPluginBinding.binaryMessenger, "azure_speech/tts_events")
ttsEventChannel.setStreamHandler(object : EventChannel.StreamHandler {
override fun onListen(arguments: Any?, events: EventChannel.EventSink?) {
ttsEventSink = events
setupTtsEventListener() // 在事件通道准备好时设置TTS事件监听器
}
override fun onCancel(arguments: Any?) {
ttsEventSink = null
}
})
// 初始化Azure语音服务
azureTtsHelper = AzureTtsHelper(context)
azureAsrHelper = AzureAsrHelper(context)
}
/**
* 设置TTS事件监听器
*/
private fun setupTtsEventListener() {
if (!isTtsListenerAdded) {
// 添加TTS事件监听器
azureTtsHelper.addListener(object : TtsEventListener {
override fun onEvent(event: TtsEvent) {
// 将TTS事件转换为Flutter可理解的格式
val eventMap = when (event.type) {
TtsEventType.SYNTHESIS_STARTED -> mapOf(
"type" to "synthesis_started"
)
TtsEventType.SYNTHESIS_COMPLETED -> mapOf(
"type" to "synthesis_completed"
)
TtsEventType.SYNTHESIS_CANCELED -> {
val params = event.params
mapOf(
"type" to "synthesis_canceled",
"reason" to (params["reason"] ?: ""),
"errorDetails" to (params["errorDetails"] ?: "")
)
}
TtsEventType.ERROR -> {
val params = event.params
mapOf(
"type" to "error",
"errorCode" to (params["errorCode"] ?: "UNKNOWN_ERROR"),
"errorMessage" to (params["errorMessage"] ?: "未知错误")
)
// ASR相关
private lateinit var asrChannel: MethodChannel
private lateinit var asrEventChannel: EventChannel
private var asrEventSink: EventChannel.EventSink? = null
private lateinit var azureAsrHelper: AzureAsrHelper
// TTS相关
private lateinit var ttsChannel: MethodChannel
private lateinit var ttsEventChannel: EventChannel
private var ttsEventSink: EventChannel.EventSink? = null
private lateinit var azureTtsHelper: AzureTtsHelper
// 是否已添加TTS事件监听器
private var isTtsListenerAdded = false
// 是否已注册BLE回调
private var isRegisteredToBle = false
// ASR 事件发送方法
private fun sendAsrEvent(event: Map<String, Any>) {
if (asrEventSink == null) {
FileLogger.w(tag, "无法发送ASR事件：事件通道未准备好")
return
}
mainHandler.post {
try {
asrEventSink?.success(event)
} catch (e: Exception) {
FileLogger.e(tag, "发送ASR事件失败: ${e.message}")
}
else -> mapOf(
"type" to event.type.toString().lowercase(),
"params" to event.params
)
}
// 发送事件
sendTtsEvent(eventMap)
}
})
isTtsListenerAdded = true
}
}
// ASR方法处理器
inner class AsrMethodHandler : MethodCallHandler {
override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) {
when (call.method) {
"initialize" -> {
val subscriptionKey = call.argument<String>("subscriptionKey") ?: ""
val region = call.argument<String>("region") ?: ""
val supportedLanguages = call.argument<List<String>>("supportedLanguages") ?: listOf("zh-CN")
val useExternalAudio = call.argument<Boolean>("useExternalAudio") ?: false
try {
FileLogger.d(tag, "选择音频源类型: ${useExternalAudio}") //
// 选择音频源类型
val audioSourceType = if (useExternalAudio) {
AzureAsrHelper.AudioSourceType.EXTERNAL
} else {
AzureAsrHelper.AudioSourceType.MICROPHONE
}
val success = azureAsrHelper.initialize(
subscriptionKey = subscriptionKey,
region = region,
supportedLanguages = supportedLanguages.toTypedArray(),
audioSourceType = audioSourceType,
)
result.success(success)
} catch (e: Exception) {
result.error("INITIALIZATION_ERROR", e.message, null)
}
// TTS 事件发送方法
private fun sendTtsEvent(event: Map<String, Any>) {
if (ttsEventSink == null) {
FileLogger.w(tag, "无法发送TTS事件：事件通道未准备好")
return
}
"recognizeOnce" -> {
val useExternalAudio = call.argument<Boolean>("useExternalAudio") ?: false
try {
FileLogger.d(tag, "选择音频源类型: ${useExternalAudio}") //
// 选择音频源类型
val audioSourceType = if (useExternalAudio) {
AzureAsrHelper.AudioSourceType.EXTERNAL
} else {
AzureAsrHelper.AudioSourceType.MICROPHONE
// 确保在主线程上发送事件
mainHandler.post {
try {
ttsEventSink?.success(event)
} catch (e: Exception) {
FileLogger.e(tag, "发送TTS事件失败: ${e.message}")
}
// 直接使用同步方式进行一次性识别
azureAsrHelper.recognizeOnce(object : AzureAsrHelper.RecognizeCallback {
override fun onResult(text: String, detectedLanguage: String) {
// 直接返回结果，不发送事件
result.success(mapOf(
"text" to text,
"detectedLanguage" to detectedLanguage
))
}
override fun onError(error: String) {
result.error("RECOGNITION_ERROR", error, null)
}
},audioSourceType)
} catch (e: Exception) {
result.error("RECOGNITION_ERROR", "识别过程异常: ${e.message}", null)
}
}
"startContinuousRecognition" -> {
val useExternalAudio = call.argument<Boolean>("audioSourceType") ?: false
// 确保事件通道已准备好
if (asrEventSink == null) {
result.error("EVENT_CHANNEL_NOT_READY", "事件通道未准备好，无法开始连续识别", null)
return
}
try {
FileLogger.d(tag, "选择音频源类型: ${useExternalAudio}") //
// 选择音频源类型
val audioSourceType = if (useExternalAudio) {
AzureAsrHelper.AudioSourceType.EXTERNAL
}
override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) {
context = flutterPluginBinding.applicationContext
// 初始化ASR通道
asrChannel = MethodChannel(flutterPluginBinding.binaryMessenger, "azure_speech/asr")
asrChannel.setMethodCallHandler(AsrMethodHandler())
// 初始化TTS通道
ttsChannel = MethodChannel(flutterPluginBinding.binaryMessenger, "azure_speech/tts")
ttsChannel.setMethodCallHandler(TtsMethodHandler())
// 初始化ASR事件通道
asrEventChannel =
EventChannel(flutterPluginBinding.binaryMessenger, "azure_speech/asr_events")
asrEventChannel.setStreamHandler(object : EventChannel.StreamHandler {
override fun onListen(arguments: Any?, events: EventChannel.EventSink?) {
asrEventSink = events
}
override fun onCancel(arguments: Any?) {
asrEventSink = null
}
})
// 初始化TTS事件通道
ttsEventChannel =
EventChannel(flutterPluginBinding.binaryMessenger, "azure_speech/tts_events")
ttsEventChannel.setStreamHandler(object : EventChannel.StreamHandler {
override fun onListen(arguments: Any?, events: EventChannel.EventSink?) {
ttsEventSink = events
setupTtsEventListener() // 在事件通道准备好时设置TTS事件监听器
}
override fun onCancel(arguments: Any?) {
ttsEventSink = null
}
})
// 初始化Azure语音服务
azureTtsHelper = AzureTtsHelper(context)
azureAsrHelper = AzureAsrHelper(context)
// 2. 初始化BleService并注册回调
if (BleService.initialize(context)) {
if (BleService.addCallback(this)) {
isRegisteredToBle = true
FileLogger.d(tag, "已注册BLE回调")
} else {
AzureAsrHelper.AudioSourceType.MICROPHONE
FileLogger.w(tag, "BLE回调注册失败")
}
val success = azureAsrHelper.startContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback {
override fun onResult(text: String, detectedLanguage: String) {
sendAsrEvent(mapOf(
"type" to "result",
"text" to text,
"detectedLanguage" to detectedLanguage
))
}
override fun onRecognizing(recognizing: String, detectedLanguage: String) {
sendAsrEvent(mapOf(
"type" to "recognizing",
"text" to recognizing,
"detectedLanguage" to detectedLanguage
))
}
override fun onSessionStarted() {
sendAsrEvent(mapOf("type" to "sessionStarted"))
}
override fun onSessionStopped() {
sendAsrEvent(mapOf("type" to "sessionStopped"))
}
override fun onCanceled(reason: String, errorDetails: String) {
sendAsrEvent(mapOf(
"type" to "canceled",
"reason" to reason,
"errorDetails" to errorDetails
))
}
override fun onError(error: String) {
sendAsrEvent(mapOf(
"type" to "error",
"message" to error
))
}
},audioSourceType)
result.success(success)
} catch (e: Exception) {
result.error("START_RECOGNITION_ERROR", e.message, null)
}
}
"stopContinuousRecognition" -> {
try {
val success = azureAsrHelper.stopContinuousRecognition()
result.success(success)
} catch (e: Exception) {
result.error("STOP_ERROR", e.message, null)
}
}
"isContinuousRecognitionActive" -> {
result.success(azureAsrHelper.isContinuousRecognitionActive())
}
"dispose" -> {
azureAsrHelper.dispose()
result.success(true)
}
"pushAudioData" -> {
val data = call.argument<ByteArray>("data") ?: return result.error("INVALID_ARGUMENTS", "音频数据不能为空", null)
azureAsrHelper.pushAudioData(data)
result.success(true)
} else {
FileLogger.e(tag, "BLE服务初始化失败")
}
else -> {
result.notImplemented()
}
/**
* 设置TTS事件监听器
*/
private fun setupTtsEventListener() {
if (!isTtsListenerAdded) {
// 添加TTS事件监听器
azureTtsHelper.addListener(object : TtsEventListener {
override fun onEvent(event: TtsEvent) {
// 将TTS事件转换为Flutter可理解的格式
val eventMap = when (event.type) {
TtsEventType.SYNTHESIS_STARTED -> mapOf(
"type" to "synthesis_started"
)
TtsEventType.SYNTHESIS_COMPLETED -> mapOf(
"type" to "synthesis_completed"
)
TtsEventType.SYNTHESIS_CANCELED -> {
val params = event.params
mapOf(
"type" to "synthesis_canceled",
"reason" to (params["reason"] ?: ""),
"errorDetails" to (params["errorDetails"] ?: "")
)
}
TtsEventType.ERROR -> {
val params = event.params
mapOf(
"type" to "error",
"errorCode" to (params["errorCode"] ?: "UNKNOWN_ERROR"),
"errorMessage" to (params["errorMessage"] ?: "未知错误")
)
}
else -> mapOf(
"type" to event.type.toString().lowercase(),
"params" to event.params
)
}
// 发送事件
sendTtsEvent(eventMap)
}
})
isTtsListenerAdded = true
}
}
}
}
// TTS方法处理器
inner class TtsMethodHandler : MethodCallHandler {
override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) {
when (call.method) {
"initialize" -> {
val subscriptionKey = call.argument<String>("subscriptionKey") ?: ""
val region = call.argument<String>("region") ?: ""
val useCustomAudioOutput = call.argument<Boolean>("useCustomAudioOutput") ?: false
try {
// 初始化TTS引擎（使用新的接口方法）
val success = azureTtsHelper.initialize(
ttsAppId = "", // Azure TTS不需要appId
ttsAppToken = subscriptionKey,
ttsResource = region,
language = "zh-CN"
)
// 如果初始化成功且需要使用自定义音频输出，则设置自定义音频播放器
if (success && useCustomAudioOutput) {
// 创建SimpleAudioPlayer实例
val customPlayer = SimpleAudioPlayer(context)
// 获取并设置音频输出流
val outputStream = customPlayer.getAudioOutputStream()
if (outputStream != null) {
azureTtsHelper.setCustomAudioOutputStream(outputStream)
}
// ASR方法处理器
inner class AsrMethodHandler : MethodCallHandler {
override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) {
when (call.method) {
"initialize" -> {
val subscriptionKey = call.argument<String>("subscriptionKey") ?: ""
val region = call.argument<String>("region") ?: ""
val supportedLanguages =
call.argument<List<String>>("supportedLanguages") ?: listOf("zh-CN")
val useExternalAudio = call.argument<Boolean>("useExternalAudio") ?: false
try {
FileLogger.d(tag, "选择音频源类型: ${useExternalAudio}") //
// 选择音频源类型
val audioSourceType = if (useExternalAudio) {
AzureAsrHelper.AudioSourceType.EXTERNAL
} else {
AzureAsrHelper.AudioSourceType.MICROPHONE
}
val success = azureAsrHelper.initialize(
subscriptionKey = subscriptionKey,
region = region,
supportedLanguages = supportedLanguages.toTypedArray(),
audioSourceType = audioSourceType,
)
result.success(success)
} catch (e: Exception) {
result.error("INITIALIZATION_ERROR", e.message, null)
}
}
"recognizeOnce" -> {
val useExternalAudio = call.argument<Boolean>("useExternalAudio") ?: false
try {
FileLogger.d(tag, "选择音频源类型: ${useExternalAudio}") //
// 选择音频源类型
val audioSourceType = if (useExternalAudio) {
AzureAsrHelper.AudioSourceType.EXTERNAL
} else {
AzureAsrHelper.AudioSourceType.MICROPHONE
}
// 直接使用同步方式进行一次性识别
azureAsrHelper.recognizeOnce(object : AzureAsrHelper.RecognizeCallback {
override fun onResult(text: String, detectedLanguage: String) {
// 直接返回结果，不发送事件
result.success(
mapOf(
"text" to text,
"detectedLanguage" to detectedLanguage
)
)
}
override fun onError(error: String) {
result.error("RECOGNITION_ERROR", error, null)
}
}, audioSourceType)
} catch (e: Exception) {
result.error("RECOGNITION_ERROR", "识别过程异常: ${e.message}", null)
}
}
"startContinuousRecognition" -> {
val useExternalAudio = call.argument<Boolean>("audioSourceType") ?: false
// 确保事件通道已准备好
if (asrEventSink == null) {
result.error(
"EVENT_CHANNEL_NOT_READY",
"事件通道未准备好，无法开始连续识别",
null
)
return
}
try {
FileLogger.d(tag, "选择音频源类型: ${useExternalAudio}") //
// 选择音频源类型
val audioSourceType = if (useExternalAudio) {
AzureAsrHelper.AudioSourceType.EXTERNAL
} else {
AzureAsrHelper.AudioSourceType.MICROPHONE
}
val success = azureAsrHelper.startContinuousRecognition(object :
AzureAsrHelper.ContinuousRecognizeCallback {
override fun onResult(text: String, detectedLanguage: String) {
sendAsrEvent(
mapOf(
"type" to "result",
"text" to text,
"detectedLanguage" to detectedLanguage
)
)
}
override fun onRecognizing(
recognizing: String,
detectedLanguage: String
) {
sendAsrEvent(
mapOf(
"type" to "recognizing",
"text" to recognizing,
"detectedLanguage" to detectedLanguage
)
)
}
override fun onSessionStarted() {
sendAsrEvent(mapOf("type" to "sessionStarted"))
}
override fun onSessionStopped() {
sendAsrEvent(mapOf("type" to "sessionStopped"))
}
override fun onCanceled(reason: String, errorDetails: String) {
sendAsrEvent(
mapOf(
"type" to "canceled",
"reason" to reason,
"errorDetails" to errorDetails
)
)
}
override fun onError(error: String) {
sendAsrEvent(
mapOf(
"type" to "error",
"message" to error
)
)
}
}, audioSourceType)
result.success(success)
} catch (e: Exception) {
result.error("START_RECOGNITION_ERROR", e.message, null)
}
}
"stopContinuousRecognition" -> {
try {
val success = azureAsrHelper.stopContinuousRecognition()
result.success(success)
} catch (e: Exception) {
result.error("STOP_ERROR", e.message, null)
}
}
"isContinuousRecognitionActive" -> {
result.success(azureAsrHelper.isContinuousRecognitionActive())
}
"dispose" -> {
azureAsrHelper.dispose()
result.success(true)
}
"pushAudioData" -> {
val data = call.argument<ByteArray>("data") ?: return result.error(
"INVALID_ARGUMENTS",
"音频数据不能为空",
null
)
azureAsrHelper.pushAudioData(data)
result.success(true)
}
"enableRecord" -> {
azureAsrHelper.enableRecord()
result.success(true)
}
"stopRecord" -> {
azureAsrHelper.stopRecord()
result.success(true)
}
else -> {
result.notImplemented()
}
}
// 确保设置了TTS事件监听器
setupTtsEventListener()
result.success(success)
} catch (e: Exception) {
result.error("INITIALIZATION_ERROR", "TTS初始化失败: ${e.message}", null)
}
}
"setVoice" -> {
val voiceName = call.argument<String>("voiceName") ?: return result.error("INVALID_ARGUMENTS", "语音名称不能为空", null)
result.success(azureTtsHelper.setVoice(voiceName))
}
"speakText" -> {
val text = call.argument<String>("text") ?: return result.error("INVALID_ARGUMENTS", "文本不能为空", null)
// 直接调用speakOnce方法
try {
val success = azureTtsHelper.speakOnce(text)
result.success(success)
} catch (e: Exception) {
result.error("SPEAK_ERROR", "语音合成失败: ${e.message}", null)
}
}
"stopSpeaking" -> {
// 直接调用stop方法
try {
val success = azureTtsHelper.stop()
result.success(success)
} catch (e: Exception) {
result.error("STOP_ERROR", "停止语音合成失败: ${e.message}", null)
}
}
"isSpeaking" -> {
result.success(azureTtsHelper.isSpeaking())
}
// TTS方法处理器
inner class TtsMethodHandler : MethodCallHandler {
override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) {
when (call.method) {
"initialize" -> {
val subscriptionKey = call.argument<String>("subscriptionKey") ?: ""
val region = call.argument<String>("region") ?: ""
val useCustomAudioOutput =
call.argument<Boolean>("useCustomAudioOutput") ?: false
try {
// 初始化TTS引擎（使用新的接口方法）
val success = azureTtsHelper.initialize(
ttsAppId = "", // Azure TTS不需要appId
ttsAppToken = subscriptionKey,
ttsResource = region,
language = "zh-CN"
)
// 如果初始化成功且需要使用自定义音频输出，则设置自定义音频播放器
if (success && useCustomAudioOutput) {
// 创建SimpleAudioPlayer实例
val customPlayer = SimpleAudioPlayer(context)
// 获取并设置音频输出流
val outputStream = customPlayer.getAudioOutputStream()
if (outputStream != null) {
azureTtsHelper.setCustomAudioOutputStream(outputStream)
}
}
// 确保设置了TTS事件监听器
setupTtsEventListener()
result.success(success)
} catch (e: Exception) {
result.error("INITIALIZATION_ERROR", "TTS初始化失败: ${e.message}", null)
}
}
"setVoice" -> {
val voiceName = call.argument<String>("voiceName")
?: return result.error("INVALID_ARGUMENTS", "语音名称不能为空", null)
result.success(azureTtsHelper.setVoice(voiceName))
}
"speakText" -> {
val text = call.argument<String>("text") ?: return result.error(
"INVALID_ARGUMENTS",
"文本不能为空",
null
)
// 直接调用speakOnce方法
try {
val success = azureTtsHelper.speakOnce(text)
result.success(success)
} catch (e: Exception) {
result.error("SPEAK_ERROR", "语音合成失败: ${e.message}", null)
}
}
"stopSpeaking" -> {
// 直接调用stop方法
try {
val success = azureTtsHelper.stop()
result.success(success)
} catch (e: Exception) {
result.error("STOP_ERROR", "停止语音合成失败: ${e.message}", null)
}
}
"isSpeaking" -> {
result.success(azureTtsHelper.isSpeaking())
}
"dispose" -> {
azureTtsHelper.release()
result.success(true)
}
else -> {
result.notImplemented()
}
}
}
"dispose" -> {
azureTtsHelper.release()
result.success(true)
}
override fun onDetachedFromEngine(@NonNull binding: FlutterPlugin.FlutterPluginBinding) {
asrChannel.setMethodCallHandler(null)
ttsChannel.setMethodCallHandler(null)
asrEventChannel.setStreamHandler(null)
ttsEventChannel.setStreamHandler(null)
try {
if (isRegisteredToBle) {
if (BleService.removeCallback(this)) {
isRegisteredToBle = false
FileLogger.d(tag, "已取消BLE回调")
} else {
FileLogger.w(tag, "BLE回调取消失败")
}
}
// 释放资源
azureTtsHelper.release()
azureAsrHelper.dispose()
} catch (e: Exception) {
FileLogger.e(tag, "释放资源失败: ${e.message}")
}
else -> {
result.notImplemented()
}
// BleService.Callback 接口实现
override fun onScanResult(devices: List<Map<String, Any>>) {
// 不处理扫描结果
}
override fun onConnectionStateChanged(state: Int) {
}
override fun onAudioDataReceived(data: ByteArray) {
if (azureAsrHelper.audioSourceType == AzureAsrHelper.AudioSourceType.EXTERNAL) {
azureAsrHelper.pushAudioData(data)
}
}
// 可选：处理音频数据
}
/**
* 处理唤醒信号
* 在收到唤醒信号时启动语音识别
*/
override fun onWakeupSignalReceived() {
}
}
override fun onDetachedFromEngine(@NonNull binding: FlutterPlugin.FlutterPluginBinding) {
asrChannel.setMethodCallHandler(null)
ttsChannel.setMethodCallHandler(null)
asrEventChannel.setStreamHandler(null)
ttsEventChannel.setStreamHandler(null)
try {
// 释放资源
azureTtsHelper.release()
azureAsrHelper.dispose()
} catch (e: Exception) {
FileLogger.e(tag, "释放资源失败: ${e.message}")
override fun onDeviceInfoReceived(infoType: Int, infoData: Map<String, Any>) {
// 不处理设备信息
}
}
}

5
local_plugins/azure_speech/pubspec.yaml
View File

@ -10,11 +10,14 @@ environment:
dependencies:
flutter:
sdk: flutter
ble_service:
path: ../ble_service
dev_dependencies:
flutter_test:
sdk: flutter
flutter_lints: ^2.0.0
# For information on the generic Dart part of this file, see the
# following page: https://dart.dev/tools/pub/pubspec

22
local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt
View File

@ -1,38 +1,18 @@
package com.yunqiinnovation.ble_service
import android.Manifest
import android.app.PendingIntent
import android.bluetooth.*
import android.bluetooth.le.*
import android.content.Context
import android.content.Intent
import android.os.*
import androidx.lifecycle.MutableLiveData
import android.util.Log
import java.io.File
import java.io.FileOutputStream
import java.text.SimpleDateFormat
import java.util.*
import android.os.ParcelUuid
import android.annotation.SuppressLint
import android.content.BroadcastReceiver
import android.content.IntentFilter
import android.os.SystemClock
import android.content.pm.PackageManager
import androidx.core.content.ContextCompat
import java.util.concurrent.ConcurrentHashMap
import android.content.pm.ServiceInfo
import java.util.concurrent.CopyOnWriteArrayList
// Import Jieli Opus SDK classes
import com.jieli.jl_audio_decode.opus.model.OpusOption
import com.jieli.jl_audio_decode.callback.OnDecodeStreamCallback
import com.jieli.jl_audio_decode.callback.OnStateCallback
import com.jieli.jl_audio_decode.opus.OpusManager
import com.jieli.jl_audio_decode.exceptions.OpusException
import com.yunqiinnovation.ble_service.BleCompanionManager
import com.jieli.jl_bt_ota.model.BluetoothOTAConfigure;
import com.jieli.jl_bt_ota.constant.ErrorCode;
import com.jieli.jl_bt_ota.constant.StateCode;
/**
* BLE服务类：提供蓝牙低功耗设备的扫描、连接和通信功能
@ -1386,4 +1366,4 @@ object BleService {
}
}
}
}
}

43
local_plugins/ota/android/src/main/kotlin/com/example/ota/OtaPlugin.kt
View File

@ -9,28 +9,18 @@ import io.flutter.plugin.common.MethodChannel.Result
import android.bluetooth.BluetoothDevice
import android.content.Context
import com.jieli.otasdk.tool.ota.OTAManager
import com.jieli.jl_bt_ota.constant.ErrorCode
import com.jieli.jl_bt_ota.constant.StateCode
import com.jieli.jl_bt_ota.interfaces.BtEventCallback
import com.jieli.jl_bt_ota.interfaces.IUpgradeCallback
import com.jieli.jl_bt_ota.model.base.BaseError
import com.jieli.jl_bt_ota.model.response.TargetInfoResponse
import com.jieli.jl_bt_ota.util.BluetoothUtil
import com.jieli.jl_bt_ota.util.CHexConver
import com.jieli.jl_bt_ota.util.JL_Log
import com.jieli.otasdk.tool.bluetooth.BluetoothHelper
import androidx.lifecycle.MutableLiveData
import android.util.Log
import androidx.annotation.NonNull
import com.jieli.otasdk.tool.config.ConfigHelper
import com.yunqiinnovation.ble_service.BleService
import com.yunqiinnovation.ble_service.BleConst
import android.os.Handler
import android.os.Looper
import java.io.File
import java.io.FileOutputStream
import java.io.RandomAccessFile
import io.flutter.plugin.common.EventChannel
class CustomUpdateCallback(
@ -129,7 +119,10 @@ class CustomUpdateCallback(
Log.d(TAG, "升级失败: ${error?.message ?: "未知错误"}")
if (streamSink != null) {
if (Looper.myLooper() == Looper.getMainLooper()) {
Log.d(TAG, "升级失败:error?.message ${error?.message}未知错误 error?.code${error?.code}")
Log.d(
TAG,
"升级失败:error?.message ${error?.message}未知错误 error?.code${error?.code}"
)
streamSink.success(
mapOf(
"event" to "onError",
@ -281,8 +274,20 @@ class OtaPlugin : BleService.Callback, FlutterPlugin, MethodCallHandler {
bluetoothHelper = BluetoothHelper(context)
// 如果需要额外的初始化逻辑，请在这里添加
otaManager.registerBluetoothCallback(bluetoothCallback)
BleService.addCallback(this)
isRegisteredToBle = true
// 初始化BleService
// 2. 初始化BleService并注册回调
if (BleService.initialize(context)) {
if (BleService.addCallback(this)) {
isRegisteredToBle = true
Log.d(TAG, "已注册BLE回调")
} else {
Log.w(TAG, "BLE回调注册失败")
}
} else {
Log.e(TAG, "BLE服务初始化失败")
}
Log.i("TAG", "onAttachedToEngine")
}
@ -308,11 +313,10 @@ class OtaPlugin : BleService.Callback, FlutterPlugin, MethodCallHandler {
otaDevice = bluetoothGatt.device
BleService.disconnect()
} else if(otaDevice != null)
{ isDisconnect = true
BleService.disconnect()
}
else{
} else if (otaDevice != null) {
isDisconnect = true
BleService.disconnect()
} else {
Log.e("TAG", "BluetoothGatt 或 device 为 null，无法执行操作")
sendEventOnMainThread(
mapOf(
@ -324,7 +328,7 @@ class OtaPlugin : BleService.Callback, FlutterPlugin, MethodCallHandler {
}
} else if (call.method == "onOtaClose") {
} else if (call.method == "onOtaClose") {
BleService.isEnterOta = false
isDisconnect = false
otaManager.bluetoothHelper.stopScan()
@ -350,6 +354,7 @@ class OtaPlugin : BleService.Callback, FlutterPlugin, MethodCallHandler {
}
// BleService.Callback 接口实现
override fun onScanResult(devices: List<Map<String, Any>>) {
// 不处理扫描结果
}

Write Preview
Loading…
Cancel
Save