feat: Speech-to-text for conversations

ui/package.json

@@ -48,6 +48,7 @@
     "nprogress": "^0.2.0",
     "pinia": "^3.0.4",
     "prettier": "3.8.4",
+    "recorder-core": "^1.3.26070800",
     "screenfull": "^6.0.2",
     "uuid": "^14.0.2",
     "vue": "^3.5.38",

ui/src/conversation-panel/components/message-input/index.ts

@@ -49,6 +49,8 @@ export interface MessageInputDeps {
   sendMessage: (opts: SendMessageOptions) => void
   uploadFile: (file: File, chatId: string, onProgress?: (percent: number) => void) => Promise<string>
   stop: () => void
+  /** 语音转文字，表单字段 file 为录音文件；不传时不显示语音输入。 */
+  speechToText?: (data: FormData) => Promise<unknown>
   /** 当前智能体信息：只读，由 view 加载维护。 */
   appInfo: Readonly<Ref<ConversationAppInfo | null>>
   /** 禁用输入与发送，如门户没有可用智能体时；不传时始终可用。 */
@@ -78,8 +80,13 @@ export function createMessageInputStore(deps: MessageInputDeps) {
   // TODO 格式 不行
   const acceptList = [...IMAGE_EXTENSIONS, ...DOCUMENT_EXTENSIONS, ...VIDEO_EXTENSIONS, ...AUDIO_EXTENSIONS].map((e) => `.${e}`).join(',')
 
+  // 语音输入：智能体开启语音转文字且当前模式提供接口时可用
+  const transcribing = ref(false)
+  const speechEnabled = computed(() => !!deps.speechToText && !!appInfo.value?.stt_model_enable)
+
   const placeholder = computed(() => {
     if (disabled.value) return '暂无可用智能体'
+    if (transcribing.value) return '语音转文字中...'
     return streamLoading.value ? '正在回复中...' : '输入消息...'
   })
   // 附件上传完成前不能发送，否则消息中的附件没有地址
@@ -90,6 +97,7 @@ export function createMessageInputStore(deps: MessageInputDeps) {
       (question.value.trim().length > 0 || fileList.value.length > 0) &&
       !streamLoading.value &&
       !sending.value &&
+      !transcribing.value &&
       attachmentsReady.value,
   )
 
@@ -231,6 +239,29 @@ export function createMessageInputStore(deps: MessageInputDeps) {
     }
   }
 
+  /** 录音转写后追加到输入框；智能体开启自动发送时直接发送。切换会话后丢弃旧结果。 */
+  const transcribe = (audio: Blob) => {
+    if (!deps.speechToText || transcribing.value) return
+    const resetSignal = composerResetSignal.value
+    const formData = new FormData()
+    formData.append('file', audio, 'recording.mp3')
+    transcribing.value = true
+    deps
+      .speechToText(formData)
+      .then((result) => {
+        if (resetSignal !== composerResetSignal.value) return
+        const text = typeof result === 'string' ? result.trim() : ''
+        if (!text) return MsgError('未识别到语音内容')
+        question.value = question.value ? `${question.value} ${text}` : text
+        transcribing.value = false
+        if (appInfo.value?.stt_autosend) void send()
+      })
+      .catch(() => {})
+      .finally(() => {
+        transcribing.value = false
+      })
+  }
+
   /** 由开场白、回答中的快捷问题或 HTML 渲染发起提问；不读取也不清空输入框与暂存附件。 */
   const sendQuestion = (content: string, options: QuestionOptions = {}) => {
     if (disabled.value || streamLoading.value || sending.value || !content.trim()) return
@@ -261,6 +292,9 @@ export function createMessageInputStore(deps: MessageInputDeps) {
     disabled,
     placeholder,
     canSend,
+    speechEnabled,
+    transcribing,
+    transcribe,
     imageFiles,
     addFile,
     addFiles,

ui/src/conversation-panel/components/message-input/index.vue

@@ -1,10 +1,13 @@
 <script setup lang="ts">
+import { Phone } from '@element-plus/icons-vue'
 import { isAudio, isVideo } from '@/utils/icon'
 import { useInputEvents } from '../../core/input-events'
 import { useMessageInputStore } from './index'
 import FileCard from './file-card/index.vue'
 import ImageUploadCard from '@/components/mk-drag-upload/ImageUploadCard.vue'
 import UserForm from './user-form/index.vue'
+import VoiceRecorder from './voice-recorder/index.vue'
+import { useVoiceRecorder } from './voice-recorder/index'
 
 const {
   question,
@@ -16,6 +19,9 @@ const {
   disabled,
   placeholder,
   canSend,
+  speechEnabled,
+  transcribing,
+  transcribe,
   imageFiles,
   addFiles,
   removeFile,
@@ -32,10 +38,15 @@ const { fileInputRef, handleFileSelect, handlePaste, handleDragover, handleDrop,
     if (canSend.value) send()
   },
 })
+
+// 语音输入：点击麦克风开始录音，点击停止后转写到输入框
+const { recording, seconds: recordSeconds, waveLevels, start: startRecording, stop: stopRecording } = useVoiceRecorder(transcribe)
 </script>
 
 <template>
-  <div class="mk-input-box mk-conversation-message-input w-full shadow-lg" @dragover="handleDragover" @drop="handleDrop" @paste="handlePaste">
+  <!-- 录音中 -->
+  <VoiceRecorder v-if="recording" :seconds="recordSeconds" :wave-levels="waveLevels" @stop="stopRecording" />
+  <div v-else class="mk-input-box mk-conversation-message-input w-full shadow-lg" @dragover="handleDragover" @drop="handleDrop" @paste="handlePaste">
     <!-- 附件按上传顺序排列，超出可视范围时显示滚动箭头。 -->
     <MkHorizontalScroll v-if="fileList.length" class="mb-3">
       <template v-for="(file, index) in fileList" :key="file.uid">
@@ -104,6 +115,18 @@ const { fileInputRef, handleFileSelect, handlePaste, handleDragover, handleDrop,
           <MkIcon name="icon_attachment_outlined" :size="18" class="text-N900!" />
         </el-button>
       </MkTooltip>
+      <!-- 语音输入：点击开始录音 -->
+      <el-button
+        v-if="speechEnabled"
+        text
+        class="ml-1!"
+        :loading="transcribing"
+        :disabled="disabled || streamLoading || transcribing"
+        @click="startRecording"
+      >
+        <!-- todo 这块icon记得换 先用饿了么的了 -->
+        <MkIcon v-if="!transcribing"  :icon="Phone" :size="18" class="text-N900!" />
+      </el-button>
       <el-divider direction="vertical" class="ml-3! mr-4!" />
       <!-- 停止回复 -->
       <el-button v-if="streamLoading" circle type="primary" @click="stop">

ui/src/conversation-panel/components/message-input/voice-recorder/index.ts

@@ -0,0 +1,101 @@
+import { onBeforeUnmount, ref } from 'vue'
+import Recorder from 'recorder-core'
+import 'recorder-core/src/engine/mp3'
+import 'recorder-core/src/engine/mp3-engine'
+import { MsgError } from '@/utils/message'
+
+/** 单次录音最长秒数，到时自动结束并转写。 */
+export const VOICE_RECORD_MAX_SECONDS = 60
+/** 录音波形柱数量。 */
+const WAVE_BAR_COUNT = 60
+
+// 关闭 recorder-core 的控制台日志
+Recorder.CLog = () => {}
+
+/** recorder-core 只导出 any，这里声明本文件用到的实例方法。 */
+interface RecorderInstance {
+  open: (success: () => void, fail: (message: string, isUserNotAllow: boolean) => void) => void
+  start: () => void
+  stop: (success: (audio: Blob, duration: number) => void, fail: (message: string) => void) => void
+  close: () => void
+}
+
+/**
+ * 输入框语音录制：录制 16kHz MP3（与语音转文字模型的入参格式一致），结束后回调录音文件。
+ * 录音期间记录音量用于绘制波形；组件卸载时释放麦克风。
+ */
+export function useVoiceRecorder(onRecorded: (audio: Blob) => void) {
+  const recording = ref(false)
+  const seconds = ref(0)
+  const waveLevels = ref<number[]>(Array(WAVE_BAR_COUNT).fill(0))
+
+  let recorder: RecorderInstance | null = null
+  let timer: ReturnType<typeof setInterval> | null = null
+
+  const stopTimer = () => {
+    if (timer) clearInterval(timer)
+    timer = null
+  }
+
+  const release = () => {
+    stopTimer()
+    recorder?.close()
+    recorder = null
+    recording.value = false
+  }
+
+  const start = () => {
+    if (recorder) return
+    const instance: RecorderInstance = new Recorder({
+      type: 'mp3',
+      bitRate: 128,
+      sampleRate: 16000,
+      onProcess: (_buffers: unknown, powerLevel: number) => {
+        waveLevels.value = [...waveLevels.value.slice(1), powerLevel]
+      },
+    })
+    recorder = instance
+    instance.open(
+      () => {
+        // 打开权限期间已取消
+        if (recorder !== instance) return instance.close()
+        seconds.value = 0
+        waveLevels.value = Array(WAVE_BAR_COUNT).fill(0)
+        instance.start()
+        recording.value = true
+        timer = setInterval(() => {
+          seconds.value++
+          if (seconds.value >= VOICE_RECORD_MAX_SECONDS) stop()
+        }, 1000)
+      },
+      (message: string) => {
+        if (recorder === instance) recorder = null
+        MsgError(`无法录音：${message}`)
+      },
+    )
+  }
+
+  /** 结束录音并回调录音文件。 */
+  const stop = () => {
+    const instance = recorder
+    if (!instance || !recording.value) return
+    stopTimer()
+    instance.stop(
+      (audio: Blob) => {
+        release()
+        onRecorded(audio)
+      },
+      (message: string) => {
+        release()
+        MsgError(`录音失败：${message}`)
+      },
+    )
+  }
+
+  /** 放弃本次录音，不转写。 */
+  const cancel = release
+
+  onBeforeUnmount(release)
+
+  return { recording, seconds, waveLevels, start, stop, cancel }
+}

ui/src/conversation-panel/components/message-input/voice-recorder/index.vue

@@ -0,0 +1,44 @@
+<script setup lang="ts">
+import { computed } from 'vue'
+
+/** 录音中面板：替换输入框显示波形、录音时长和结束按钮，录音状态由 useVoiceRecorder 维护。 */
+const props = defineProps<{
+  /** 已录制秒数 */
+  seconds: number
+  /** 各波形柱音量，0-100 */
+  waveLevels: number[]
+}>()
+
+const emit = defineEmits<{
+  stop: []
+}>()
+
+/** 波形柱最大高度（px），与波形容器 h-8 一致。 */
+const WAVE_MAX_HEIGHT = 32
+
+const recordTime = computed(() => {
+  const minutes = String(Math.floor(props.seconds / 60)).padStart(2, '0')
+  const seconds = String(props.seconds % 60).padStart(2, '0')
+  return `${minutes}:${seconds}`
+})
+</script>
+
+<template>
+  <div class="mk-input-box mk-conversation-message-input border-primary! bg-primary/[0.06] w-full shadow-lg">
+    <div class="flex h-8 items-center justify-center gap-0.5">
+      <span
+        v-for="(level, index) in waveLevels"
+        :key="index"
+        class="bg-primary w-0.5 rounded-full transition-[height] duration-100"
+        :style="{ height: `${Math.max(2, (level / 100) * WAVE_MAX_HEIGHT)}px` }"
+      />
+    </div>
+    <div class="mt-3 flex items-center justify-end">
+      <span class="text-N600 mr-3 text-xs">{{ recordTime }}</span>
+      <!-- 结束录音并转文字 -->
+      <el-button circle type="primary" @click="emit('stop')">
+        <MkIcon name="icon_square_filled" />
+      </el-button>
+    </div>
+  </div>
+</template>

ui/src/conversation-panel/view/chat/index.ts

@@ -114,6 +114,7 @@ export function createChatConversation(routing: ChatConversationRouting = useApp
     sendMessage: msgs.sendMessage,
     uploadFile: msgs.uploadFile,
     stop: msgs.stop,
+    speechToText: (data) => ConversationApi.postSpeechToText(getApplicationId(), data),
     appInfo,
     // 门户首页、全部智能体页尚未确定智能体时禁止发送
     disabled: computed(() => !getApplicationId()),

ui/src/conversation-panel/view/debug/index.ts

@@ -69,6 +69,7 @@ export function createDebugConversation(application: Readonly<Ref<ApplicationDet
     sendMessage: msgs.sendMessage,
     uploadFile: msgs.uploadFile,
     stop: msgs.stop,
+    speechToText: (data) => ConversationApi.postSpeechToText(appId.value, data),
     appInfo,
     apiInputEditable: true,
   })