/** * 实时语音识别(阿里云 NLS 版)—— 与旧厂商 SDK(`./index.ts`)**对外契约一致**: * * new ASR(appKey?, CONSTANTS?) · openMicroPhone() · startRecord() · openSocket() * · transform() · close() · clear() · 事件 `transform` / `error` * * 要点: * - 麦克风用 `navigator.mediaDevices.getUserMedia` 采集,PCM 以 16k / 16bit / 单声道推给网关 * - **重采样按实际采样率算**;**定稿只认 `SentenceEnd`**(中间结果会重叠,不能拼) * - 凭据:**前端不持有任何阿里云凭据**,token + appkey 由后端网关下发(见 `nls-token.ts`) * * 协议细节见 `./nls-protocol.ts`。 */ import { BaseEvent } from "@/utils/base-event"; import { fetchNlsToken, getNlsTokenAppkey } from "@/network/api/nls-token"; import { NLS_FLUSH_TICK_MS, NLS_FRAME_MS, NLS_GATEWAY_URL, NLS_MAX_FRAMES_PER_TICK, NLS_SAMPLE_RATE, NLS_STATUS_SUCCESS, buildStartTranscription, buildStopTranscription, createNlsId, createNlsTranscriptState, describeNlsFailure, encodePcm16kMono, getNlsEventName, parseNlsMessage, reduceNlsTranscript, } from "./nls-protocol"; /** 收尾时最多等网关多久(超过就用手上已有的文本收场,不让"停止"按钮卡住) */ const STOP_TIMEOUT_MS = 3000; /** 建连/开始识别最多等多久 */ const START_TIMEOUT_MS = 8000; export default class NlsAsr extends BaseEvent { /** 构造参数里的第一个仍是 appKey,但 NLS 版的 appkey 由端点下发,这里只作兜底 */ appKey: string; CONSTANTS: MediaStreamConstraints; stream: MediaStream | null = null; audioContext: AudioContext | null = null; source: MediaStreamAudioSourceNode | null = null; processor: ScriptProcessorNode | null = null; /** 已定稿全文(`transform()` 的返回值来源) */ private transcript = createNlsTranscriptState(); private socket: WebSocket | null = null; private taskId = ""; private appkey = ""; private pendingAudio: Uint8Array[] = []; private sendTimer: number | null = null; private processing = false; private stopping = false; private startResolve: (() => void) | null = null; private startReject: ((error: Error) => void) | null = null; private stopResolve: (() => void) | null = null; constructor(appKey = "", CONSTANTS: MediaStreamConstraints = { audio: true, video: false }) { super(); this.appKey = appKey; this.CONSTANTS = CONSTANTS; } /** 申请麦克风。失败抛 `Error`(调用方 catch 后 toast)——与旧实现的语义一致 */ async openMicroPhone(): Promise { try { this.stream = await navigator.mediaDevices.getUserMedia(this.CONSTANTS); } catch (error) { /* console.warn("[nls] 获取麦克风失败", error); */ // 区分"用户拒绝"与"设备异常":前者要给可操作的提示 if ((error as DOMException)?.name === "NotAllowedError") { throw new Error("麦克风权限被拒绝,请在浏览器或系统设置里允许后重试"); } throw new Error("获取麦克风流失败,请检查麦克风是否正常工作"); } return true; } /** * 开始采集音频。与旧实现一样**必须在 `openSocket()` 之前调用**。 * 采集到的 PCM 先缓存,连上后再按实时节奏推给网关。 */ startRecord(): void { this.pendingAudio = []; this.transcript = createNlsTranscriptState(); this.processing = true; this.stopping = false; if (!this.stream) { this._eventbus.emit("error", { message: "未获取到麦克风流" }); return; } this.audioContext = new window.AudioContext(); this.source = this.audioContext.createMediaStreamSource(this.stream); // ScriptProcessorNode 已废弃,但仓库的波形绘制依赖 audioContext/source, // 所以与旧实现共用同一套采集方式 this.processor = this.audioContext.createScriptProcessor(0, 1, 1); this.processor.connect(this.audioContext.destination); this.source.connect(this.processor); const sampleRate = this.audioContext.sampleRate; let capturedChunks = 0; let silenceWarned = false; /* console.log( `[nls] 开始采集:AudioContext=${this.audioContext.state} 采样率=${sampleRate}Hz ` + `processor 缓冲=${this.processor.bufferSize}` ); */ // 音轨自己的状态 —— 分辨「系统/硬件把麦克风静音了」与「接线有问题」的关键证据: // muted=true 时 getUserMedia 照样成功、回调照样触发,但采样全是 0。 const track = this.stream.getAudioTracks?.()[0]; if (track) { const settings = typeof track.getSettings === "function" ? track.getSettings() : {}; /* console.log( `[nls] 音轨:label=${JSON.stringify(track.label)} muted=${track.muted} enabled=${track.enabled} ` + `state=${track.readyState} deviceId=${String(settings.deviceId || "").slice(0, 8)}… ` + `声道=${settings.channelCount ?? "?"}` ); */ const reportTrackState = (tag: string) => { /* console.log(`[nls] ${tag}音轨状态:muted=${track.muted} enabled=${track.enabled} state=${track.readyState}`); */ }; window.setTimeout(() => reportTrackState("1.5s 后"), 1500); track.addEventListener?.("mute", () => reportTrackState("被静音时(mute 事件)")); track.addEventListener?.("unmute", () => reportTrackState("解除静音时(unmute 事件)")); } else { /* console.warn("[nls] ⚠️ 流里没有音轨(getUserMedia 拿到的不是音频流?)"); */ } this.processor.onaudioprocess = (event: AudioProcessingEvent) => { if (!this.processing || !this.stream || !this.stream.active) return; const channel = event.inputBuffer.getChannelData(0); const encoded = encodePcm16kMono(channel, sampleRate); capturedChunks += 1; if (capturedChunks === 1) { /* console.log(`[nls] 收到第一块音频:输入 ${channel.length} 采样 → 编码后 ${encoded.length} 字节`); */ } // 电平日志(每 ~2s 一条):峰值长期 ≈0 说明采集端没拿到声音,问题在设备/权限而不在协议 if (capturedChunks % 50 === 1) { let peak = 0; let sum = 0; for (let i = 0; i < channel.length; i++) { const value = Math.abs(channel[i]); if (value > peak) peak = value; sum += value * value; } const rms = Math.sqrt(sum / channel.length); /* console.log(`[nls] 采集电平:峰值=${peak.toFixed(4)} RMS=${rms.toFixed(4)}`); */ // 连续 3 秒全静音 → 把「当前用的是哪个输入设备」摊开说清楚 // (虚拟设备 / 系统静音 / 被会议软件独占,都会是"能采集但电平恒为 0") if (peak === 0 && !silenceWarned) { silenceWarned = true; const deviceLabel = this.stream?.getAudioTracks?.()[0]?.label || "(未知设备)"; /* console.warn( `[nls] ⚠️ 连续采集到全静音:当前输入设备是 ${JSON.stringify(deviceLabel)}。\n` + `[nls] 若它不是你正在说话的麦克风(常见的坑:Steam Streaming Microphone 这类**虚拟设备**、\n` + `[nls] 系统静音/隐私开关、被会议软件独占),请在 Windows「声音设置 → 输入」或\n` + `[nls] 浏览器「站点设置 → 麦克风」里换一个真实麦克风后重试。` ); */ } } this.pendingAudio.push(encoded); }; // AudioContext 有时会以 suspended 起步(那样 onaudioprocess 根本不会回调),主动 resume 一次 if (this.audioContext.state === "suspended") { void this.audioContext.resume().catch(() => undefined); } window.setTimeout(() => { if (capturedChunks === 0 && this.processing) { /* console.warn( `[nls] ⚠️ 1.5s 内没有采集到任何音频(AudioContext=${this.audioContext?.state},` + `stream.active=${this.stream?.active})—— 麦克风可能被别的程序占用或未真正启动` ); */ } }, 1500); } /** 取 token → 连网关 → 发 StartTranscription → 等 TranscriptionStarted */ async openSocket(): Promise { const { token } = await fetchNlsToken(); // appkey 来自配置(见 getNlsAppkey);构造参数只作兜底 this.appkey = getNlsTokenAppkey() || this.appKey; this.taskId = createNlsId(); await new Promise((resolve, reject) => { this.startResolve = resolve; this.startReject = reject; /** 诊断用:中间结果事件计数(日志里每 4 条打一条,避免刷屏) */ let midEvents = 0; const url = `${NLS_GATEWAY_URL}?token=${encodeURIComponent(token)}`; const socket = new WebSocket(url); this.socket = socket; const startTimer = window.setTimeout(() => { reject(new Error("语音识别连接超时(未收到 TranscriptionStarted)")); this.closeSocket(); }, START_TIMEOUT_MS); const settle = () => window.clearTimeout(startTimer); socket.onopen = () => { socket.send(buildStartTranscription({ appkey: this.appkey, taskId: this.taskId })); }; socket.onmessage = (event: MessageEvent) => { const message = parseNlsMessage(event.data); const name = getNlsEventName(message); if (!name) return; // 诊断日志:把网关下行事件打出来(ResultChanged 很密,每 4 条打一条), // 排查「进录音态但不出字」时能区分「音频没上去」和「服务端没回/回了错」 if (name === "TaskFailed") { /* console.warn(`[nls] ← TaskFailed:${describeNlsFailure(message)}`); */ } else if (name !== "TranscriptionResultChanged" && name !== "SentenceBegin") { /* console.log(`[nls] ← ${name}${message?.payload?.result ? ` ${JSON.stringify(message.payload.result)}` : ""}`); */ } else if (name === "TranscriptionResultChanged") { midEvents += 1; if (midEvents % 4 === 1) { /* console.log(`[nls] ← 中间结果:${JSON.stringify(message?.payload?.result ?? "")}`); */ } } if (name === "TranscriptionStarted") { settle(); this.startFlushTimer(); this.startResolve?.(); return; } if (name === "TaskFailed") { const error = new Error(describeNlsFailure(message)); settle(); this._eventbus.emit("error", { message: error.message }); this.startReject?.(error); return; } const { state, event: payload } = reduceNlsTranscript(this.transcript, message); this.transcript = state; if (payload) this._eventbus.emit("transform", payload); if (name === "TranscriptionCompleted") { this.stopResolve?.(); } }; socket.onerror = () => { settle(); const error = new Error("语音识别连接出错"); this._eventbus.emit("error", { message: error.message }); this.startReject?.(error); }; socket.onclose = () => { settle(); // 还没开始就断了 → 让 openSocket 的 await 失败,别静默卡住 this.startReject?.(new Error("语音识别连接已关闭")); this.stopResolve?.(); }; }); } /** * 结束一轮:停麦克风 → 把余下的音频发完 → 发 StopTranscription → 等 TranscriptionCompleted。 * 返回**累计定稿文本**(与旧实现的 `{text, command}` 形态一致)。 */ async transform(): Promise<{ text: string; command: string }> { this.stopping = true; this.processing = false; this.stopSendTimer(); if (this.stream) { this.stream.getTracks().forEach((track) => track.stop()); } this.source?.disconnect(); this.processor?.disconnect(); // 余下的音频尽快发完:用户此刻正在等最后一段文字,按 100ms 一帧的实时节奏发 // 会让"停止后还要等"的体感更差 —— 这里用 20ms 一帧(≈5× 实时) const frames = this.pendingAudio; this.pendingAudio = []; for (const frame of frames) { if (this.socket?.readyState !== WebSocket.OPEN) break; this.socket.send(frame); await new Promise((r) => window.setTimeout(r, 20)); } const finalText = this.transcript.finalText; if (this.socket?.readyState === WebSocket.OPEN) { this.socket.send(buildStopTranscription({ appkey: this.appkey, taskId: this.taskId })); await new Promise((resolve) => { this.stopResolve = resolve; const timer = window.setTimeout(() => { this.stopResolve = null; resolve(); }, STOP_TIMEOUT_MS); const original = this.stopResolve; this.stopResolve = () => { window.clearTimeout(timer); original?.(); }; }); } return { text: this.transcript.finalText || finalText, command: "end" }; } /** 停采集、收音频、断连接(壳组件在 `transform()` 之后调用) */ close(): void { this.processing = false; this.stopping = false; this.pendingAudio = []; this.stopSendTimer(); if (this.stream) { this.stream.getTracks().forEach((track) => track.stop()); } this.closeSocket(); } /** 清干净(断开音频节点、清事件监听)——调用后实例作废,调用方置 null */ clear(): void { this.source?.disconnect(); this.processor?.disconnect(); this.audioContext?.close?.().catch(() => { /* 关闭失败无所谓:上下文本来就随页面回收 */ }); this.audioContext = null; this.source = null; this.processor = null; this.stream = null; this.socket = null; this.pendingAudio = []; this.allClear(); } private closeSocket(): void { const socket = this.socket; this.socket = null; if (socket && (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING)) { try { socket.close(); } catch { /* ignore */ } } } private stopSendTimer(): void { if (this.sendTimer !== null) { window.clearInterval(this.sendTimer); this.sendTimer = null; } } /** * 把缓存里的音频按帧推给网关:**平时按实时节奏,积压时最多 4× 追赶** * (连上网关前麦克风已在采集,只按 1× 排空那段积压永远排不掉,表现为"出字整体慢半拍"; * 常量与说明见 nls-protocol.ts)。 */ private startFlushTimer(): void { this.stopSendTimer(); const bytesPerFrame = (NLS_SAMPLE_RATE * 2 * NLS_FRAME_MS) / 1000; // 3200 字节 = 100ms let carry = new Uint8Array(0); let sentFrames = 0; let lastLoggedFrames = 0; let stallWarnedAt = 0; const flushStartedAt = Date.now(); this.sendTimer = window.setInterval(() => { if (this.socket?.readyState !== WebSocket.OPEN) return; // 1) 把采集到的块**全部**并进 carry。 // ⚠️ 合并**不设上限**:一旦按「carry + 下一块 ≤ 两帧」限制,浏览器给的 buffer // 比两帧大时就永远合并不进来,结果**一帧音频都发不出去**(界面看着仍在正常录音)。 if (this.pendingAudio.length) { const pending = this.pendingAudio; this.pendingAudio = []; const merged = new Uint8Array(carry.length + pending.reduce((n, c) => n + c.length, 0)); merged.set(carry, 0); let offset = carry.length; for (const chunk of pending) { merged.set(chunk, offset); offset += chunk.length; } carry = merged; } // 2) 按帧发(有积压时一个 tick 最多发 NLS_MAX_FRAMES_PER_TICK 帧) let sentThisTick = 0; while (carry.length >= bytesPerFrame && sentThisTick < NLS_MAX_FRAMES_PER_TICK) { if (this.socket?.readyState !== WebSocket.OPEN) return; this.socket.send(carry.subarray(0, bytesPerFrame)); carry = carry.subarray(bytesPerFrame); sentThisTick += 1; sentFrames += 1; } if (sentFrames - lastLoggedFrames >= 20) { lastLoggedFrames = sentFrames; /* console.log(`[nls] 已推音频 ${Math.round((sentFrames * bytesPerFrame) / 32)}ms(${sentFrames} 帧)`); */ } // 3) 诊断:录着音却 >=2 秒一帧都没发出去 → 明确报出来,别"静默不出声" if (this.processing && sentFrames === 0 && Date.now() - flushStartedAt > 2000 && Date.now() - stallWarnedAt > 5000) { stallWarnedAt = Date.now(); /* console.warn( `[nls] ⚠️ 录音中但音频一帧都没发出去(carry=${carry.length}B,缓存块=${this.pendingAudio.length})` + ` —— 若持续如此,检查采集端是否真的在出数据(见上一条「采集开始」日志里的采样率)` ); */ } }, NLS_FLUSH_TICK_MS); } }