import { BaseEvent } from "@/utils/base-event"; import { isMobile } from "@/utils"; /** * language:语种 * zh_cn:中文(支持简单的英文识别) * en_us:英文 * domain:应用领域 * iat:日常用语 * accent:方言 * mandarin:中文普通话、其他语种 * vad_eos:用于设置端点检测的静默时间,单位是毫秒。 * ptt:(仅中文支持)是否开启标点符号添加 * 1:开启(默认值) * 0:关闭 * nunum:(中文普通话和日语支持)将返回结果的数字格式规则为阿拉伯数字格式,默认开启 * 0:关闭 * 1:开启 */ type AsrServerOptions = { language: string; domain: string; accent: string; vad_eos: number; dwa: string; }; const DEFAULT_ASR_SERVER_OPTIONS: AsrServerOptions = { language: "zh-cn", domain: "iat", accent: "mandarin", vad_eos: 10000, dwa: "wpgs", }; const isXunFei = globalThis.isXF; const closeCommand = isXunFei ? { data: { status: 2 } } : { command: "close" }; const CHUNK_MULTIPLE = isXunFei ? 1 : 4; const CHUNK_LENGTH = 1280 * CHUNK_MULTIPLE; const CHECK_INTERVAL = isXunFei ? 40 : 80; /** * * @example 音频流 实时文本翻译 * ``` * let audioTextTransformer = new AudioTextTransformer(); * await audioTextTransformer.openMicroPhone(); * audioTextTramsformter.addEventListener("transform",(arg)=>{ * console.log(`当前累计接受到的文本为:` ,arg.transformText) * }) * await audioTextTramsformer.openSocket(); * * // 关闭socket * audioTextTransformer.close() * ``` * * @example 适用整段音频翻译,适用手机长按交互方式 * ``` * let audioTextTransformer = new AudioTextTransformer(); * await audioTextTransformer.openMicroPhone(); * await audioTextTransformer.openSocket(); * let transformText = await audioTextTransformer.transform(); * ``` */ export default class ASR extends BaseEvent { private socket: WebSocket | null = null; private _isTransformed: boolean = false; appKey: string; CONSTANTS: { [key: string]: any }; audioData: number[] = []; audioContext: AudioContext | null = null; source: MediaStreamAudioSourceNode | null = null; processor: ScriptProcessorNode | null = null; stream: MediaStream | null = null; transformText: string = ""; transformTextFix: number = 0; handleSendInterval: number | null = null; token: string = ""; socketPromise: Promise | null = null; asrServerOptions: AsrServerOptions; isPrivilegeGranted: boolean = false; processing: boolean = false; constructor( appKey: string, CONSTANTS = { audio: { echoCancellation: false, noiseSuppression: false, autoGainControl: false }, video: false, }, asrServerOptions: AsrServerOptions = DEFAULT_ASR_SERVER_OPTIONS ) { super(); this.appKey = appKey; this.CONSTANTS = CONSTANTS; this.asrServerOptions = { ...DEFAULT_ASR_SERVER_OPTIONS, ...asrServerOptions }; } private _init() { this.socket = null; this.audioData = []; this.transformText = ""; this.transformTextFix = 0; this._isTransformed = false; } async fetchToken() { if (isXunFei) { const asr_socket = import.meta.env.VITE_XF_ASR; const url = `${asr_socket}/api/human_asr/v2/asr/gen_auth`; const response = await fetch(url, { method: "post", body: JSON.stringify({ app_key: this.appKey }), }); const result = await response.json(); if (result.err_code !== 0) { throw new Error(result.err_msg); } this.token = result.ret; } else { // this.token = globalThis.token; this.token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyaWQiOjkxLCJjb21wYW55X2lkIjo1OSwiZXhwIjoyMDUwOTkxNjU2LCJpYXQiOjE3MzU2MzE2NTYsImlzcyI6Imh1bWFuLWxhcmdlLXNjcmVlbiIsInNjb3BlIjoiIn0.PTR5i63VlosjiepcyV2_KAIQnT-ySFJjggOhqvG5l1Y"; } } async openMicroPhone() { try { this.stream = await navigator.mediaDevices.getUserMedia(this.CONSTANTS); if (navigator.permissions) { navigator.permissions.query({ name: "microphone" }).then((permissionStatus) => { console.log("Microphone permission state:", permissionStatus.state); if (permissionStatus.state === "denied") { throw new Error("麦克风权限被拒绝,请在系统设置或浏览器设置中启用!"); } }); } } catch (e) { console.warn(e); throw new Error("获取麦克风流失败,请检查麦克风是否正常工作"); return false; } if (!this.isPrivilegeGranted) { this.isPrivilegeGranted = true; return true; } return true; } startRecord() { this._init(); this.audioContext = new window.AudioContext(); if (!this.stream) { this._eventbus.emit("error", { message: "未获取到麦克风流" }); return; } this.source = this.audioContext.createMediaStreamSource(this.stream); this.processor = this.audioContext.createScriptProcessor(0, 1, 1); this.processor.connect(this.audioContext.destination); this.source.connect(this.processor); this.processor.onaudioprocess = (audioProcessEvent) => { if (!this.stream || !this.stream.active) { return; } const inputData = audioProcessEvent.inputBuffer.getChannelData(0); const temp = to16kHZ(inputData); const data = to16BitPCM(temp); this.audioData.push(...data); }; this.processing = true; this._isTransformed = false; } async openSocket(): Promise { const mid = ({ text: str, command = "" }) => { const is_quit = command; if (isXunFei) { this.transformText = this.transformText.slice(0, this.transformTextFix) + str; } else { this.transformText = str; } console.log("mid ===========", this.transformText); this._eventbus.emit("transform", { type: "mid", current: str, transformText: this.transformText, command: is_quit ? "quit" : "", }); }; const fin = ({ text: str, command = "" }) => { this.transformTextFix = this.transformText.length; const is_quit = command; if (isXunFei) { this.transformText = this.transformText + str; } else { this.transformText = str; } console.log("fin ===========", this.transformText); this._eventbus.emit("transform", { type: "fin", current: str, transformText: this.transformText, command: is_quit ? "quit" : "", }); }; const breath = ({ text: str, command = "" }) => { const is_quit = command; if (isXunFei) { this.transformText = this.transformText.slice(0, this.transformTextFix) + str; } else { this.transformText = str; } this._eventbus.emit("transform", { type: "breath", current: str, transformText: this.transformText, command: is_quit ? "quit" : "", }); }; if (this.handleSendInterval) { clearInterval(this.handleSendInterval); } await this.fetchToken(); this.socketPromise = new Promise((resolve, reject) => { let ws; const protocol = location.href.includes("https") ? "wss://" : "ws://"; const domain = location.host; const asrSocket = isXunFei ? import.meta.env.VITE_XF_ASR : import.meta.env.VITE_ASR; let socketHost = asrSocket.startsWith("https") || asrSocket.startsWith("ws") ? new URL(asrSocket).host : domain + (asrSocket.startsWith("/") ? asrSocket : ""); let url = ""; if (isXunFei) { url = `${protocol}${socketHost}/api/human_asr/v2/asr?Authorization=${this.token}`; } else { const project = "qingpu"; const engine = "aliyun_dashscope"; url = `${protocol}${socketHost}/common/asr_hub?project=${project}&engine=${engine}`; url += `&is_long_connection=true` url += `&heartbeat=true` if (isMobile()) { const max_sentence_silence = 5000; url += `&max_sentence_silence=${max_sentence_silence}`; } } ws = new WebSocket(url); this.socket = ws; ws.onopen = () => { const params = { business: { language: this.asrServerOptions.language, domain: this.asrServerOptions.domain, accent: this.asrServerOptions.accent, vad_eos: this.asrServerOptions.vad_eos, dwa: this.asrServerOptions.dwa, }, data: { status: 0, format: "audio/L16;rate=16000", encoding: "raw", audio: "", }, }; if (isXunFei) { ws.send(JSON.stringify(params)); } else { const firstCommand = { card_id: this.asrServerOptions.card_id }; ws.send(JSON.stringify(firstCommand)); } if (this.processing) { this._handleSendAudio(); } else { this.close(); } resolve(); }; ws.onmessage = async (e: MessageEvent) => { if (!e.data) { return; } const response = JSON.parse(e.data); if (isXunFei) { if (response.code == "10165") { await this.openSocket(); return; } else if (response.code != "0") { console.warn(response); return; } const data = response.data.result; let str = ""; const ws = data.ws; for (let i = 0; i < ws.length; i++) { str += ws[i].cw[0].w; } if (data.pgs) { if (data.pgs === "apd") { fin({ text: str }); } else { mid({ text: str }); } } else { fin({ text: str }); } if (data.ls && this.audioData.length && !this._isTransformed) { await this.openSocket(); } } else { const data = response; if (data.command === "start") { mid({ text: data.text, command: data.is_quit }); } else if (data.command === "stop") { fin({ text: data.text, command: data.is_quit }); } const commas = [ "。", ",", "!", ",", "?", ";", ":", "、", "”", "’", ")", "》", "】", ]; if (data.text && commas.includes(data.text[data.text.length - 1]) && data.is_sentence) { breath({ text: data.text, command: data.is_quit }); } } }; ws.onerror = (e) => { this._eventbus.emit("error", e); reject(); }; }); return this.socketPromise; } private _handleSendAudio() { this._isTransformed = false; this.handleSendInterval = setInterval(() => { if (!isXunFei && this.audioData.length < CHUNK_LENGTH) { return; } const data = this.audioData.splice(0, CHUNK_LENGTH); if ( !this.socket || this.socket.readyState !== WebSocket.OPEN || data.length === 0 || this._isTransformed ) { return; } const buffer = new Int8Array(data); this.socket.send(buffer); }, CHECK_INTERVAL); } private _takeOverSendAudioData(): Promise { return new Promise((resolve, reject) => { if (this.audioData.length === 0) { return resolve(); } const handleSendInterval = setInterval(() => { if (this.audioData.length === 0) { clearInterval(handleSendInterval); return resolve(); } if (!this.socket) { clearInterval(handleSendInterval); reject(); return; } if ( this.socket.readyState == WebSocket.CONNECTING || this.socket.readyState == WebSocket.CLOSED || this.socket.readyState == WebSocket.CLOSING ) { clearInterval(handleSendInterval); reject(); return; } const data = this.audioData.splice(0, CHUNK_LENGTH); if (data.length === 0) { return; } const buffer = new Int8Array(data); this.socket.send(buffer); }, CHECK_INTERVAL) as unknown as number; }); } /** * 停止录音,发送 end,等待讯飞关闭连接,完成所有文字转换 */ async transform(): Promise<{ text: string; command: string }> { this._isTransformed = true; if (this.stream) { // 断开麦克风的输入 this.stream.getTracks().forEach((track) => track.stop()); } this.source?.disconnect(); this.processor?.disconnect(); // 等待所有音频数据发送完毕 clearInterval(this.handleSendInterval); this.handleSendInterval = null; let tryCount = 0; const MaxTryCount = 5; while (this.audioData.length) { if ( this.socket && this.socket.readyState !== WebSocket.OPEN && this.socket.readyState !== WebSocket.CONNECTING ) { if (!isXunFei) { break; } try { if (this.audioData.length) { await this.openSocket(); } else { // 没有音频数据,直接跳出循环 break; } } catch (e) { tryCount += 1; // Socket 连接失败尝试, 超过5次无视剩余数据,直接跳出循环,并放回当前获得的文本 if (MaxTryCount < tryCount) { break; } continue; } } try { await this.socketPromise; await this._takeOverSendAudioData(); tryCount = 0; } catch (e) { console.warn(e); } } const socket = this.socket; const promise: Promise<{ text: string; command: string }> = new Promise((resolve) => { if (!socket || socket.readyState !== WebSocket.OPEN) { resolve({ text: this.transformText, command: "end" }); this.close(); return; } const onLastMessage = (e: MessageEvent) => { if (!e.data) { return; } const response = JSON.parse(e.data); if (isXunFei) { if (response.code !== 0) { return; } const data = response.data.result; if (data.ls && this.audioData.length === 0 && this._isTransformed) { resolve({ text: this.transformText, command: "end" }); this.close(); } } else { const { text, command, is_sentence } = response; if (is_sentence && this.audioData.length == 0 && this._isTransformed) { resolve({ text, command }); } this.close(); } }; const onClose = () => { socket?.removeEventListener("close", onClose); resolve({ text: this.transformText, command: "end" }); this.close(); }; socket.addEventListener("message", onLastMessage); socket.addEventListener("close", onClose); }); socket.send(JSON.stringify(closeCommand)); return promise; } /** * 停止录音 并关闭Socket连接。 */ close() { this.audioData = []; if (this.handleSendInterval) { clearInterval(this.handleSendInterval); this.handleSendInterval = null; } this.processing = false; this._isTransformed = false; if (this.socket && this.socket.readyState === WebSocket.OPEN) { this.socket.send(JSON.stringify(closeCommand)); this.clear(); } this.stream?.getTracks().forEach((track) => track.stop()); } /** * 清除Socket连接,事件 */ clear() { this.source?.disconnect(); this.processor?.disconnect(); if ( isXunFei && this.socket && (this.socket.readyState === WebSocket.OPEN || this.socket.readyState === WebSocket.CONNECTING) ) { this.socket.close(); } if (this.handleSendInterval) { clearInterval(this.handleSendInterval); } this._init(); this.allClear(); } } function to16kHZ(buffer: Float32Array): Float32Array { const data = new Float32Array(buffer); const fitCount = Math.round(data.length * (16000 / 44100)); const newData = new Float32Array(fitCount); const springFactor = (data.length - 1) / (fitCount - 1); newData[0] = data[0]; for (let i = 1; i < fitCount - 1; i++) { const tmp = i * springFactor; const before = Math.floor(tmp); const after = Math.ceil(tmp); const atPoint = tmp - before; newData[i] = data[before] + (data[after] - data[before]) * atPoint; } newData[fitCount - 1] = data[data.length - 1]; return newData; } function to16BitPCM(input) { const dataLength = input.length * (16 / 8); const dataBuffer = new ArrayBuffer(dataLength); const dataView = new DataView(dataBuffer); let offset = 0; for (let i = 0; i < input.length; i++, offset += 2) { const s = Math.max(-1, Math.min(1, input[i])); dataView.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true); } return Array.from(new Int8Array(dataView.buffer)); }