zhaoshang_client_ui/src/three-libs/asr/index.ts

import { BaseEvent } from "@/utils/base-event";
import { isMobile } from "@/utils";

/**
 * language:语种
 *    zh_cn：中文（支持简单的英文识别）
 *    en_us：英文
 * domain：应用领域
 *    iat：日常用语
 * accent：方言
 *    mandarin：中文普通话、其他语种
 * vad_eos：用于设置端点检测的静默时间，单位是毫秒。
 * ptt：（仅中文支持）是否开启标点符号添加
 *    1：开启（默认值）
 *    0：关闭
 * nunum：（中文普通话和日语支持）将返回结果的数字格式规则为阿拉伯数字格式，默认开启
 *    0：关闭
 *    1：开启
 */
type AsrServerOptions = {
  language: string;
  domain: string;
  accent: string;
  vad_eos: number;
  dwa: string;
};

const DEFAULT_ASR_SERVER_OPTIONS: AsrServerOptions = {
  language: "zh-cn",
  domain: "iat",
  accent: "mandarin",
  vad_eos: 10000,
  dwa: "wpgs",
};

const isXunFei = globalThis.isXF;
const closeCommand = isXunFei ? { data: { status: 2 } } : { command: "close" };
const CHUNK_MULTIPLE = isXunFei ? 1 : 4;
const CHUNK_LENGTH = 1280 * CHUNK_MULTIPLE;
const CHECK_INTERVAL = isXunFei ? 40 : 80;

/**
 *
 * @example 音频流 实时文本翻译
 *  ```
 *  let audioTextTransformer  = new AudioTextTransformer();
 * await audioTextTransformer.openMicroPhone();
 * audioTextTramsformter.addEventListener("transform",(arg)=>{
 *   console.log(`当前累计接受到的文本为:` ,arg.transformText)
 * })
 * await audioTextTramsformer.openSocket();
 *
 * // 关闭socket
 * audioTextTransformer.close()
 * ```
 *
 * @example 适用整段音频翻译，适用手机长按交互方式
 * ```
 * let audioTextTransformer = new AudioTextTransformer();
 * await audioTextTransformer.openMicroPhone();
 * await audioTextTransformer.openSocket();
 * let transformText = await audioTextTransformer.transform();
 * ```
 */
export default class ASR extends BaseEvent {
  private socket: WebSocket | null = null;
  private _isTransformed: boolean = false;
  appKey: string;
  CONSTANTS: { [key: string]: any };
  audioData: number[] = [];
  audioContext: AudioContext | null = null;
  source: MediaStreamAudioSourceNode | null = null;
  processor: ScriptProcessorNode | null = null;
  stream: MediaStream | null = null;
  transformText: string = "";
  transformTextFix: number = 0;
  handleSendInterval: number | null = null;
  token: string = "";
  socketPromise: Promise<void> | null = null;
  asrServerOptions: AsrServerOptions;
  isPrivilegeGranted: boolean = false;
  processing: boolean = false;

  constructor(
    appKey: string,
    CONSTANTS = {
      audio: { echoCancellation: false, noiseSuppression: false, autoGainControl: false },
      video: false,
    },
    asrServerOptions: AsrServerOptions = DEFAULT_ASR_SERVER_OPTIONS
  ) {
    super();
    this.appKey = appKey;
    this.CONSTANTS = CONSTANTS;
    this.asrServerOptions = { ...DEFAULT_ASR_SERVER_OPTIONS, ...asrServerOptions };
  }

  private _init() {
    this.socket = null;
    this.audioData = [];
    this.transformText = "";
    this.transformTextFix = 0;
    this._isTransformed = false;
  }

  async fetchToken() {
    if (isXunFei) {
      const asr_socket = import.meta.env.VITE_XF_ASR;
      const url = `${asr_socket}/api/human_asr/v2/asr/gen_auth`;
      const response = await fetch(url, {
        method: "post",
        body: JSON.stringify({ app_key: this.appKey }),
      });
      const result = await response.json();
      if (result.err_code !== 0) {
        throw new Error(result.err_msg);
      }
      this.token = result.ret;
    } else {
      // this.token = globalThis.token;
      this.token =
        "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyaWQiOjkxLCJjb21wYW55X2lkIjo1OSwiZXhwIjoyMDUwOTkxNjU2LCJpYXQiOjE3MzU2MzE2NTYsImlzcyI6Imh1bWFuLWxhcmdlLXNjcmVlbiIsInNjb3BlIjoiIn0.PTR5i63VlosjiepcyV2_KAIQnT-ySFJjggOhqvG5l1Y";
    }
  }

  async openMicroPhone() {
    try {
      this.stream = await navigator.mediaDevices.getUserMedia(this.CONSTANTS);

      if (navigator.permissions) {
        navigator.permissions.query({ name: "microphone" }).then((permissionStatus) => {
          console.log("Microphone permission state:", permissionStatus.state);

          if (permissionStatus.state === "denied") {
            throw new Error("麦克风权限被拒绝，请在系统设置或浏览器设置中启用！");
          }
        });
      }
    } catch (e) {
      console.warn(e);
      throw new Error("获取麦克风流失败，请检查麦克风是否正常工作");
      return false;
    }

    if (!this.isPrivilegeGranted) {
      this.isPrivilegeGranted = true;
      return true;
    }

    return true;
  }

  startRecord() {
    this._init();
    this.audioContext = new window.AudioContext();

    if (!this.stream) {
      this._eventbus.emit("error", { message: "未获取到麦克风流" });
      return;
    }

    this.source = this.audioContext.createMediaStreamSource(this.stream);
    this.processor = this.audioContext.createScriptProcessor(0, 1, 1);
    this.processor.connect(this.audioContext.destination);
    this.source.connect(this.processor);
    this.processor.onaudioprocess = (audioProcessEvent) => {
      if (!this.stream || !this.stream.active) {
        return;
      }
      const inputData = audioProcessEvent.inputBuffer.getChannelData(0);
      const temp = to16kHZ(inputData);
      const data = to16BitPCM(temp);
      this.audioData.push(...data);
    };
    this.processing = true;
    this._isTransformed = false;
  }

  async openSocket(): Promise<void> {
    const mid = ({ text: str, command = "" }) => {
      const is_quit = command;
      if (isXunFei) {
        this.transformText = this.transformText.slice(0, this.transformTextFix) + str;
      } else {
        this.transformText = str;
      }
      console.log("mid  ===========", this.transformText);
      this._eventbus.emit("transform", {
        type: "mid",
        current: str,
        transformText: this.transformText,
        command: is_quit ? "quit" : "",
      });
    };
    const fin = ({ text: str, command = "" }) => {
      this.transformTextFix = this.transformText.length;
      const is_quit = command;
      if (isXunFei) {
        this.transformText = this.transformText + str;
      } else {
        this.transformText = str;
      }
      console.log("fin  ===========", this.transformText);
      this._eventbus.emit("transform", {
        type: "fin",
        current: str,
        transformText: this.transformText,
        command: is_quit ? "quit" : "",
      });
    };
    const breath = ({ text: str, command = "" }) => {
      const is_quit = command;
      if (isXunFei) {
        this.transformText = this.transformText.slice(0, this.transformTextFix) + str;
      } else {
        this.transformText = str;
      }
      this._eventbus.emit("transform", {
        type: "breath",
        current: str,
        transformText: this.transformText,
        command: is_quit ? "quit" : "",
      });
    };

    if (this.handleSendInterval) {
      clearInterval(this.handleSendInterval);
    }

    await this.fetchToken();

    this.socketPromise = new Promise((resolve, reject) => {
      let ws;
      const protocol = location.href.includes("https") ? "wss://" : "ws://";
      const domain = location.host;
      const asrSocket = isXunFei ? import.meta.env.VITE_XF_ASR : import.meta.env.VITE_ASR;
      let socketHost =
        asrSocket.startsWith("https") || asrSocket.startsWith("ws")
          ? new URL(asrSocket).host
          : domain + (asrSocket.startsWith("/") ? asrSocket : "");
      let url = "";

      if (isXunFei) {
        url = `${protocol}${socketHost}/api/human_asr/v2/asr?Authorization=${this.token}`;
      } else {
        const project = "qingpu";
        const engine = "aliyun_dashscope";
        url = `${protocol}${socketHost}/common/asr_hub?project=${project}&engine=${engine}`;

        url += `&is_long_connection=true`
        url += `&heartbeat=true`

        if (isMobile()) {
          const max_sentence_silence = 5000;
          url += `&max_sentence_silence=${max_sentence_silence}`;
        }
      }

      ws = new WebSocket(url);
      this.socket = ws;
      ws.onopen = () => {
        const params = {
          business: {
            language: this.asrServerOptions.language,
            domain: this.asrServerOptions.domain,
            accent: this.asrServerOptions.accent,
            vad_eos: this.asrServerOptions.vad_eos,
            dwa: this.asrServerOptions.dwa,
          },
          data: {
            status: 0,
            format: "audio/L16;rate=16000",
            encoding: "raw",
            audio: "",
          },
        };
        if (isXunFei) {
          ws.send(JSON.stringify(params));
        } else {
          const firstCommand = { card_id: this.asrServerOptions.card_id };
          ws.send(JSON.stringify(firstCommand));
        }

        if (this.processing) {
          this._handleSendAudio();
        } else {
          this.close();
        }
        resolve();
      };

      ws.onmessage = async (e: MessageEvent<string>) => {
        if (!e.data) {
          return;
        }

        const response = JSON.parse(e.data);
        if (isXunFei) {
          if (response.code == "10165") {
            await this.openSocket();
            return;
          } else if (response.code != "0") {
            console.warn(response);
            return;
          }

          const data = response.data.result;
          let str = "";
          const ws = data.ws;
          for (let i = 0; i < ws.length; i++) {
            str += ws[i].cw[0].w;
          }
          if (data.pgs) {
            if (data.pgs === "apd") {
              fin({ text: str });
            } else {
              mid({ text: str });
            }
          } else {
            fin({ text: str });
          }
          if (data.ls && this.audioData.length && !this._isTransformed) {
            await this.openSocket();
          }
        } else {
          const data = response;
          if (data.command === "start") {
            mid({ text: data.text, command: data.is_quit });
          } else if (data.command === "stop") {
            fin({ text: data.text, command: data.is_quit });
          }

          const commas = [
            "。",
            "，",
            "！",
            ",",
            "？",
            "；",
            "：",
            "、",
            "”",
            "’",
            "）",
            "》",
            "】",
          ];
          if (data.text && commas.includes(data.text[data.text.length - 1]) && data.is_sentence) {
            breath({ text: data.text, command: data.is_quit });
          }
        }
      };

      ws.onerror = (e) => {
        this._eventbus.emit("error", e);
        reject();
      };
    });

    return this.socketPromise;
  }

  private _handleSendAudio() {
    this._isTransformed = false;
    this.handleSendInterval = setInterval(() => {
      if (!isXunFei && this.audioData.length < CHUNK_LENGTH) {
        return;
      }

      const data = this.audioData.splice(0, CHUNK_LENGTH);
      if (
        !this.socket ||
        this.socket.readyState !== WebSocket.OPEN ||
        data.length === 0 ||
        this._isTransformed
      ) {
        return;
      }

      const buffer = new Int8Array(data);
      this.socket.send(buffer);
    }, CHECK_INTERVAL);
  }

  private _takeOverSendAudioData(): Promise<void> {
    return new Promise((resolve, reject) => {
      if (this.audioData.length === 0) {
        return resolve();
      }
      const handleSendInterval = setInterval(() => {
        if (this.audioData.length === 0) {
          clearInterval(handleSendInterval);
          return resolve();
        }
        if (!this.socket) {
          clearInterval(handleSendInterval);
          reject();
          return;
        }
        if (
          this.socket.readyState == WebSocket.CONNECTING ||
          this.socket.readyState == WebSocket.CLOSED ||
          this.socket.readyState == WebSocket.CLOSING
        ) {
          clearInterval(handleSendInterval);
          reject();
          return;
        }
        const data = this.audioData.splice(0, CHUNK_LENGTH);
        if (data.length === 0) {
          return;
        }
        const buffer = new Int8Array(data);
        this.socket.send(buffer);
      }, CHECK_INTERVAL) as unknown as number;
    });
  }

  /**
   * 停止录音,发送 end，等待讯飞关闭连接，完成所有文字转换
   */
  async transform(): Promise<{ text: string; command: string }> {
    this._isTransformed = true;
    if (this.stream) {
      // 断开麦克风的输入
      this.stream.getTracks().forEach((track) => track.stop());
    }
    this.source?.disconnect();
    this.processor?.disconnect();
    // 等待所有音频数据发送完毕
    clearInterval(this.handleSendInterval);
    this.handleSendInterval = null;

    let tryCount = 0;
    const MaxTryCount = 5;
    while (this.audioData.length) {
      if (
        this.socket &&
        this.socket.readyState !== WebSocket.OPEN &&
        this.socket.readyState !== WebSocket.CONNECTING
      ) {
        if (!isXunFei) {
          break;
        }

        try {
          if (this.audioData.length) {
            await this.openSocket();
          } else {
            // 没有音频数据，直接跳出循环
            break;
          }
        } catch (e) {
          tryCount += 1;
          // Socket 连接失败尝试， 超过5次无视剩余数据，直接跳出循环，并放回当前获得的文本
          if (MaxTryCount < tryCount) {
            break;
          }
          continue;
        }
      }
      try {
        await this.socketPromise;
        await this._takeOverSendAudioData();
        tryCount = 0;
      } catch (e) {
        console.warn(e);
      }
    }

    const socket = this.socket;
    const promise: Promise<{ text: string; command: string }> = new Promise((resolve) => {
      if (!socket || socket.readyState !== WebSocket.OPEN) {
        resolve({ text: this.transformText, command: "end" });
        this.close();
        return;
      }
      const onLastMessage = (e: MessageEvent<string>) => {
        if (!e.data) {
          return;
        }

        const response = JSON.parse(e.data);
        if (isXunFei) {
          if (response.code !== 0) {
            return;
          }
          const data = response.data.result;
          if (data.ls && this.audioData.length === 0 && this._isTransformed) {
            resolve({ text: this.transformText, command: "end" });
            this.close();
          }
        } else {
          const { text, command, is_sentence } = response;
          if (is_sentence && this.audioData.length == 0 && this._isTransformed) {
            resolve({ text, command });
          }
          this.close();
        }
      };
      const onClose = () => {
        socket?.removeEventListener("close", onClose);
        resolve({ text: this.transformText, command: "end" });
        this.close();
      };
      socket.addEventListener("message", onLastMessage);
      socket.addEventListener("close", onClose);
    });

    socket.send(JSON.stringify(closeCommand));
    return promise;
  }
  /**
   * 停止录音 并关闭Socket连接。
   */
  close() {
    this.audioData = [];
    if (this.handleSendInterval) {
      clearInterval(this.handleSendInterval);
      this.handleSendInterval = null;
    }
    this.processing = false;
    this._isTransformed = false;
    if (this.socket && this.socket.readyState === WebSocket.OPEN) {
      this.socket.send(JSON.stringify(closeCommand));
      this.clear();
    }
    this.stream?.getTracks().forEach((track) => track.stop());
  }
  /**
   * 清除Socket连接，事件
   */
  clear() {
    this.source?.disconnect();
    this.processor?.disconnect();
    if (
      isXunFei &&
      this.socket &&
      (this.socket.readyState === WebSocket.OPEN || this.socket.readyState === WebSocket.CONNECTING)
    ) {
      this.socket.close();
    }
    if (this.handleSendInterval) {
      clearInterval(this.handleSendInterval);
    }
    this._init();
    this.allClear();
  }
}

function to16kHZ(buffer: Float32Array): Float32Array {
  const data = new Float32Array(buffer);
  const fitCount = Math.round(data.length * (16000 / 44100));
  const newData = new Float32Array(fitCount);
  const springFactor = (data.length - 1) / (fitCount - 1);
  newData[0] = data[0];
  for (let i = 1; i < fitCount - 1; i++) {
    const tmp = i * springFactor;
    const before = Math.floor(tmp);
    const after = Math.ceil(tmp);
    const atPoint = tmp - before;
    newData[i] = data[before] + (data[after] - data[before]) * atPoint;
  }
  newData[fitCount - 1] = data[data.length - 1];
  return newData;
}

function to16BitPCM(input) {
  const dataLength = input.length * (16 / 8);
  const dataBuffer = new ArrayBuffer(dataLength);
  const dataView = new DataView(dataBuffer);
  let offset = 0;
  for (let i = 0; i < input.length; i++, offset += 2) {
    const s = Math.max(-1, Math.min(1, input[i]));
    dataView.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
  }
  return Array.from(new Int8Array(dataView.buffer));
}