zhaoshang_client_ui/src/asr/nls.ts

/**
 * 实时语音识别（阿里云 NLS 版）—— 与旧厂商 SDK（`./index.ts`）**对外契约一致**：
 *
 *   new ASR(appKey?, CONSTANTS?)  ·  openMicroPhone()  ·  startRecord()  ·  openSocket()
 *   ·  transform()  ·  close()  ·  clear()  ·  事件 `transform` / `error`
 *
 * 要点：
 *   - 麦克风用 `navigator.mediaDevices.getUserMedia` 采集，PCM 以 16k / 16bit / 单声道推给网关
 *   - **重采样按实际采样率算**；**定稿只认 `SentenceEnd`**（中间结果会重叠，不能拼）
 *   - 凭据：**前端不持有任何阿里云凭据**，token + appkey 由后端网关下发（见 `nls-token.ts`）
 *
 * 协议细节见 `./nls-protocol.ts`。
 */

import { BaseEvent } from "@/utils/base-event";
import { fetchNlsToken, getNlsTokenAppkey } from "@/network/api/nls-token";
import {
  NLS_FLUSH_TICK_MS,
  NLS_FRAME_MS,
  NLS_GATEWAY_URL,
  NLS_MAX_FRAMES_PER_TICK,
  NLS_SAMPLE_RATE,
  NLS_STATUS_SUCCESS,
  buildStartTranscription,
  buildStopTranscription,
  createNlsId,
  createNlsTranscriptState,
  describeNlsFailure,
  encodePcm16kMono,
  getNlsEventName,
  parseNlsMessage,
  reduceNlsTranscript,
} from "./nls-protocol";

/** 收尾时最多等网关多久（超过就用手上已有的文本收场，不让"停止"按钮卡住） */
const STOP_TIMEOUT_MS = 3000;
/** 建连/开始识别最多等多久 */
const START_TIMEOUT_MS = 8000;

export default class NlsAsr extends BaseEvent {
  /** 构造参数里的第一个仍是 appKey，但 NLS 版的 appkey 由端点下发，这里只作兜底 */
  appKey: string;
  CONSTANTS: MediaStreamConstraints;

  stream: MediaStream | null = null;
  audioContext: AudioContext | null = null;
  source: MediaStreamAudioSourceNode | null = null;
  processor: ScriptProcessorNode | null = null;

  /** 已定稿全文（`transform()` 的返回值来源） */
  private transcript = createNlsTranscriptState();
  private socket: WebSocket | null = null;
  private taskId = "";
  private appkey = "";
  private pendingAudio: Uint8Array[] = [];
  private sendTimer: number | null = null;
  private processing = false;
  private stopping = false;
  private startResolve: (() => void) | null = null;
  private startReject: ((error: Error) => void) | null = null;
  private stopResolve: (() => void) | null = null;

  constructor(appKey = "", CONSTANTS: MediaStreamConstraints = { audio: true, video: false }) {
    super();
    this.appKey = appKey;
    this.CONSTANTS = CONSTANTS;
  }

  /** 申请麦克风。失败抛 `Error`（调用方 catch 后 toast）——与旧实现的语义一致 */
  async openMicroPhone(): Promise<boolean> {
    try {
      this.stream = await navigator.mediaDevices.getUserMedia(this.CONSTANTS);
    } catch (error) {
      /* console.warn("[nls] 获取麦克风失败", error); */
      // 区分"用户拒绝"与"设备异常"：前者要给可操作的提示
      if ((error as DOMException)?.name === "NotAllowedError") {
        throw new Error("麦克风权限被拒绝，请在浏览器或系统设置里允许后重试");
      }
      throw new Error("获取麦克风流失败，请检查麦克风是否正常工作");
    }
    return true;
  }

  /**
   * 开始采集音频。与旧实现一样**必须在 `openSocket()` 之前调用**。
   * 采集到的 PCM 先缓存，连上后再按实时节奏推给网关。
   */
  startRecord(): void {
    this.pendingAudio = [];
    this.transcript = createNlsTranscriptState();
    this.processing = true;
    this.stopping = false;

    if (!this.stream) {
      this._eventbus.emit("error", { message: "未获取到麦克风流" });
      return;
    }

    this.audioContext = new window.AudioContext();
    this.source = this.audioContext.createMediaStreamSource(this.stream);
    // ScriptProcessorNode 已废弃，但仓库的波形绘制依赖 audioContext/source，
    // 所以与旧实现共用同一套采集方式
    this.processor = this.audioContext.createScriptProcessor(0, 1, 1);
    this.processor.connect(this.audioContext.destination);
    this.source.connect(this.processor);
    const sampleRate = this.audioContext.sampleRate;
    let capturedChunks = 0;
    let silenceWarned = false;
    /* console.log(
      `[nls] 开始采集：AudioContext=${this.audioContext.state} 采样率=${sampleRate}Hz ` +
        `processor 缓冲=${this.processor.bufferSize}`
    ); */
    // 音轨自己的状态 —— 分辨「系统/硬件把麦克风静音了」与「接线有问题」的关键证据：
    // muted=true 时 getUserMedia 照样成功、回调照样触发，但采样全是 0。
    const track = this.stream.getAudioTracks?.()[0];
    if (track) {
      const settings = typeof track.getSettings === "function" ? track.getSettings() : {};
      /* console.log(
        `[nls] 音轨：label=${JSON.stringify(track.label)} muted=${track.muted} enabled=${track.enabled} ` +
          `state=${track.readyState} deviceId=${String(settings.deviceId || "").slice(0, 8)}… ` +
          `声道=${settings.channelCount ?? "?"}`
      ); */
      const reportTrackState = (tag: string) => {
        /* console.log(`[nls] ${tag}音轨状态：muted=${track.muted} enabled=${track.enabled} state=${track.readyState}`); */
      };
      window.setTimeout(() => reportTrackState("1.5s 后"), 1500);
      track.addEventListener?.("mute", () => reportTrackState("被静音时（mute 事件）"));
      track.addEventListener?.("unmute", () => reportTrackState("解除静音时（unmute 事件）"));
    } else {
      /* console.warn("[nls] ⚠️ 流里没有音轨（getUserMedia 拿到的不是音频流？）"); */
    }
    this.processor.onaudioprocess = (event: AudioProcessingEvent) => {
      if (!this.processing || !this.stream || !this.stream.active) return;
      const channel = event.inputBuffer.getChannelData(0);
      const encoded = encodePcm16kMono(channel, sampleRate);
      capturedChunks += 1;
      if (capturedChunks === 1) {
        /* console.log(`[nls] 收到第一块音频：输入 ${channel.length} 采样 → 编码后 ${encoded.length} 字节`); */
      }
      // 电平日志（每 ~2s 一条）：峰值长期 ≈0 说明采集端没拿到声音，问题在设备/权限而不在协议
      if (capturedChunks % 50 === 1) {
        let peak = 0;
        let sum = 0;
        for (let i = 0; i < channel.length; i++) {
          const value = Math.abs(channel[i]);
          if (value > peak) peak = value;
          sum += value * value;
        }
        const rms = Math.sqrt(sum / channel.length);
        /* console.log(`[nls] 采集电平：峰值=${peak.toFixed(4)} RMS=${rms.toFixed(4)}`); */
        // 连续 3 秒全静音 → 把「当前用的是哪个输入设备」摊开说清楚
        // （虚拟设备 / 系统静音 / 被会议软件独占，都会是"能采集但电平恒为 0"）
        if (peak === 0 && !silenceWarned) {
          silenceWarned = true;
          const deviceLabel = this.stream?.getAudioTracks?.()[0]?.label || "(未知设备)";
          /* console.warn(
            `[nls] ⚠️ 连续采集到全静音：当前输入设备是 ${JSON.stringify(deviceLabel)}。\n` +
              `[nls]    若它不是你正在说话的麦克风（常见的坑：Steam Streaming Microphone 这类**虚拟设备**、\n` +
              `[nls]    系统静音/隐私开关、被会议软件独占），请在 Windows「声音设置 → 输入」或\n` +
              `[nls]    浏览器「站点设置 → 麦克风」里换一个真实麦克风后重试。`
          ); */
        }
      }
      this.pendingAudio.push(encoded);
    };

    // AudioContext 有时会以 suspended 起步（那样 onaudioprocess 根本不会回调），主动 resume 一次
    if (this.audioContext.state === "suspended") {
      void this.audioContext.resume().catch(() => undefined);
    }
    window.setTimeout(() => {
      if (capturedChunks === 0 && this.processing) {
        /* console.warn(
          `[nls] ⚠️ 1.5s 内没有采集到任何音频（AudioContext=${this.audioContext?.state}，` +
            `stream.active=${this.stream?.active}）—— 麦克风可能被别的程序占用或未真正启动`
        ); */
      }
    }, 1500);
  }

  /** 取 token → 连网关 → 发 StartTranscription → 等 TranscriptionStarted */
  async openSocket(): Promise<void> {
    const { token } = await fetchNlsToken();
    // appkey 来自配置（见 getNlsAppkey）；构造参数只作兜底
    this.appkey = getNlsTokenAppkey() || this.appKey;
    this.taskId = createNlsId();

    await new Promise<void>((resolve, reject) => {
      this.startResolve = resolve;
      this.startReject = reject;
      /** 诊断用：中间结果事件计数（日志里每 4 条打一条，避免刷屏） */
      let midEvents = 0;

      const url = `${NLS_GATEWAY_URL}?token=${encodeURIComponent(token)}`;
      const socket = new WebSocket(url);
      this.socket = socket;

      const startTimer = window.setTimeout(() => {
        reject(new Error("语音识别连接超时（未收到 TranscriptionStarted）"));
        this.closeSocket();
      }, START_TIMEOUT_MS);

      const settle = () => window.clearTimeout(startTimer);

      socket.onopen = () => {
        socket.send(buildStartTranscription({ appkey: this.appkey, taskId: this.taskId }));
      };

      socket.onmessage = (event: MessageEvent<string>) => {
        const message = parseNlsMessage(event.data);
        const name = getNlsEventName(message);
        if (!name) return;

        // 诊断日志：把网关下行事件打出来（ResultChanged 很密，每 4 条打一条），
        // 排查「进录音态但不出字」时能区分「音频没上去」和「服务端没回/回了错」
        if (name === "TaskFailed") {
          /* console.warn(`[nls] ← TaskFailed：${describeNlsFailure(message)}`); */
        } else if (name !== "TranscriptionResultChanged" && name !== "SentenceBegin") {
          /* console.log(`[nls] ← ${name}${message?.payload?.result ? ` ${JSON.stringify(message.payload.result)}` : ""}`); */
        } else if (name === "TranscriptionResultChanged") {
          midEvents += 1;
          if (midEvents % 4 === 1) {
            /* console.log(`[nls] ← 中间结果：${JSON.stringify(message?.payload?.result ?? "")}`); */
          }
        }

        if (name === "TranscriptionStarted") {
          settle();
          this.startFlushTimer();
          this.startResolve?.();
          return;
        }
        if (name === "TaskFailed") {
          const error = new Error(describeNlsFailure(message));
          settle();
          this._eventbus.emit("error", { message: error.message });
          this.startReject?.(error);
          return;
        }

        const { state, event: payload } = reduceNlsTranscript(this.transcript, message);
        this.transcript = state;
        if (payload) this._eventbus.emit("transform", payload);

        if (name === "TranscriptionCompleted") {
          this.stopResolve?.();
        }
      };

      socket.onerror = () => {
        settle();
        const error = new Error("语音识别连接出错");
        this._eventbus.emit("error", { message: error.message });
        this.startReject?.(error);
      };

      socket.onclose = () => {
        settle();
        // 还没开始就断了 → 让 openSocket 的 await 失败，别静默卡住
        this.startReject?.(new Error("语音识别连接已关闭"));
        this.stopResolve?.();
      };
    });
  }

  /**
   * 结束一轮：停麦克风 → 把余下的音频发完 → 发 StopTranscription → 等 TranscriptionCompleted。
   * 返回**累计定稿文本**（与旧实现的 `{text, command}` 形态一致）。
   */
  async transform(): Promise<{ text: string; command: string }> {
    this.stopping = true;
    this.processing = false;
    this.stopSendTimer();

    if (this.stream) {
      this.stream.getTracks().forEach((track) => track.stop());
    }
    this.source?.disconnect();
    this.processor?.disconnect();

    // 余下的音频尽快发完：用户此刻正在等最后一段文字，按 100ms 一帧的实时节奏发
    // 会让"停止后还要等"的体感更差 —— 这里用 20ms 一帧（≈5× 实时）
    const frames = this.pendingAudio;
    this.pendingAudio = [];
    for (const frame of frames) {
      if (this.socket?.readyState !== WebSocket.OPEN) break;
      this.socket.send(frame);
      await new Promise((r) => window.setTimeout(r, 20));
    }

    const finalText = this.transcript.finalText;

    if (this.socket?.readyState === WebSocket.OPEN) {
      this.socket.send(buildStopTranscription({ appkey: this.appkey, taskId: this.taskId }));
      await new Promise<void>((resolve) => {
        this.stopResolve = resolve;
        const timer = window.setTimeout(() => {
          this.stopResolve = null;
          resolve();
        }, STOP_TIMEOUT_MS);
        const original = this.stopResolve;
        this.stopResolve = () => {
          window.clearTimeout(timer);
          original?.();
        };
      });
    }

    return { text: this.transcript.finalText || finalText, command: "end" };
  }

  /** 停采集、收音频、断连接（壳组件在 `transform()` 之后调用） */
  close(): void {
    this.processing = false;
    this.stopping = false;
    this.pendingAudio = [];
    this.stopSendTimer();
    if (this.stream) {
      this.stream.getTracks().forEach((track) => track.stop());
    }
    this.closeSocket();
  }

  /** 清干净（断开音频节点、清事件监听）——调用后实例作废，调用方置 null */
  clear(): void {
    this.source?.disconnect();
    this.processor?.disconnect();
    this.audioContext?.close?.().catch(() => {
      /* 关闭失败无所谓：上下文本来就随页面回收 */
    });
    this.audioContext = null;
    this.source = null;
    this.processor = null;
    this.stream = null;
    this.socket = null;
    this.pendingAudio = [];
    this.allClear();
  }

  private closeSocket(): void {
    const socket = this.socket;
    this.socket = null;
    if (socket && (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING)) {
      try {
        socket.close();
      } catch {
        /* ignore */
      }
    }
  }

  private stopSendTimer(): void {
    if (this.sendTimer !== null) {
      window.clearInterval(this.sendTimer);
      this.sendTimer = null;
    }
  }

  /**
   * 把缓存里的音频按帧推给网关：**平时按实时节奏，积压时最多 4× 追赶**
   * （连上网关前麦克风已在采集，只按 1× 排空那段积压永远排不掉，表现为"出字整体慢半拍"；
   *  常量与说明见 nls-protocol.ts）。
   */
  private startFlushTimer(): void {
    this.stopSendTimer();
    const bytesPerFrame = (NLS_SAMPLE_RATE * 2 * NLS_FRAME_MS) / 1000; // 3200 字节 = 100ms
    let carry = new Uint8Array(0);
    let sentFrames = 0;
    let lastLoggedFrames = 0;
    let stallWarnedAt = 0;
    const flushStartedAt = Date.now();

    this.sendTimer = window.setInterval(() => {
      if (this.socket?.readyState !== WebSocket.OPEN) return;

      // 1) 把采集到的块**全部**并进 carry。
      // ⚠️ 合并**不设上限**：一旦按「carry + 下一块 ≤ 两帧」限制，浏览器给的 buffer
      //    比两帧大时就永远合并不进来，结果**一帧音频都发不出去**（界面看着仍在正常录音）。
      if (this.pendingAudio.length) {
        const pending = this.pendingAudio;
        this.pendingAudio = [];
        const merged = new Uint8Array(carry.length + pending.reduce((n, c) => n + c.length, 0));
        merged.set(carry, 0);
        let offset = carry.length;
        for (const chunk of pending) {
          merged.set(chunk, offset);
          offset += chunk.length;
        }
        carry = merged;
      }

      // 2) 按帧发（有积压时一个 tick 最多发 NLS_MAX_FRAMES_PER_TICK 帧）
      let sentThisTick = 0;
      while (carry.length >= bytesPerFrame && sentThisTick < NLS_MAX_FRAMES_PER_TICK) {
        if (this.socket?.readyState !== WebSocket.OPEN) return;
        this.socket.send(carry.subarray(0, bytesPerFrame));
        carry = carry.subarray(bytesPerFrame);
        sentThisTick += 1;
        sentFrames += 1;
      }
      if (sentFrames - lastLoggedFrames >= 20) {
        lastLoggedFrames = sentFrames;
        /* console.log(`[nls] 已推音频 ${Math.round((sentFrames * bytesPerFrame) / 32)}ms（${sentFrames} 帧）`); */
      }

      // 3) 诊断：录着音却 >=2 秒一帧都没发出去 → 明确报出来，别"静默不出声"
      if (this.processing && sentFrames === 0 && Date.now() - flushStartedAt > 2000 && Date.now() - stallWarnedAt > 5000) {
        stallWarnedAt = Date.now();
        /* console.warn(
          `[nls] ⚠️ 录音中但音频一帧都没发出去（carry=${carry.length}B，缓存块=${this.pendingAudio.length}）` +
            ` —— 若持续如此，检查采集端是否真的在出数据（见上一条「采集开始」日志里的采样率）`
        ); */
      }
    }, NLS_FLUSH_TICK_MS);
  }
}