Added inference_scripts_vllm/unified_s2s_scripts/web/app.js

inference_scripts_vllm/unified_s2s_scripts/web/app.js

@@ -0,0 +1,458 @@
+import { PCMPlayerWorklet } from "./PCMPlayerWorklet.js";
+
+const els = {
+  modelSelect: document.getElementById("model-select"),
+  modelState: document.getElementById("model-state"),
+  autoE2E: document.getElementById("auto-e2e"),
+  enableReasoning: document.getElementById("enable-reasoning"),
+  audioFile: document.getElementById("audio-file"),
+  recordBtn: document.getElementById("record-btn"),
+  audioName: document.getElementById("audio-name"),
+  promptInput: document.getElementById("prompt-input"),
+  textInput: document.getElementById("text-input"),
+  transcribeBtn: document.getElementById("transcribe-btn"),
+  generateResponseBtn: document.getElementById("generate-response-btn"),
+  generateAudioBtn: document.getElementById("generate-audio-btn"),
+  clearBtn: document.getElementById("clear-btn"),
+  statusBox: document.getElementById("status-box"),
+  textOutput: document.getElementById("text-output"),
+  audioOutput: document.getElementById("audio-output"),
+  audioState: document.getElementById("audio-state"),
+  visualizer: document.getElementById("visualizer"),
+};
+
+let config = { sample_rate: 16000 };
+let selectedAudioFile = null;
+let recordedAudio = null;
+let mediaRecorder = null;
+let mediaStream = null;
+let audioContext = null;
+let player = null;
+let pendingAudioInfo = null;
+let currentAudioChunks = [];
+let completedAudioUrl = null;
+let activeSocket = null;
+let busy = false;
+
+function setStatus(message, isError = false) {
+  els.statusBox.value = message || "";
+  els.modelState.textContent = message || "Idle";
+  els.modelState.classList.toggle("error", isError);
+}
+
+function setReadyState(status) {
+  els.modelState.textContent = status.status || "Idle";
+  els.modelState.classList.toggle("ready", Boolean(status.ready));
+  els.modelState.classList.toggle("error", Boolean(status.error));
+  if (!busy) els.statusBox.value = status.status || "";
+}
+
+function setBusy(value) {
+  busy = value;
+  els.transcribeBtn.disabled = value;
+  els.generateResponseBtn.disabled = value;
+  els.generateAudioBtn.disabled = value;
+  els.recordBtn.disabled = value && mediaRecorder?.state !== "recording";
+}
+
+async function loadConfig() {
+  const response = await fetch("/api/config");
+  const data = await response.json();
+  config = data;
+  els.modelSelect.innerHTML = "";
+  for (const model of data.models) {
+    const option = document.createElement("option");
+    option.value = model;
+    option.textContent = model;
+    option.selected = model === data.default_model;
+    els.modelSelect.appendChild(option);
+  }
+  els.modelSelect.disabled = data.models.length <= 1;
+  els.promptInput.value = data.default_prompt || "";
+  setReadyState(data);
+}
+
+async function pollStatus() {
+  try {
+    const response = await fetch("/api/status");
+    setReadyState(await response.json());
+  } catch (error) {
+    setStatus(`Status check failed: ${error.message}`, true);
+  }
+}
+
+async function readNdjson(response, onEvent) {
+  if (!response.ok) throw new Error(await response.text());
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = "";
+  while (true) {
+    const { value, done } = await reader.read();
+    buffer += decoder.decode(value || new Uint8Array(), { stream: !done });
+    const lines = buffer.split("\n");
+    buffer = lines.pop() || "";
+    for (const line of lines) {
+      if (line.trim()) onEvent(JSON.parse(line));
+    }
+    if (done) break;
+  }
+  if (buffer.trim()) onEvent(JSON.parse(buffer));
+}
+
+function currentAudioFile() {
+  if (selectedAudioFile) return selectedAudioFile;
+  if (!recordedAudio) return null;
+  return new File([recordedAudio], "recording.webm", { type: recordedAudio.type || "audio/webm" });
+}
+
+async function transcribeAudio() {
+  const audio = currentAudioFile();
+  if (!audio) {
+    setStatus("Please record or upload audio first.", true);
+    return false;
+  }
+
+  setBusy(true);
+  const form = new FormData();
+  form.append("audio", audio);
+  form.append("model_name", els.modelSelect.value);
+  try {
+    const response = await fetch("/api/transcribe", { method: "POST", body: form });
+    let finalText = "";
+    await readNdjson(response, (event) => {
+      if (event.type === "status") setStatus(event.status);
+      if (event.type === "transcript") {
+        finalText = event.text || "";
+        els.textInput.value = finalText;
+      }
+      if (event.type === "complete") {
+        finalText = event.text || finalText;
+        els.textInput.value = finalText;
+        setStatus(event.status);
+      }
+      if (event.type === "error") throw new Error(event.error);
+    });
+    return Boolean(finalText);
+  } catch (error) {
+    setStatus(`Transcription failed: ${error.message}`, true);
+    return false;
+  } finally {
+    setBusy(false);
+  }
+}
+
+async function generateResponse() {
+  const text = els.textInput.value.trim();
+  if (!text) {
+    setStatus("Please provide text or transcribe audio first.", true);
+    return false;
+  }
+
+  setBusy(true);
+  els.textOutput.value = "";
+  try {
+    const response = await fetch("/api/generate-response", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        model_name: els.modelSelect.value,
+        enable_reasoning: els.enableReasoning.checked,
+        prompt: els.promptInput.value,
+        text,
+      }),
+    });
+    let finalText = "";
+    await readNdjson(response, (event) => {
+      if (event.type === "status") setStatus(event.status);
+      if (event.type === "text") {
+        finalText = event.text || "";
+        els.textOutput.value = finalText;
+        els.textOutput.scrollTop = els.textOutput.scrollHeight;
+      }
+      if (event.type === "complete") {
+        finalText = event.text || finalText;
+        els.textOutput.value = finalText;
+        setStatus(event.status);
+      }
+      if (event.type === "error") throw new Error(event.error);
+    });
+    return Boolean(finalText);
+  } catch (error) {
+    setStatus(`Response generation failed: ${error.message}`, true);
+    return false;
+  } finally {
+    setBusy(false);
+  }
+}
+
+async function ensurePlayer() {
+  if (!window.isSecureContext) {
+    throw new Error("AudioWorklet requires HTTPS or localhost.");
+  }
+  if (!audioContext) {
+    audioContext = new (window.AudioContext || window.webkitAudioContext)({
+      sampleRate: config.sample_rate || 16000,
+      latencyHint: "interactive",
+    });
+    if (!audioContext.audioWorklet) throw new Error("AudioWorklet is not supported in this browser.");
+    player = new PCMPlayerWorklet(audioContext, { minBufferBeforePlaybackMs: 250 });
+    await player.initPromise;
+    player.addEventListener("audioEnded", finalizeAudioPlayback);
+  }
+  if (audioContext.state === "suspended") await audioContext.resume();
+  return player;
+}
+
+async function generateAudio() {
+  const text = els.textOutput.value.trim();
+  if (!text) {
+    setStatus("Please generate text output first.", true);
+    return false;
+  }
+
+  try {
+    await ensurePlayer();
+  } catch (error) {
+    setStatus(`Audio init failed: ${error.message}`, true);
+    return false;
+  }
+
+  setBusy(true);
+  resetAudioOutput();
+  currentAudioChunks = [];
+  els.audioState.textContent = "Connecting audio stream...";
+
+  const wsScheme = window.location.protocol === "https:" ? "wss" : "ws";
+  activeSocket = new WebSocket(`${wsScheme}://${window.location.host}/ws/generate-audio`);
+  activeSocket.binaryType = "arraybuffer";
+
+  return new Promise((resolve) => {
+    let ok = false;
+    activeSocket.onopen = () => {
+      activeSocket.send(JSON.stringify({ model_name: els.modelSelect.value, text }));
+    };
+    activeSocket.onmessage = (event) => {
+      if (typeof event.data === "string") {
+        const message = JSON.parse(event.data);
+        if (message.type === "audio") pendingAudioInfo = message;
+        if (message.status) {
+          setStatus(message.status);
+          els.audioState.textContent = message.status;
+        }
+        if (message.type === "complete") {
+          ok = true;
+          player.notifyStreamEnded();
+          els.audioState.textContent = "Finishing playback...";
+        }
+        if (message.type === "error") {
+          setStatus(`Audio generation failed: ${message.error}`, true);
+          els.audioState.textContent = "Error";
+          activeSocket.close();
+        }
+        return;
+      }
+      handleAudioBytes(event.data);
+    };
+    activeSocket.onerror = () => {
+      setStatus("Audio stream failed.", true);
+      els.audioState.textContent = "Error";
+      setBusy(false);
+      resolve(false);
+    };
+    activeSocket.onclose = () => {
+      setBusy(false);
+      resolve(ok);
+    };
+  });
+}
+
+function handleAudioBytes(data) {
+  const sourceRate = pendingAudioInfo?.sample_rate || config.sample_rate || 16000;
+  const serverChunk = new Float32Array(data);
+  currentAudioChunks.push(new Float32Array(serverChunk));
+  const playbackChunk = resampleFloat32(serverChunk, sourceRate, audioContext.sampleRate);
+  player.playAudio(playbackChunk);
+  pendingAudioInfo = null;
+}
+
+function resetAudioOutput() {
+  player?.reset();
+  pendingAudioInfo = null;
+  if (activeSocket && activeSocket.readyState < WebSocket.CLOSING) activeSocket.close();
+  activeSocket = null;
+  if (completedAudioUrl) URL.revokeObjectURL(completedAudioUrl);
+  completedAudioUrl = null;
+  els.audioOutput.removeAttribute("src");
+  els.audioOutput.load();
+}
+
+function finalizeAudioPlayback() {
+  if (!currentAudioChunks.length) {
+    els.audioState.textContent = "Idle";
+    return;
+  }
+  const totalSamples = currentAudioChunks.reduce((sum, chunk) => sum + chunk.length, 0);
+  const combined = new Float32Array(totalSamples);
+  let offset = 0;
+  for (const chunk of currentAudioChunks) {
+    combined.set(chunk, offset);
+    offset += chunk.length;
+  }
+  completedAudioUrl = URL.createObjectURL(float32ToWavBlob(combined, config.sample_rate || 16000));
+  els.audioOutput.src = completedAudioUrl;
+  els.audioState.textContent = "Complete.";
+}
+
+function resampleFloat32(input, sourceRate, targetRate) {
+  if (!sourceRate || sourceRate === targetRate) return input;
+  const outputLength = Math.max(1, Math.round((input.length * targetRate) / sourceRate));
+  const output = new Float32Array(outputLength);
+  const ratio = sourceRate / targetRate;
+  for (let i = 0; i < outputLength; i++) {
+    const position = i * ratio;
+    const index = Math.floor(position);
+    const fraction = position - index;
+    const first = input[index] || 0;
+    const second = input[Math.min(index + 1, input.length - 1)] || 0;
+    output[i] = first + (second - first) * fraction;
+  }
+  return output;
+}
+
+function float32ToWavBlob(samples, sampleRate) {
+  const buffer = new ArrayBuffer(44 + samples.length * 2);
+  const view = new DataView(buffer);
+  writeAscii(view, 0, "RIFF");
+  view.setUint32(4, 36 + samples.length * 2, true);
+  writeAscii(view, 8, "WAVE");
+  writeAscii(view, 12, "fmt ");
+  view.setUint32(16, 16, true);
+  view.setUint16(20, 1, true);
+  view.setUint16(22, 1, true);
+  view.setUint32(24, sampleRate, true);
+  view.setUint32(28, sampleRate * 2, true);
+  view.setUint16(32, 2, true);
+  view.setUint16(34, 16, true);
+  writeAscii(view, 36, "data");
+  view.setUint32(40, samples.length * 2, true);
+  let offset = 44;
+  for (const sample of samples) {
+    const clamped = Math.max(-1, Math.min(1, sample));
+    view.setInt16(offset, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true);
+    offset += 2;
+  }
+  return new Blob([buffer], { type: "audio/wav" });
+}
+
+function writeAscii(view, offset, text) {
+  for (let i = 0; i < text.length; i++) view.setUint8(offset + i, text.charCodeAt(i));
+}
+
+async function runEndToEnd() {
+  if (!(await transcribeAudio())) return;
+  if (!(await generateResponse())) return;
+  await generateAudio();
+}
+
+async function startRecording() {
+  if (!navigator.mediaDevices?.getUserMedia) {
+    setStatus("Microphone recording is not supported in this browser.", true);
+    return;
+  }
+  selectedAudioFile = null;
+  recordedAudio = null;
+  els.audioFile.value = "";
+  els.audioName.textContent = "Recording...";
+  mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+  const chunks = [];
+  mediaRecorder = new MediaRecorder(mediaStream);
+  mediaRecorder.ondataavailable = (event) => {
+    if (event.data.size) chunks.push(event.data);
+  };
+  mediaRecorder.onstop = async () => {
+    recordedAudio = new Blob(chunks, { type: mediaRecorder.mimeType || "audio/webm" });
+    selectedAudioFile = null;
+    mediaStream.getTracks().forEach((track) => track.stop());
+    mediaStream = null;
+    els.recordBtn.textContent = "Record Audio";
+    els.audioName.textContent = `Recorded audio (${Math.round(recordedAudio.size / 1024)} KB).`;
+    if (els.autoE2E.checked) await runEndToEnd();
+  };
+  mediaRecorder.start();
+  els.recordBtn.textContent = "Stop Recording";
+}
+
+function stopRecording() {
+  if (mediaRecorder?.state === "recording") mediaRecorder.stop();
+}
+
+function clearAll() {
+  selectedAudioFile = null;
+  recordedAudio = null;
+  els.audioFile.value = "";
+  els.audioName.textContent = "No audio selected.";
+  els.textInput.value = "";
+  els.textOutput.value = "";
+  resetAudioOutput();
+  currentAudioChunks = [];
+  els.audioState.textContent = "Idle";
+  setStatus("Cleared.");
+}
+
+function drawVisualizer() {
+  requestAnimationFrame(drawVisualizer);
+  const canvas = els.visualizer;
+  const ctx = canvas.getContext("2d");
+  const width = canvas.width;
+  const height = canvas.height;
+  ctx.clearRect(0, 0, width, height);
+  ctx.fillStyle = "#101827";
+  ctx.fillRect(0, 0, width, height);
+
+  if (!player?.analyser) {
+    ctx.fillStyle = "#334155";
+    ctx.fillRect(0, height / 2 - 1, width, 2);
+    return;
+  }
+
+  const data = new Uint8Array(player.analyser.frequencyBinCount);
+  player.analyser.getByteFrequencyData(data);
+  const bars = 80;
+  const barWidth = width / bars;
+  ctx.fillStyle = "#7fffd4";
+  for (let i = 0; i < bars; i++) {
+    const value = data[Math.floor((i * data.length) / bars)] / 255;
+    const barHeight = Math.max(2, value * height * 0.8);
+    ctx.fillRect(i * barWidth + 1, height - barHeight, Math.max(1, barWidth - 2), barHeight);
+  }
+}
+
+function attachEvents() {
+  els.audioFile.addEventListener("change", async () => {
+    selectedAudioFile = els.audioFile.files[0] || null;
+    recordedAudio = null;
+    els.audioName.textContent = selectedAudioFile ? selectedAudioFile.name : "No audio selected.";
+    if (selectedAudioFile && els.autoE2E.checked) await runEndToEnd();
+  });
+  els.recordBtn.addEventListener("click", async () => {
+    if (mediaRecorder?.state === "recording") stopRecording();
+    else await startRecording();
+  });
+  els.transcribeBtn.addEventListener("click", transcribeAudio);
+  els.generateResponseBtn.addEventListener("click", generateResponse);
+  els.generateAudioBtn.addEventListener("click", generateAudio);
+  els.clearBtn.addEventListener("click", clearAll);
+}
+
+async function init() {
+  attachEvents();
+  drawVisualizer();
+  try {
+    await loadConfig();
+    window.setInterval(pollStatus, 2000);
+  } catch (error) {
+    setStatus(`Initialization failed: ${error.message}`, true);
+  }
+}
+
+init();