#!/usr/bin/env node import { spawn } from "node:child_process"; import { existsSync } from "node:fs"; import { mkdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, resolve } from "node:path"; const url = process.env.BROWSER_SPEAK_URL ?? "http://127.0.0.1:5174/"; const chrome = process.env.CHROME_BIN ?? (existsSync("/opt/google/chrome/chrome") ? "/opt/google/chrome/chrome" : existsSync("/usr/bin/google-chrome") ? "/usr/bin/google-chrome" : "chromium"); const resultPath = resolve(process.env.BROWSER_SPEAK_TTS_SWEEP_JSON ?? `${tmpdir()}/browser-speak-tts-sweep.json`); const profileDir = resolve(process.env.BROWSER_SPEAK_TTS_SWEEP_PROFILE_DIR ?? `${tmpdir()}/browser-speak-tts-sweep-profile`); const protocolTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_TIMEOUT_MS ?? 60000); const pollTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_POLL_TIMEOUT_MS ?? 5000); const loadTimeoutMs = Number(process.env.BROWSER_SPEAK_LOAD_TIMEOUT_MS ?? 360000); const taskTimeoutMs = Number(process.env.BROWSER_SPEAK_TASK_TIMEOUT_MS ?? 120000); const headless = process.env.BROWSER_SPEAK_HEADLESS !== "false"; const dryRun = process.env.BROWSER_SPEAK_TTS_SWEEP_DRY_RUN === "true"; const voices = parseList(process.env.BROWSER_SPEAK_TTS_SWEEP_VOICES ?? "F1,F2,M1,M2"); const steps = parseList(process.env.BROWSER_SPEAK_TTS_SWEEP_STEPS ?? "2,4,8").map((value) => Number(value)); const stack = { device: process.env.BROWSER_SPEAK_DEVICE ?? "wasm", llm: process.env.BROWSER_SPEAK_LLM ?? "HuggingFaceTB/SmolLM2-135M-Instruct", asr: process.env.BROWSER_SPEAK_ASR ?? "onnx-community/moonshine-base-ONNX", voice: voices[0] ?? "F2", ttsSteps: steps[0] ?? 2, vadSilenceMs: Number(process.env.BROWSER_SPEAK_VAD_SILENCE_MS ?? 480), partialAsr: process.env.BROWSER_SPEAK_PARTIAL_ASR !== "false", }; const candidates = buildCandidates(); async function main() { if (dryRun) { await mkdir(dirname(resultPath), { recursive: true }); await writeJson({ generatedAt: new Date().toISOString(), dryRun: true, url, config: benchmarkConfig(), candidates: candidates.map((candidate) => ({ ...candidate, status: "planned" })), }); console.log(`Wrote TTS sweep dry-run JSON: ${resultPath}`); console.log(`Planned ${candidates.length} TTS candidate(s): ${candidates.map(candidateLabel).join("; ")}`); return; } await ensureServer(); await mkdir(dirname(resultPath), { recursive: true }); await rm(profileDir, { recursive: true, force: true }); const browser = launchBrowser(9345, profileDir); try { const client = await connectToPage(9345, browser); await waitForBenchApi(client); await runPageTask(client, `window.browserSpeakBench.loadStack(${JSON.stringify(stack)})`, { label: "model load", timeoutMs: loadTimeoutMs, }); await runPageTask(client, "window.browserSpeakBench.clearResults()", { label: "clear results" }); for (const candidate of candidates) { console.log(`Running TTS sweep candidate: ${candidateLabel(candidate)}`); try { await runPageTask(client, `window.browserSpeakBench.setRuntimeOptions(${JSON.stringify(candidate)})`, { label: `${candidateLabel(candidate)} options`, timeoutMs: 20000, }); await runPageTask( client, `window.browserSpeakBench.preloadVoice(${JSON.stringify({ voice: candidate.voice })})`, { label: `${candidateLabel(candidate)} voice preload`, timeoutMs: 45000, }, ); const before = await runPageTask(client, "window.browserSpeakBench.state()", { label: `${candidateLabel(candidate)} preflight`, }); const snapshot = await runPageTask(client, "window.browserSpeakBench.runTts({ timeoutMs: 90000 })", { label: `${candidateLabel(candidate)} tts`, timeoutMs: taskTimeoutMs, }); const added = snapshot.results.slice(0, Math.max(0, snapshot.results.length - before.results.length)); const row = added.find((result) => result.kind === "tts"); candidate.status = row && !row.error ? "complete" : "failed"; candidate.rowId = row?.id ?? null; candidate.error = row?.error || ""; candidate.metrics = row ? { firstAudioMs: row.firstAudioMs, synthesisMs: row.firstTtsSynthesisMs, audioEndMs: row.audioEndMs, firstTtsRoundTripMs: row.firstTtsRoundTripMs, firstTtsPlaybackDelayMs: row.firstTtsPlaybackDelayMs, modelLoadMs: row.stack?.modelLoadMs ?? null, } : null; } catch (error) { candidate.status = "failed"; candidate.error = error.message; console.error(`${candidateLabel(candidate)} failed: ${error.message}`); } finally { await writeJson(await exportPayload(client)); } } const payload = await exportPayload(client); await writeJson(payload); console.log(`Wrote TTS sweep benchmark JSON: ${resultPath}`); summarizeCandidates(candidates); client.closeBrowser(); } finally { await stopBrowser(browser, profileDir); } } async function exportPayload(client) { const exportResult = await runPageTask(client, "window.browserSpeakBench.exportResults()", { label: "export results", timeoutMs: 30000, }).catch(() => ({ summary: null, results: [] })); return { generatedAt: new Date().toISOString(), url, config: benchmarkConfig(), candidates, summary: exportResult.summary, results: exportResult.results, }; } function benchmarkConfig() { return { stack, voices, steps, candidateCount: candidates.length, loadTimeoutMs, taskTimeoutMs, protocolTimeoutMs, pollTimeoutMs, headless, chrome, profileDir, extraChromeArgs: parseChromeArgs(), }; } function buildCandidates() { const list = []; for (const voice of voices) { for (const ttsSteps of steps) { list.push({ voice, ttsSteps }); } } return list; } async function writeJson(payload) { await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`); } async function ensureServer() { const response = await fetch(url).catch((error) => { throw new Error(`Could not reach ${url}: ${error.message}`); }); if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`); } async function waitForBenchApi(client) { const deadline = Date.now() + 15000; while (Date.now() < deadline) { try { if (await client.evaluate("Boolean(window.browserSpeakBench?.setRuntimeOptions)")) return; } catch { // The target may still be navigating and can destroy the execution context. } await sleep(100); } throw new Error("window.browserSpeakBench.setRuntimeOptions was not installed."); } async function runPageTask(client, expression, { label = "page task", timeoutMs = 30000 } = {}) { const taskId = `task_${Date.now()}_${Math.random().toString(16).slice(2)}`; await client.evaluate(`(() => { const taskId = ${JSON.stringify(taskId)}; window.__browserSpeakHarnessTasks ||= {}; window.__browserSpeakHarnessTasks[taskId] = { done: false, label: ${JSON.stringify(label)} }; Promise.resolve(${expression}) .then((value) => { window.__browserSpeakHarnessTasks[taskId] = { done: true, value }; }) .catch((error) => { window.__browserSpeakHarnessTasks[taskId] = { done: true, error: error?.stack || error?.message || String(error), }; }); return true; })()`); const deadline = Date.now() + timeoutMs; let lastEvents = []; let lastPollError = ""; while (Date.now() < deadline) { let task = null; try { task = await client.evaluate( `window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}] ?? null`, pollTimeoutMs, ); lastPollError = ""; } catch (error) { const message = error.message ?? String(error); if (message !== lastPollError) { console.log(`${label}: waiting for page response (${message})`); lastPollError = message; } await sleep(500); continue; } if (task?.done) { await client .evaluate(`delete window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}]`, pollTimeoutMs) .catch(() => {}); if (task.error) throw new Error(`${label} failed: ${task.error}`); return task.value; } const snapshot = await client .evaluate(`(() => { const state = window.browserSpeakBench?.state?.(); return state ? { modelsLoaded: state.modelsLoaded, modelsLoading: state.modelsLoading, activeBenchmark: state.activeBenchmark?.kind ?? null, events: state.events?.slice(0, 3) ?? [], } : null; })()`, pollTimeoutMs) .catch(() => null); const events = snapshot?.events ?? []; if (events.join("\\n") !== lastEvents.join("\\n")) { lastEvents = events; if (events[0]) console.log(`${label}: ${events[0]}`); } await sleep(500); } throw new Error(`${label} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`); } function launchBrowser(port, profileDir) { const child = spawn( chrome, [ ...(headless ? ["--headless=new"] : []), "--no-sandbox", "--disable-gpu", "--disable-dev-shm-usage", "--disable-background-networking", "--disable-extensions", "--no-default-browser-check", "--no-first-run", "--autoplay-policy=no-user-gesture-required", `--remote-debugging-port=${port}`, `--user-data-dir=${profileDir}`, ...parseChromeArgs(), ], { stdio: ["ignore", "pipe", "pipe"] }, ); child.browserLog = ""; const appendLog = (chunk) => { child.browserLog = `${child.browserLog}${chunk}`; if (child.browserLog.length > 8000) child.browserLog = child.browserLog.slice(-8000); }; child.stdout.on("data", appendLog); child.stderr.on("data", appendLog); return child; } async function stopBrowser(child, profileDir) { if (child.exitCode == null) child.kill("SIGTERM"); await new Promise((resolve) => { child.once("exit", resolve); setTimeout(resolve, 3000); }); if (child.exitCode == null) child.kill("SIGKILL"); for (let attempt = 0; attempt < 5; attempt += 1) { try { await rm(profileDir, { recursive: true, force: true }); return; } catch (error) { if (attempt === 4) { console.warn(`Could not remove ${profileDir}: ${error.message}`); return; } await sleep(500); } } } async function connectToPage(port, child) { const deadline = Date.now() + 60000; let lastError = null; while (Date.now() < deadline) { if (child.exitCode != null) { throw new Error(`Chrome exited before DevTools became available.\n${child.browserLog}`); } try { const version = await fetch(`http://127.0.0.1:${port}/json/version`).then((response) => response.json()); if (version.webSocketDebuggerUrl) { const page = await createPageTarget(port); return new CdpClient(page.webSocketDebuggerUrl); } } catch (error) { lastError = error; } await sleep(250); } throw new Error( `Could not connect to Chrome DevTools on port ${port}: ${lastError?.message ?? "unknown error"}\n${child.browserLog}`, ); } async function createPageTarget(port) { for (const method of ["PUT", "GET"]) { const response = await fetch(`http://127.0.0.1:${port}/json/new?${encodeURIComponent(url)}`, { method, }).catch(() => null); if (response?.ok) { const target = await response.json(); if (target.webSocketDebuggerUrl) return target; } } const targets = await fetch(`http://127.0.0.1:${port}/json`).then((response) => response.json()); const page = targets.find((target) => target.type === "page" && target.url === url); if (page?.webSocketDebuggerUrl) return page; throw new Error("Could not create or find a page target."); } class CdpClient { constructor(webSocketUrl) { this.nextId = 1; this.pending = new Map(); this.socket = new WebSocket(webSocketUrl); this.opened = new Promise((resolve, reject) => { this.socket.onopen = resolve; this.socket.onerror = reject; this.socket.onmessage = (event) => this.onMessage(event); }); } onMessage(event) { const message = JSON.parse(event.data); if (!message.id || !this.pending.has(message.id)) return; const { resolve: onResolve, reject } = this.pending.get(message.id); this.pending.delete(message.id); if (message.error) reject(new Error(message.error.message)); else onResolve(message.result); } async call(method, params = {}, timeoutMs = protocolTimeoutMs) { await this.opened; const id = this.nextId++; this.socket.send(JSON.stringify({ id, method, params })); return new Promise((resolvePromise, reject) => { const timer = setTimeout(() => { this.pending.delete(id); reject(new Error(`${method} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`)); }, timeoutMs); this.pending.set(id, { resolve: (value) => { clearTimeout(timer); resolvePromise(value); }, reject: (error) => { clearTimeout(timer); reject(error); }, }); }); } async evaluate(expression, timeoutMs = protocolTimeoutMs) { const result = await this.call("Runtime.evaluate", { expression, returnByValue: true, }, timeoutMs); if (result.exceptionDetails) throw new Error(formatException(result.exceptionDetails)); return result.result.value; } closeBrowser() { try { this.socket.send(JSON.stringify({ id: this.nextId++, method: "Browser.close", params: {} })); } catch { // The surrounding process cleanup handles already-closed targets. } try { this.socket.close(); } catch { // Ignore close races. } } } function formatException(exceptionDetails) { const exception = exceptionDetails.exception; return exception?.description ?? exception?.value ?? exceptionDetails.text ?? "Evaluation failed."; } function parseList(value) { return String(value ?? "") .split(",") .map((item) => item.trim()) .filter(Boolean); } function parseChromeArgs() { const raw = process.env.BROWSER_SPEAK_CHROME_ARGS ?? ""; return ( raw .match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g) ?.map((arg) => arg.replace(/^["']|["']$/g, "")) ?? [] ); } function candidateLabel(candidate) { return `${candidate.voice} / ${candidate.ttsSteps} steps`; } function summarizeCandidates(items) { for (const candidate of items) { const metrics = candidate.metrics ?? {}; const suffix = Number.isFinite(metrics.firstAudioMs) ? `, first audio ${formatMs(metrics.firstAudioMs)}` : ""; const error = candidate.error ? ` (${candidate.error})` : ""; console.log(`${candidateLabel(candidate)}: ${candidate.status ?? "pending"}${suffix}${error}`); } } function formatMs(value) { if (!Number.isFinite(value)) return "-"; if (value < 1000) return `${Math.round(value)} ms`; return `${(value / 1000).toFixed(2)} s`; } function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } await main();