#!/usr/bin/env node import { spawn } from "node:child_process"; import { existsSync } from "node:fs"; import { mkdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, resolve } from "node:path"; import { sourceFingerprint } from "./source-fingerprint.mjs"; const url = process.env.BROWSER_SPEAK_URL ?? "http://127.0.0.1:5174/"; const chrome = process.env.CHROME_BIN ?? (existsSync("/opt/google/chrome/chrome") ? "/opt/google/chrome/chrome" : existsSync("/usr/bin/google-chrome") ? "/usr/bin/google-chrome" : "chromium"); const resultPath = resolve( process.env.BROWSER_SPEAK_CLIENT_SIDE_JSON ?? `${tmpdir()}/browser-speak-client-side-smoke.json`, ); const profileDir = resolve( process.env.BROWSER_SPEAK_CLIENT_SIDE_PROFILE_DIR ?? `${tmpdir()}/browser-speak-client-side-profile`, ); const protocolTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_TIMEOUT_MS ?? 60000); const pollTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_POLL_TIMEOUT_MS ?? 5000); const pageUnresponsiveTimeoutMs = Number(process.env.BROWSER_SPEAK_PAGE_UNRESPONSIVE_TIMEOUT_MS ?? 120000); const loadTimeoutMs = Number(process.env.BROWSER_SPEAK_LOAD_TIMEOUT_MS ?? 600000); const taskTimeoutMs = Number(process.env.BROWSER_SPEAK_TASK_TIMEOUT_MS ?? 180000); const voicePreloadTimeoutMs = Number( process.env.BROWSER_SPEAK_VOICE_PRELOAD_TIMEOUT_MS ?? Math.max(180000, taskTimeoutMs), ); const reuseProfile = process.env.BROWSER_SPEAK_CLIENT_SIDE_REUSE_PROFILE === "true"; const ttsWarmup = process.env.BROWSER_SPEAK_TTS_WARMUP === "true"; const headless = process.env.BROWSER_SPEAK_HEADLESS !== "false"; const tasks = parseList(process.env.BROWSER_SPEAK_CLIENT_SIDE_TASKS ?? "tts,identity,loopback"); const stack = { device: process.env.BROWSER_SPEAK_DEVICE ?? "wasm", llm: process.env.BROWSER_SPEAK_LLM ?? "HuggingFaceTB/SmolLM2-135M-Instruct", asr: process.env.BROWSER_SPEAK_ASR ?? "onnx-community/moonshine-base-ONNX", voice: process.env.BROWSER_SPEAK_VOICE ?? "F2", ttsSteps: Number(process.env.BROWSER_SPEAK_TTS_STEPS ?? 2), vadSilenceMs: Number(process.env.BROWSER_SPEAK_VAD_SILENCE_MS ?? 480), partialAsr: process.env.BROWSER_SPEAK_PARTIAL_ASR !== "false", ttsWarmup, }; async function main() { await ensureServer(); await mkdir(dirname(resultPath), { recursive: true }); if (!reuseProfile) await rm(profileDir, { recursive: true, force: true }); const browser = launchBrowser(9346, profileDir); const recorder = new NetworkRecorder(url); let client = null; try { client = await connectToPage(9346, browser); client.on("Network.requestWillBeSent", (event) => recorder.requestWillBeSent(event)); client.on("Network.responseReceived", (event) => recorder.responseReceived(event)); client.on("Network.loadingFailed", (event) => recorder.loadingFailed(event)); await client.call("Network.enable"); await waitForBenchApi(client); recorder.markStage("load"); await runPageTask(client, `window.browserSpeakBench.loadStack(${JSON.stringify(stack)})`, { label: "model load", timeoutMs: loadTimeoutMs, }); await runPageTask( client, `Promise.all(${JSON.stringify(["F1", "F2", "M1", "M2"])}.map((voice) => window.browserSpeakBench.preloadVoice({ voice, timeoutMs: ${JSON.stringify(voicePreloadTimeoutMs)} })))`, { label: "voice preload", timeoutMs: voicePreloadTimeoutMs }, ); await runPageTask(client, "window.browserSpeakBench.clearResults()", { label: "clear restored benchmark rows", timeoutMs: 30000, }); await sleep(1000); recorder.markStage("benchmark"); for (const task of tasks) { await runBenchmarkTask(client, task); } await sleep(1500); const snapshot = await runPageTask(client, "window.browserSpeakBench.exportResults()", { label: "export results", timeoutMs: 30000, }); const requests = recorder.requests(); const benchmarkRequests = requests.filter((request) => request.stage === "benchmark"); const suspects = benchmarkRequests.flatMap((request) => classifyRequest(request).map((reason) => ({ reason, request })), ); const benchmarkErrors = snapshot.results.filter((result) => result.error); const missingTasks = missingTaskRows(snapshot.results, tasks); const payload = { generatedAt: new Date().toISOString(), sourceFingerprint: await sourceFingerprint(), url, passed: suspects.length === 0 && benchmarkErrors.length === 0 && missingTasks.length === 0, config: { stack, tasks, loadTimeoutMs, taskTimeoutMs, voicePreloadTimeoutMs, protocolTimeoutMs, pollTimeoutMs, pageUnresponsiveTimeoutMs, reuseProfile, headless, chrome, extraChromeArgs: parseChromeArgs(), }, summary: { totalRequests: requests.length, loadRequests: requests.filter((request) => request.stage === "load").length, benchmarkRequests: benchmarkRequests.length, benchmarkHosts: summarizeHosts(benchmarkRequests), benchmarkMethods: summarizeMethods(benchmarkRequests), serverInferenceSuspects: suspects.length, benchmarkErrors: benchmarkErrors.length, missingTasks, }, suspects, benchmarkErrors, requests, browserExport: snapshot, browserExportMetadata: exportMetadata(snapshot), hostMetadata: snapshot.hostMetadata ?? null, benchmarkResults: snapshot.results, benchmarkSummary: snapshot.summary, }; await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`); console.log(`Wrote client-side smoke JSON: ${resultPath}`); console.log( `Benchmark network requests: ${benchmarkRequests.length}, server-inference suspects: ${suspects.length}.`, ); for (const suspect of suspects.slice(0, 8)) { console.log(`${suspect.reason}: ${suspect.request.method} ${suspect.request.url}`); } for (const errorRow of benchmarkErrors.slice(0, 8)) { console.log(`${errorRow.kind} benchmark error: ${errorRow.error}`); } for (const missingTask of missingTasks) { console.log(`Missing benchmark row for task: ${missingTask}`); } if (!payload.passed) process.exitCode = 1; await client.closeBrowser(); } catch (error) { await writeFailurePayload(error, recorder, browser).catch((writeError) => { console.error(`Could not write client-side smoke failure JSON: ${writeError.message}`); }); if (client) await client.closeBrowser().catch(() => {}); throw error; } finally { await stopBrowser(browser, reuseProfile ? null : profileDir); } } async function writeFailurePayload(error, recorder, browser = null) { const requests = recorder.requests(); const benchmarkRequests = requests.filter((request) => request.stage === "benchmark"); const suspects = benchmarkRequests.flatMap((request) => classifyRequest(request).map((reason) => ({ reason, request })), ); const payload = { generatedAt: new Date().toISOString(), sourceFingerprint: await sourceFingerprint(), url, passed: false, error: error.stack ?? error.message ?? String(error), browserLog: browser?.browserLog ?? "", config: { stack, tasks, loadTimeoutMs, taskTimeoutMs, voicePreloadTimeoutMs, protocolTimeoutMs, pollTimeoutMs, pageUnresponsiveTimeoutMs, reuseProfile, headless, chrome, extraChromeArgs: parseChromeArgs(), }, summary: { totalRequests: requests.length, loadRequests: requests.filter((request) => request.stage === "load").length, benchmarkRequests: benchmarkRequests.length, benchmarkHosts: summarizeHosts(benchmarkRequests), benchmarkMethods: summarizeMethods(benchmarkRequests), serverInferenceSuspects: suspects.length, benchmarkErrors: null, missingTasks: [], }, suspects, requests, }; await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`); console.log(`Wrote client-side smoke failure JSON: ${resultPath}`); } async function runBenchmarkTask(client, task) { const normalized = task.toLowerCase(); const loopbackOptions = { timeoutMs: 180000, ...(process.env.BROWSER_SPEAK_LOOPBACK_TEXT ? { text: process.env.BROWSER_SPEAK_LOOPBACK_TEXT } : {}), ...(Number.isFinite(Number(process.env.BROWSER_SPEAK_LOOPBACK_SPEED)) ? { speed: Number(process.env.BROWSER_SPEAK_LOOPBACK_SPEED) } : {}), }; const expression = { tts: "window.browserSpeakBench.runTts({ timeoutMs: 90000 })", identity: "window.browserSpeakBench.runIdentity({ timeoutMs: 180000 })", chat: "window.browserSpeakBench.runChat({ timeoutMs: 180000 })", loopback: `window.browserSpeakBench.runLoopback(${JSON.stringify(loopbackOptions)})`, barge: "window.browserSpeakBench.runBargeIn({ timeoutMs: 90000 })", "barge-in": "window.browserSpeakBench.runBargeIn({ timeoutMs: 90000 })", }[normalized]; if (!expression) throw new Error(`Unknown client-side smoke task: ${task}`); await runPageTask(client, expression, { label: `${normalized} benchmark`, timeoutMs: normalized === "loopback" ? Math.max(taskTimeoutMs, 240000) : taskTimeoutMs, }); } function exportMetadata(snapshot) { if (!snapshot || typeof snapshot !== "object") return null; return { schemaVersion: snapshot.schemaVersion ?? null, exportId: snapshot.exportId ?? null, generatedAt: snapshot.generatedAt ?? null, hostMetadata: snapshot.hostMetadata ?? null, runtime: snapshot.runtime ?? null, evidence: snapshot.evidence ?? null, evidenceGuide: snapshot.evidenceGuide ?? null, summary: snapshot.summary ?? null, resultCount: Array.isArray(snapshot.results) ? snapshot.results.length : null, }; } function missingTaskRows(results, requestedTasks) { const aliases = new Map([ ["barge", "barge-in"], ["barge-in", "barge-in"], ]); const kinds = new Set(results.map((result) => result.kind)); return requestedTasks .map((task) => task.toLowerCase()) .map((task) => aliases.get(task) ?? task) .filter((kind, index, requestedKinds) => requestedKinds.indexOf(kind) === index) .filter((kind) => !kinds.has(kind)); } function classifyRequest(request) { const reasons = []; const parsed = new URL(request.url); const host = parsed.hostname.toLowerCase(); const path = parsed.pathname.toLowerCase(); const method = request.method.toUpperCase(); if (!["GET", "HEAD"].includes(method)) reasons.push(`non-asset HTTP method ${method}`); if (isForbiddenHost(host)) reasons.push(`forbidden inference host ${host}`); if (isInferencePath(path)) reasons.push(`inference-like path ${parsed.pathname}`); if (!isAllowedHost(host)) reasons.push(`unexpected host ${host}`); return reasons; } function isAllowedHost(host) { if (["localhost", "127.0.0.1", "::1"].includes(host)) return true; if (host === "cdn.jsdelivr.net") return true; if (host === "huggingface.co" || host.endsWith(".huggingface.co")) return true; if (host === "hf.co" || host.endsWith(".hf.co")) return true; if (host === "xethub.hf.co" || host.endsWith(".xethub.hf.co")) return true; return false; } function isForbiddenHost(host) { return /(^|\.)api-inference\.huggingface\.co$/.test(host) || /(^|\.)router\.hf\.co$/.test(host); } function isInferencePath(path) { return /\/(chat\/completions|completions|generate|predictions|queue\/(join|data)|run\/predict|api\/predict|v1\/)/.test( path, ); } function summarizeHosts(requests) { return summarize(requests.map((request) => new URL(request.url).hostname.toLowerCase())); } function summarizeMethods(requests) { return summarize(requests.map((request) => request.method.toUpperCase())); } function summarize(values) { const counts = new Map(); for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1); return Object.fromEntries([...counts.entries()].sort((a, b) => a[0].localeCompare(b[0]))); } class NetworkRecorder { constructor(pageUrl) { this.pageOrigin = new URL(pageUrl).origin; this.stage = "startup"; this.byId = new Map(); } markStage(stage) { this.stage = stage; } requestWillBeSent(event) { const request = event.request; if (!request?.url || shouldIgnoreUrl(request.url)) return; this.byId.set(event.requestId, { stage: this.stage, requestId: event.requestId, url: request.url, method: request.method ?? "GET", resourceType: event.type ?? "", initiatorType: event.initiator?.type ?? "", hasPostData: Boolean(request.hasPostData), status: null, failed: false, failureText: "", }); } responseReceived(event) { const item = this.byId.get(event.requestId); if (!item) return; item.status = event.response?.status ?? null; item.mimeType = event.response?.mimeType ?? ""; item.fromDiskCache = Boolean(event.response?.fromDiskCache); item.fromServiceWorker = Boolean(event.response?.fromServiceWorker); } loadingFailed(event) { const item = this.byId.get(event.requestId); if (!item) return; item.failed = true; item.failureText = event.errorText ?? ""; } requests() { return [...this.byId.values()].sort((a, b) => a.requestId.localeCompare(b.requestId)); } } function shouldIgnoreUrl(rawUrl) { return /^(data|blob|devtools):/i.test(rawUrl); } async function ensureServer() { const response = await fetch(url).catch((error) => { throw new Error(`Could not reach ${url}: ${error.message}`); }); if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`); } async function waitForBenchApi(client) { const deadline = Date.now() + 15000; while (Date.now() < deadline) { try { if (await client.evaluate("Boolean(window.browserSpeakBench)")) return; } catch { // The target may still be navigating. } await sleep(100); } throw new Error("window.browserSpeakBench was not installed."); } async function runPageTask(client, expression, { label = "page task", timeoutMs = 30000 } = {}) { const taskId = `task_${Date.now()}_${Math.random().toString(16).slice(2)}`; await client.evaluate(`(() => { const taskId = ${JSON.stringify(taskId)}; window.__browserSpeakHarnessTasks ||= {}; window.__browserSpeakHarnessTasks[taskId] = { done: false, label: ${JSON.stringify(label)} }; Promise.resolve(${expression}) .then((value) => { window.__browserSpeakHarnessTasks[taskId] = { done: true, value }; }) .catch((error) => { window.__browserSpeakHarnessTasks[taskId] = { done: true, error: error?.stack || error?.message || String(error), }; }); return true; })()`); const deadline = Date.now() + timeoutMs; let lastEvents = []; let lastPollError = ""; let pollErrorSince = 0; while (Date.now() < deadline) { let task = null; try { task = await client.evaluate( `window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}] ?? null`, pollTimeoutMs, ); lastPollError = ""; pollErrorSince = 0; } catch (error) { const message = error.message ?? String(error); pollErrorSince ||= Date.now(); if (message !== lastPollError) { console.log(`${label}: waiting for page response (${message})`); lastPollError = message; } if (Date.now() - pollErrorSince > pageUnresponsiveTimeoutMs) { throw new Error( `${label} page stayed unresponsive for ${(pageUnresponsiveTimeoutMs / 1000).toFixed(0)} seconds; last poll error: ${message}`, ); } await sleep(500); continue; } if (task?.done) { await client .evaluate(`delete window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}]`, pollTimeoutMs) .catch(() => {}); if (task.error) throw new Error(`${label} failed: ${task.error}`); return task.value; } const snapshot = await client .evaluate(`(() => { const state = window.browserSpeakBench?.state?.(); return state ? { modelsLoaded: state.modelsLoaded, modelsLoading: state.modelsLoading, activeBenchmark: state.activeBenchmark?.kind ?? null, events: state.events?.slice(0, 3) ?? [], } : null; })()`, pollTimeoutMs) .catch(() => null); const events = snapshot?.events ?? []; if (events.join("\\n") !== lastEvents.join("\\n")) { lastEvents = events; if (events[0]) console.log(`${label}: ${events[0]}`); } await sleep(500); } throw new Error(`${label} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`); } function launchBrowser(port, profileDir) { const child = spawn( chrome, [ ...(headless ? ["--headless=new"] : []), "--no-sandbox", "--disable-gpu", "--disable-dev-shm-usage", "--disable-background-networking", "--disable-extensions", "--no-default-browser-check", "--no-first-run", "--autoplay-policy=no-user-gesture-required", `--remote-debugging-port=${port}`, `--user-data-dir=${profileDir}`, ...parseChromeArgs(), ], { stdio: ["ignore", "pipe", "pipe"] }, ); child.browserLog = ""; const appendLog = (chunk) => { child.browserLog = `${child.browserLog}${chunk}`; if (child.browserLog.length > 8000) child.browserLog = child.browserLog.slice(-8000); }; child.stdout.on("data", appendLog); child.stderr.on("data", appendLog); return child; } async function stopBrowser(child, profileDir) { if (child.exitCode == null) child.kill("SIGTERM"); await new Promise((resolve) => { child.once("exit", resolve); setTimeout(resolve, 3000); }); if (child.exitCode == null) child.kill("SIGKILL"); if (!profileDir) return; for (let attempt = 0; attempt < 5; attempt += 1) { try { await rm(profileDir, { recursive: true, force: true }); return; } catch (error) { if (attempt === 4) { console.warn(`Could not remove ${profileDir}: ${error.message}`); return; } await sleep(500); } } } async function connectToPage(port, child) { const deadline = Date.now() + 60000; let lastError = null; while (Date.now() < deadline) { if (child.exitCode != null) { throw new Error(`Chrome exited before DevTools became available.\n${child.browserLog}`); } try { const version = await fetch(`http://127.0.0.1:${port}/json/version`).then((response) => response.json()); if (version.webSocketDebuggerUrl) { const page = await createPageTarget(port); return new CdpClient(page.webSocketDebuggerUrl); } } catch (error) { lastError = error; } await sleep(250); } throw new Error( `Could not connect to Chrome DevTools on port ${port}: ${lastError?.message ?? "unknown error"}\n${child.browserLog}`, ); } async function createPageTarget(port) { for (const method of ["PUT", "GET"]) { const response = await fetch(`http://127.0.0.1:${port}/json/new?${encodeURIComponent(url)}`, { method, }).catch(() => null); if (response?.ok) { const target = await response.json(); if (target.webSocketDebuggerUrl) return target; } } const targets = await fetch(`http://127.0.0.1:${port}/json`).then((response) => response.json()); const page = targets.find((target) => target.type === "page" && target.url === url); if (page?.webSocketDebuggerUrl) return page; throw new Error("Could not create or find a page target."); } class CdpClient { constructor(webSocketUrl) { this.nextId = 1; this.pending = new Map(); this.listeners = new Map(); this.socket = new WebSocket(webSocketUrl); this.opened = new Promise((resolve, reject) => { this.socket.onopen = resolve; this.socket.onerror = reject; this.socket.onmessage = (event) => this.onMessage(event); }); } on(method, handler) { if (!this.listeners.has(method)) this.listeners.set(method, new Set()); this.listeners.get(method).add(handler); } onMessage(event) { const message = JSON.parse(event.data); if (message.id && this.pending.has(message.id)) { const { resolve: onResolve, reject } = this.pending.get(message.id); this.pending.delete(message.id); if (message.error) reject(new Error(message.error.message)); else onResolve(message.result); return; } if (message.method && this.listeners.has(message.method)) { for (const handler of this.listeners.get(message.method)) handler(message.params ?? {}); } } async call(method, params = {}, timeoutMs = protocolTimeoutMs) { await this.opened; const id = this.nextId++; this.socket.send(JSON.stringify({ id, method, params })); return new Promise((resolvePromise, reject) => { const timer = setTimeout(() => { this.pending.delete(id); reject(new Error(`${method} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`)); }, timeoutMs); this.pending.set(id, { resolve: (value) => { clearTimeout(timer); resolvePromise(value); }, reject: (error) => { clearTimeout(timer); reject(error); }, }); }); } async evaluate(expression, timeoutMs = protocolTimeoutMs) { const result = await this.call( "Runtime.evaluate", { expression, awaitPromise: true, returnByValue: true, }, timeoutMs, ); if (result.exceptionDetails) throw new Error(formatException(result.exceptionDetails)); return result.result.value; } async closeBrowser() { await this.call("Browser.close", {}, 5000).catch(() => {}); try { this.socket.close(); } catch { // Ignore close races. } } } function formatException(exceptionDetails) { const exception = exceptionDetails.exception; return exception?.description ?? exception?.value ?? exceptionDetails.text ?? "Evaluation failed."; } function parseList(value) { return String(value ?? "") .split(",") .map((item) => item.trim()) .filter(Boolean); } function parseChromeArgs() { const raw = process.env.BROWSER_SPEAK_CHROME_ARGS ?? ""; return ( raw .match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g) ?.map((arg) => arg.replace(/^["']|["']$/g, "")) ?? [] ); } function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } main().catch((error) => { console.error(error.stack ?? error.message); process.exitCode = 1; });