#!/usr/bin/env node import { readFile, writeFile, access } from "node:fs/promises"; import { constants } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, resolve } from "node:path"; import { mkdir } from "node:fs/promises"; import { sourceFingerprint } from "./source-fingerprint.mjs"; const paths = { clientSide: resolve(process.env.BROWSER_SPEAK_CLIENT_SIDE_JSON ?? `${tmpdir()}/browser-speak-client-side-smoke.json`), loopback: resolve(process.env.BROWSER_SPEAK_LOOPBACK_JSON ?? `${tmpdir()}/browser-speak-loopback-series.json`), fakeMic: resolve(process.env.BROWSER_SPEAK_FAKE_MIC_JSON ?? `${tmpdir()}/browser-speak-fake-mic-results.json`), realMic: resolve(process.env.BROWSER_SPEAK_REAL_MIC_JSON ?? `${tmpdir()}/browser-speak-real-mic-series.json`), webgpu: resolve(process.env.BROWSER_SPEAK_WEBGPU_JSON ?? `${tmpdir()}/browser-speak-webgpu-results.json`), ui: resolve(process.env.BROWSER_SPEAK_UI_JSON ?? `${tmpdir()}/browser-speak-ui-smoke.json`), evidenceExport: resolve( process.env.BROWSER_SPEAK_EVIDENCE_EXPORT_JSON ?? `${tmpdir()}/browser-speak-evidence-export-smoke.json`, ), }; const resultPath = resolve(process.env.BROWSER_SPEAK_AUDIT_JSON ?? `${tmpdir()}/browser-speak-validation-audit.json`); const soft = process.env.BROWSER_SPEAK_AUDIT_SOFT === "true"; const requireRealMic = process.env.BROWSER_SPEAK_AUDIT_REQUIRE_REAL_MIC !== "false"; const requireHardwareWebgpu = process.env.BROWSER_SPEAK_AUDIT_REQUIRE_HARDWARE_WEBGPU !== "false"; const realMicMaxWer = Number(process.env.BROWSER_SPEAK_AUDIT_REAL_MIC_MAX_WER ?? 0.25); const loopbackMaxMedianWer = Number(process.env.BROWSER_SPEAK_AUDIT_LOOPBACK_MAX_MEDIAN_WER ?? 0); let currentSourceFingerprint = null; async function main() { currentSourceFingerprint = await sourceFingerprint(); const artifacts = Object.fromEntries( await Promise.all(Object.entries(paths).map(async ([key, path]) => [key, await readJson(path)])), ); const checks = [ await checkStaticFiles(), checkUi(artifacts.ui), checkEvidenceExport(artifacts.evidenceExport), checkClientSide(artifacts.clientSide), checkLoopback(artifacts.loopback), checkFirstTtsChunkSafety(artifacts), checkFakeMic(artifacts.fakeMic), checkRealMic(artifacts.realMic), checkHardwareWebgpu(artifacts.webgpu), ]; const requiredChecks = checks.filter((check) => check.required); const passed = requiredChecks.every((check) => check.status === "pass"); const payload = { generatedAt: new Date().toISOString(), sourceFingerprint: currentSourceFingerprint, passed, paths, checks, nextActions: nextActions(checks), }; await mkdir(dirname(resultPath), { recursive: true }); await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`); console.log(`Wrote validation audit JSON: ${resultPath}`); for (const check of checks) { console.log(`${check.required ? "required" : "supporting"} ${check.status}: ${check.name} - ${check.message}`); } if (!passed && !soft) process.exitCode = 1; } async function checkStaticFiles() { const files = [ "index.html", "styles.css", "app.js", "workers/asr-worker.js", "workers/llm-worker.js", "workers/tts-worker.js", ]; const missing = []; for (const file of files) { if (!(await exists(file))) missing.push(file); } return { name: "demo files", required: true, status: missing.length === 0 ? "pass" : "fail", message: missing.length === 0 ? "Static demo files are present." : `Missing files: ${missing.join(", ")}.`, evidence: { files, missing }, }; } function checkUi(artifact) { if (!artifact.ok) return missing("UI smoke", true, artifact, "UI smoke JSON is missing or unreadable."); const freshness = sourceFreshness(artifact); if (!freshness.ok) return stale("UI smoke", true, artifact, freshness.message); const passed = artifact.value?.passed === true; return { name: "UI smoke", required: true, status: passed ? "pass" : "fail", message: passed ? "Desktop/mobile UI smoke passed." : "Desktop/mobile UI smoke has not passed.", evidence: pick(artifact.value, ["passed", "config", "errors"]), }; } function checkEvidenceExport(artifact) { if (!artifact.ok) return missing("evidence export smoke", true, artifact, "Evidence export smoke JSON is missing."); const freshness = sourceFreshness(artifact); if (!freshness.ok) return stale("evidence export smoke", true, artifact, freshness.message); const summary = artifact.value?.summary ?? {}; const passed = artifact.value?.passed === true && summary.restoredRows >= 1 && summary.exportedRows >= 1 && /^browser-speak-evidence-.*\.json$/.test(summary.downloadName ?? "") && summary.downloadHrefScheme === "blob" && summary.clearedRows === 0 && summary.persistedAfterClear === null; return { name: "evidence export smoke", required: true, status: passed ? "pass" : "fail", message: passed ? "Autosaved rows restore, evidence JSON download is requested, and Clear removes saved rows." : "Evidence export, autosave restore, or Clear behavior did not pass.", evidence: pick(artifact.value, ["passed", "summary", "error", "config"]), }; } function checkClientSide(artifact) { if (!artifact.ok) return missing("client-side/no-server smoke", true, artifact, "Client-side smoke JSON is missing."); const freshness = sourceFreshness(artifact); if (!freshness.ok) return stale("client-side/no-server smoke", true, artifact, freshness.message); const summary = artifact.value?.summary ?? {}; const passed = artifact.value?.passed === true && summary.benchmarkRequests === 0 && summary.serverInferenceSuspects === 0 && summary.benchmarkErrors === 0 && Array.isArray(summary.missingTasks) && summary.missingTasks.length === 0; return { name: "client-side/no-server smoke", required: true, status: passed ? "pass" : "fail", message: passed ? "Benchmark phase had no network requests, server-inference suspects, row errors, or missing tasks." : "Client-side smoke did not prove the no-server benchmark phase.", evidence: pick(artifact.value, ["passed", "summary", "error", "config"]), }; } function checkLoopback(artifact) { if (!artifact.ok) return missing("loopback stability", true, artifact, "Loopback series JSON is missing."); const freshness = sourceFreshness(artifact); if (!freshness.ok) return stale("loopback stability", true, artifact, freshness.message); const summary = artifact.value?.summary ?? {}; const passed = artifact.value?.passed === true && summary.completedRuns >= 3 && summary.errorRuns === 0 && summary.exactTranscriptRuns >= Math.ceil(summary.completedRuns / 2) && summary.identityPasses >= summary.exactTranscriptRuns && Number.isFinite(summary.medianWer) && summary.medianWer <= loopbackMaxMedianWer && Number.isFinite(summary.medianSpeechEndToFirstAudioMs); return { name: "loopback stability", required: true, status: passed ? "pass" : "fail", message: passed ? `Synthetic loopback completed 3 rows with median WER ${formatPercent( summary.medianWer, )} and ${summary.exactTranscriptRuns}/${summary.completedRuns} exact transcripts.` : "Loopback stability evidence is missing or below threshold.", evidence: pick(artifact.value, ["passed", "summary", "config", "error"]), }; } function checkFirstTtsChunkSafety(artifacts) { const rows = [ ...rowsFromArtifact(artifacts.clientSide, "client-side smoke"), ...rowsFromArtifact(artifacts.loopback, "loopback series"), ...rowsFromArtifact(artifacts.fakeMic, "fake microphone"), ...rowsFromArtifact(artifacts.realMic, "real microphone"), ].filter(({ row }) => !row.error && row.firstTtsText); if (rows.length === 0) { return { name: "first TTS chunk safety", required: true, status: "fail", message: "No completed benchmark rows with first TTS chunk text were available.", evidence: {}, }; } const unsafe = rows.filter(({ row }) => unsafeFirstTtsChunk(row)); return { name: "first TTS chunk safety", required: true, status: unsafe.length === 0 ? "pass" : "fail", message: unsafe.length === 0 ? `${rows.length} first TTS chunk(s) ended at a safe boundary.` : `${unsafe.length}/${rows.length} first TTS chunk(s) appear to split a word.`, evidence: { checkedRows: rows.length, unsafe: unsafe.slice(0, 8).map(({ source, row }) => ({ source, kind: row.kind, firstTtsText: row.firstTtsText, output: row.output, firstTtsBoundaryKind: row.firstTtsBoundaryKind ?? null, firstTtsWordBoundarySafe: row.firstTtsWordBoundarySafe ?? null, })), }, }; } function checkFakeMic(artifact) { if (!artifact.ok) return missing("fake microphone regression", false, artifact, "Fake mic JSON is missing."); const freshness = sourceFreshness(artifact); const summary = artifact.value?.summary?.current ?? artifact.value?.summary?.all ?? {}; const passed = freshness.ok && summary.micRuns >= 3 && summary.micMedianWer === 0 && Number.isFinite(summary.micMedianSpeechEndToFirstAudioMs); return { name: "fake microphone regression", required: false, status: passed ? "pass" : "warn", message: passed ? "Fake mic browser-capture regression has 3 rows and 0% median WER." : freshness.ok ? "Fake mic regression is unavailable or below the expected threshold." : freshness.message, evidence: { summary, freshness }, }; } function checkRealMic(artifact) { if (!requireRealMic) return skipped("real microphone validation", "Real mic requirement disabled by environment."); if (!artifact.ok) return missing("real microphone validation", true, artifact, "Real human microphone JSON is missing."); if (artifact.value?.dryRun) { return { name: "real microphone validation", required: true, status: "missing", message: "Only a real-mic dry-run artifact exists; no human speech rows were collected.", evidence: pick(artifact.value, ["dryRun", "config", "preflight"]), }; } const freshness = sourceFreshness(artifact); if (!freshness.ok) return stale("real microphone validation", true, artifact, freshness.message); const summary = artifact.value?.summary ?? {}; const config = artifact.value?.config ?? {}; const expected = artifact.value?.config?.count ?? 3; const extraChromeArgs = Array.isArray(config.extraChromeArgs) ? config.extraChromeArgs : []; const fakeCaptureArgs = extraChromeArgs.filter(isFakeCaptureArg); const fakeCaptureAllowed = config.allowFakeCapture === true; const passed = artifact.value?.passed === true && fakeCaptureArgs.length === 0 && !fakeCaptureAllowed && summary.completedRows >= expected && summary.errorRows === 0 && summary.identityPasses >= expected && Number.isFinite(summary.medianSpeechEndToFirstAudioMs) && Number.isFinite(summary.medianWer) && summary.medianWer <= realMicMaxWer; return { name: "real microphone validation", required: true, status: passed ? "pass" : "fail", message: realMicMessage(passed, fakeCaptureArgs, fakeCaptureAllowed), evidence: { ...pick(artifact.value, ["passed", "summary", "config", "error"]), fakeCaptureArgs }, }; } function checkHardwareWebgpu(artifact) { if (!requireHardwareWebgpu) return skipped("hardware WebGPU benchmark", "Hardware WebGPU requirement disabled by environment."); if (!artifact.ok) return missing("hardware WebGPU benchmark", true, artifact, "WebGPU benchmark JSON is missing."); const freshness = sourceFreshness(artifact); if (!freshness.ok) return stale("hardware WebGPU benchmark", true, artifact, freshness.message); const webgpu = artifact.value?.webgpu ?? {}; const candidates = artifact.value?.candidates ?? []; const completeCandidates = candidates.filter((candidate) => candidate.status === "complete"); const passed = artifact.value?.skipped !== true && webgpu.available === true && webgpu.softwareAdapter !== true && completeCandidates.length > 0; return { name: "hardware WebGPU benchmark", required: true, status: passed ? "pass" : "missing", message: passed ? "Hardware WebGPU benchmark completed at least one candidate." : artifact.value?.reason ?? "Hardware WebGPU benchmark has not completed on a real adapter.", evidence: pick(artifact.value, ["skipped", "reason", "webgpu", "candidates", "summary"]), }; } function skipped(name, message) { return { name, required: false, status: "skip", message, evidence: {} }; } function missing(name, required, artifact, message) { return { name, required, status: "missing", message, evidence: { path: artifact.path, error: artifact.error }, }; } function stale(name, required, artifact, message) { return { name, required, status: "stale", message, evidence: { path: artifact.path, artifactHash: artifact.value?.sourceFingerprint?.hash ?? null, currentHash: currentSourceFingerprint?.hash ?? null, }, }; } function nextActions(checks) { const missingNames = checks .filter((check) => check.required && check.status !== "pass") .map((check) => check.name); const actions = []; if (missingNames.includes("real microphone validation") || missingNames.includes("hardware WebGPU benchmark")) { actions.push( "Run node tools/run-hosted-evidence-capture.mjs from a desktop Chrome session with a real microphone and hardware WebGPU to collect hosted browser evidence.", ); } return checks .filter((check) => check.required && check.status !== "pass") .map((check) => { if (check.name === "real microphone validation") return "Run node tools/run-real-mic-series.mjs on a machine with a real microphone."; if (check.name === "hardware WebGPU benchmark") return "Run node tools/run-webgpu-benchmark.mjs in a Chrome session exposing a hardware WebGPU adapter."; if (check.name === "client-side/no-server smoke") return "Run node tools/run-client-side-smoke.mjs and inspect /tmp/browser-speak-client-side-smoke.json."; if (check.name === "evidence export smoke") return "Run node tools/run-evidence-export-smoke.mjs and inspect /tmp/browser-speak-evidence-export-smoke.json."; if (check.name === "loopback stability") return "Run node tools/run-loopback-series.mjs and inspect /tmp/browser-speak-loopback-series.json."; if (check.name === "first TTS chunk safety") return "Run the client-side smoke and inspect firstTtsText / firstTtsWordBoundarySafe in benchmark rows."; if (check.name === "UI smoke") return "Run node tools/run-ui-smoke.mjs."; return `Resolve failed check: ${check.name}.`; }) .reduce((unique, action) => { if (!unique.includes(action)) unique.push(action); return unique; }, actions); } async function readJson(path) { try { return { ok: true, path, value: JSON.parse(await readFile(path, "utf8")) }; } catch (error) { return { ok: false, path, error: error.message }; } } async function exists(path) { try { await access(path, constants.F_OK); return true; } catch { return false; } } function pick(value, keys) { if (!value || typeof value !== "object") return {}; return Object.fromEntries(keys.filter((key) => key in value).map((key) => [key, value[key]])); } function realMicMessage(passed, fakeCaptureArgs, fakeCaptureAllowed) { if (passed) return "Real human microphone rows meet count, WER, latency, and identity gates."; if (fakeCaptureArgs.length > 0) { return `Real microphone evidence used fake-capture Chrome args: ${fakeCaptureArgs.join(", ")}.`; } if (fakeCaptureAllowed) return "Real microphone evidence was collected with fake capture explicitly allowed."; return "Real human microphone rows are missing or below threshold."; } function sourceFreshness(artifact) { const artifactHash = artifact.value?.sourceFingerprint?.hash; if (!artifactHash) { return { ok: false, message: "Artifact is missing source fingerprint metadata; rerun the harness." }; } if (artifactHash !== currentSourceFingerprint?.hash) { return { ok: false, message: "Artifact source fingerprint is stale; rerun the harness." }; } return { ok: true, message: "Artifact source fingerprint matches current files." }; } function rowsFromArtifact(artifact, source) { if (!artifact.ok || !artifact.value) return []; const rows = []; for (const key of ["benchmarkResults", "rows", "results"]) { if (Array.isArray(artifact.value[key])) { rows.push(...artifact.value[key].map((row) => ({ source, row }))); } } return rows; } function unsafeFirstTtsChunk(row) { return row.firstTtsWordBoundarySafe === false || firstTtsLooksMidWord(row); } function firstTtsLooksMidWord(row) { const chunk = String(row.firstTtsText ?? "").trim(); const output = String(row.output ?? "").trim(); if (!chunk || !output.startsWith(chunk)) return false; const before = chunk.at(-1) ?? ""; const after = output[chunk.length] ?? ""; return isWordLikeChar(before) && isWordLikeChar(after); } function isFakeCaptureArg(arg) { return ( arg === "--use-fake-device-for-media-stream" || arg.startsWith("--use-file-for-fake-audio-capture=") || arg.startsWith("--use-file-for-fake-video-capture=") ); } function isWordLikeChar(char) { return /[A-Za-z0-9']/.test(char); } function formatPercent(value) { if (!Number.isFinite(value)) return "-"; return `${Math.round(value * 100)}%`; } main().catch((error) => { console.error(error.stack ?? error.message); process.exitCode = 1; });