#!/usr/bin/env node import { readFile, writeFile, mkdir } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, resolve } from "node:path"; import { sourceFingerprint } from "./source-fingerprint.mjs"; const rawInputPath = process.argv[2] ?? process.env.BROWSER_SPEAK_BROWSER_EXPORT_JSON ?? ""; const inputPath = rawInputPath ? resolve(rawInputPath) : ""; const resultPath = resolve( process.env.BROWSER_SPEAK_BROWSER_AUDIT_JSON ?? `${tmpdir()}/browser-speak-browser-evidence-audit.json`, ); const requireRealMic = process.env.BROWSER_SPEAK_BROWSER_AUDIT_REQUIRE_REAL_MIC !== "false"; const requireHardwareWebgpu = process.env.BROWSER_SPEAK_BROWSER_AUDIT_REQUIRE_HARDWARE_WEBGPU !== "false"; const minRealMicRows = Math.max(1, Number(process.env.BROWSER_SPEAK_BROWSER_AUDIT_REAL_MIC_ROWS ?? 3)); const realMicMaxWer = Number(process.env.BROWSER_SPEAK_BROWSER_AUDIT_REAL_MIC_MAX_WER ?? 0.25); const expectedHostSuffix = process.env.BROWSER_SPEAK_BROWSER_AUDIT_HOST_SUFFIX ?? ".hf.space"; async function main() { if (!inputPath) { throw new Error( "Pass a downloaded benchmark JSON path, or set BROWSER_SPEAK_BROWSER_EXPORT_JSON.", ); } const exportPayload = JSON.parse(await readFile(inputPath, "utf8")); const browserEvidence = evidenceSummaryFromPayload(exportPayload); const checks = [ checkExportShape(exportPayload), checkSelfDescription(exportPayload), checkBrowserEvidenceSummary(exportPayload), checkHostedMetadata(exportPayload), checkRealMic(exportPayload), checkHardwareWebgpu(exportPayload), ]; const requiredChecks = checks.filter((check) => check.required); const passed = requiredChecks.every((check) => check.status === "pass"); const payload = { generatedAt: new Date().toISOString(), sourceFingerprint: await sourceFingerprint(), passed, inputPath, config: { requireRealMic, requireHardwareWebgpu, minRealMicRows, realMicMaxWer, expectedHostSuffix, }, browserEvidence, checks, limitations: [ "Downloaded browser benchmark JSON can prove the page-reported runtime, real-mic rows, and WebGPU adapter metadata.", "It cannot prove the absence of server inference by itself; pair it with tools/run-hosted-smoke.mjs or tools/run-client-side-smoke.mjs network evidence.", "It cannot prove Chrome was not launched with fake microphone capture flags; use tools/run-real-mic-series.mjs when that provenance matters.", ], }; await mkdir(dirname(resultPath), { recursive: true }); await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`); console.log(`Wrote browser evidence audit JSON: ${resultPath}`); for (const check of checks) { console.log(`${check.required ? "required" : "supporting"} ${check.status}: ${check.name} - ${check.message}`); } if (!passed) process.exitCode = 1; } function checkExportShape(payload) { const rows = rowsFromPayload(payload); const hostMetadata = hostMetadataFromPayload(payload); const passed = typeof generatedAtFromPayload(payload) === "string" && summaryFromPayload(payload) && typeof summaryFromPayload(payload) === "object" && Array.isArray(rows); return { name: "browser export shape", required: true, status: passed ? "pass" : "fail", message: passed ? `Export has ${rows.length} benchmark row(s).` : "Export is missing generatedAt, summary, or benchmark rows.", evidence: { generatedAt: generatedAtFromPayload(payload), schemaVersion: schemaVersionFromPayload(payload), exportId: exportIdFromPayload(payload), rowCount: rows.length, summaryScopes: summaryFromPayload(payload) ? Object.keys(summaryFromPayload(payload)) : [], hostMetadata, runtimePresent: Boolean(runtimeFromPayload(payload)), evidenceSummaryPresent: Boolean(evidenceSummaryFromPayload(payload)), }, }; } function checkSelfDescription(payload) { const schemaVersion = schemaVersionFromPayload(payload); const exportId = exportIdFromPayload(payload); const guide = evidenceGuideFromPayload(payload); const realMic = guide?.requirements?.realMic ?? {}; const hardwareWebgpu = guide?.requirements?.hardwareWebgpu ?? {}; const passed = typeof schemaVersion === "string" && schemaVersion.startsWith("browser-speak-benchmarks/") && typeof exportId === "string" && exportId.length > 12 && typeof guide?.auditCommand === "string" && guide.auditCommand.includes("audit-browser-evidence.mjs") && realMic.requiredRows >= minRealMicRows && hardwareWebgpu.requiredRows >= 1; return { name: "browser export self-description", required: true, status: passed ? "pass" : "fail", message: passed ? `Export ${exportId} declares ${schemaVersion} and embeds audit requirements.` : "Export is missing schemaVersion, exportId, or embedded audit requirements.", evidence: { schemaVersion, exportId, auditCommand: guide?.auditCommand ?? null, realMic, hardwareWebgpu, nextActions: guide?.nextActions ?? [], }, }; } function checkBrowserEvidenceSummary(payload) { const evidence = evidenceSummaryFromPayload(payload); if (!evidence) { return { name: "browser evidence summary", required: false, status: "missing", message: "Export does not include the page-generated evidence summary.", evidence: {}, }; } return { name: "browser evidence summary", required: false, status: "pass", message: summaryEvidenceMessage(evidence), evidence, }; } function checkHostedMetadata(payload) { const hostMetadata = hostMetadataFromPayload(payload); const evidence = evidenceSummaryFromPayload(payload)?.hosted ?? null; const host = hostMetadata.host ?? ""; const hosted = expectedHostSuffix ? host.endsWith(expectedHostSuffix) : Boolean(host); const commit = hostMetadata.hfSpaceCommit ?? ""; const passed = hosted && commit.length >= 7; return { name: "hosted Space metadata", required: true, status: passed ? "pass" : "fail", message: passed ? `Export came from ${host} at Space commit ${commit}.` : "Export is missing hosted Space host or commit metadata.", evidence: { hostMetadata, expectedHostSuffix, browserEvidence: evidence }, }; } function checkRealMic(payload) { if (!requireRealMic) return skipped("real microphone rows", "Real microphone requirement disabled."); const evidence = evidenceSummaryFromPayload(payload)?.realMic ?? null; const rows = completedRows(payload).filter((row) => row.kind === "mic"); const exactRows = rows.filter((row) => row.sttWer === 0).length; const identityPasses = rows.filter((row) => row.llmQualityPass === true).length; const inputRows = rows.filter(hasRealMicInputStats).length; const trackSettingsRows = rows.filter(hasInputTrackSettings).length; const medianWer = median(rows.map((row) => row.sttWer)); const medianSpeechEndToFirstAudioMs = median(rows.map((row) => row.speechEndToFirstAudioMs)); const passed = rows.length >= minRealMicRows && inputRows >= minRealMicRows && identityPasses >= minRealMicRows && Number.isFinite(medianWer) && medianWer <= realMicMaxWer && Number.isFinite(medianSpeechEndToFirstAudioMs); return { name: "real microphone rows", required: true, status: passed ? "pass" : "missing", message: passed ? `${rows.length} completed real-mic row(s), median WER ${formatPercent( medianWer, )}, speech end to first audio ${formatMs(medianSpeechEndToFirstAudioMs)}.` : `Need ${minRealMicRows} completed real-mic row(s) with mic input stats, identity pass, median WER <= ${formatPercent( realMicMaxWer, )}, and finite speech-end-to-audio latency.`, evidence: { rows: rows.length, exactRows, identityPasses, inputRows, trackSettingsRows, medianWer, medianSpeechEndToFirstAudioMs, rowSummaries: rows.slice(0, 8).map((row) => ({ id: row.id, sttWer: row.sttWer, transcript: row.transcript, llmQualityPass: row.llmQualityPass, speechEndToFirstAudioMs: row.speechEndToFirstAudioMs, micInputChunks: row.micInputChunks, micInputPeak: row.micInputPeak, inputSampleRate: row.inputSampleRate ?? row.stack?.environment?.inputSampleRate ?? null, inputTrackSettings: row.inputTrackSettings ?? row.stack?.environment?.inputTrackSettings ?? null, })), browserEvidence: evidence, }, }; } function checkHardwareWebgpu(payload) { if (!requireHardwareWebgpu) return skipped("hardware WebGPU rows", "Hardware WebGPU requirement disabled."); const evidence = evidenceSummaryFromPayload(payload)?.hardwareWebgpu ?? null; const rows = completedRows(payload).filter((row) => row.stack?.device === "webgpu"); const hardwareRows = rows.filter((row) => { const environment = row.stack?.environment ?? {}; return ( environment.webgpuAvailable === true && environment.webgpuSoftwareAdapter !== true && !isSoftwareWebGpuAdapter(environment.webgpuAdapter) ); }); const passed = hardwareRows.length > 0; return { name: "hardware WebGPU rows", required: true, status: passed ? "pass" : "missing", message: passed ? `${hardwareRows.length} completed row(s) used hardware WebGPU.` : "No completed benchmark rows prove hardware WebGPU execution.", evidence: { webgpuRows: rows.length, hardwareRows: hardwareRows.length, adapters: uniqueAdapters(rows), rowKinds: rows.map((row) => row.kind), browserEvidence: evidence, }, }; } function generatedAtFromPayload(payload) { return payload?.generatedAt ?? payload?.browserExport?.generatedAt ?? null; } function schemaVersionFromPayload(payload) { return payload?.schemaVersion ?? payload?.browserExport?.schemaVersion ?? null; } function exportIdFromPayload(payload) { return payload?.exportId ?? payload?.browserExport?.exportId ?? null; } function evidenceGuideFromPayload(payload) { return payload?.evidenceGuide ?? payload?.browserExport?.evidenceGuide ?? null; } function summaryFromPayload(payload) { return payload?.summary ?? payload?.browserExport?.summary ?? payload?.benchmarkSummary ?? null; } function hostMetadataFromPayload(payload) { return ( payload?.hostMetadata ?? payload?.browserExport?.hostMetadata ?? payload?.runtime?.hostMetadata ?? payload?.browserExport?.runtime?.hostMetadata ?? {} ); } function runtimeFromPayload(payload) { return payload?.runtime ?? payload?.browserExport?.runtime ?? null; } function evidenceSummaryFromPayload(payload) { return payload?.evidence ?? payload?.browserExport?.evidence ?? null; } function rowsFromPayload(payload) { if (Array.isArray(payload?.results)) return payload.results; if (Array.isArray(payload?.benchmarkResults)) return payload.benchmarkResults; if (Array.isArray(payload?.browserExport?.results)) return payload.browserExport.results; return []; } function completedRows(payload) { return rowsFromPayload(payload).filter((row) => !row.error); } function hasRealMicInputStats(row) { return ( Number.isFinite(row.micInputChunks) && row.micInputChunks > 0 && Number.isFinite(row.micInputPeak) && row.micInputPeak > 0 && Number.isFinite(row.inputSampleRate ?? row.stack?.environment?.inputSampleRate) ); } function hasInputTrackSettings(row) { const settings = row.inputTrackSettings ?? row.stack?.environment?.inputTrackSettings; return Boolean(settings && typeof settings === "object" && Object.keys(settings).length > 0); } function uniqueAdapters(rows) { const seen = new Set(); const adapters = []; for (const row of rows) { const adapter = row.stack?.environment?.webgpuAdapter ?? null; const label = formatAdapter(adapter); if (seen.has(label)) continue; seen.add(label); adapters.push({ label, softwareAdapter: row.stack?.environment?.webgpuSoftwareAdapter ?? null, adapter, }); } return adapters; } function isSoftwareWebGpuAdapter(adapter) { const text = [adapter?.vendor, adapter?.architecture, adapter?.device, adapter?.description] .filter(Boolean) .join(" ") .toLowerCase(); return /\b(swiftshader|llvmpipe|software rasterizer|software adapter|warp)\b/.test(text); } function formatAdapter(adapter) { if (!adapter) return "unknown adapter"; return ( [adapter.vendor, adapter.architecture, adapter.device, adapter.description] .filter(Boolean) .join(" / ") || "unknown adapter" ); } function skipped(name, message) { return { name, required: false, status: "skip", message, evidence: {} }; } function summaryEvidenceMessage(evidence) { const parts = []; if (evidence.realMic) { parts.push( `real mic ${evidence.realMic.rows ?? 0}/${evidence.realMic.requiredRows ?? minRealMicRows}`, ); } if (evidence.hardwareWebgpu) { parts.push(`hardware WebGPU rows ${evidence.hardwareWebgpu.hardwareRows ?? 0}`); } if (evidence.hosted) { parts.push(evidence.hosted.passed ? `hosted ${evidence.hosted.hfSpaceCommit ?? ""}` : "hosted missing"); } return parts.length > 0 ? `Browser summary: ${parts.join(", ")}.` : "Browser evidence summary is present."; } function median(values) { const finite = values.filter(Number.isFinite).sort((a, b) => a - b); if (finite.length === 0) return null; const middle = Math.floor(finite.length / 2); if (finite.length % 2 === 1) return finite[middle]; return (finite[middle - 1] + finite[middle]) / 2; } function formatMs(value) { if (!Number.isFinite(value)) return "-"; if (value < 1000) return `${Math.round(value)} ms`; return `${(value / 1000).toFixed(2)} s`; } function formatPercent(value) { if (!Number.isFinite(value)) return "-"; return `${Math.round(value * 100)}%`; } main().catch((error) => { console.error(error.stack ?? error.message); process.exitCode = 1; });