Spaces:
Running
Running
| import "./style.css"; | |
| import { HeadWobbler } from "./head-wobbler.js"; | |
| import { AntennasOscillator } from "./antennas.js"; | |
| import { StoryBridge, type BridgeAppState } from "./story-bridge.js"; | |
| import { loadNarrator, loadStory } from "story-engine"; | |
| import type { LoadedContent } from "story-engine"; | |
| import type { ReachyMiniInstance, RobotInfo } from "./globals.d.ts"; | |
| // βββ Content loading (Vite bundles content/ as static assets) ββββββββββββ | |
| const narratorJson = (await import("../content/narrator/narrator.json")).default; | |
| const piratesManifest = (await import("../content/story_pirates/manifest.json")).default; | |
| const piratesMap = (await import("../content/story_pirates/map.json")).default; | |
| const piratesFlags = (await import("../content/story_pirates/flags.json")).default; | |
| const piratesCharacters: Record<string, unknown> = {}; | |
| const characterModules = import.meta.glob("../content/story_pirates/characters/*.json", { eager: true }); | |
| for (const [path, mod] of Object.entries(characterModules)) { | |
| const id = path.split("/").pop()!.replace(/\.json$/, ""); | |
| piratesCharacters[id] = (mod as { default: unknown }).default; | |
| } | |
| const piratesScenes: Record<string, unknown> = {}; | |
| const sceneModules = import.meta.glob("../content/story_pirates/scenes/*.json", { eager: true }); | |
| for (const [path, mod] of Object.entries(sceneModules)) { | |
| const id = path.split("/").pop()!.replace(/\.json$/, ""); | |
| piratesScenes[id] = (mod as { default: unknown }).default; | |
| } | |
| const piratesStorylines: Record<string, unknown> = {}; | |
| const storylineModules = import.meta.glob("../content/story_pirates/storylines/chapter1/*.json", { eager: true }); | |
| for (const [path, mod] of Object.entries(storylineModules)) { | |
| const id = path.split("/").pop()!.replace(/\.json$/, ""); | |
| piratesStorylines[id] = (mod as { default: unknown }).default; | |
| } | |
| const content: LoadedContent = { | |
| narrator: loadNarrator(narratorJson), | |
| stories: { | |
| story_pirates: loadStory({ | |
| manifest: piratesManifest, | |
| map: piratesMap, | |
| flags: piratesFlags, | |
| characters: piratesCharacters, | |
| scenes: piratesScenes, | |
| storylines: piratesStorylines, | |
| }), | |
| }, | |
| }; | |
| // βββ Settings ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| const DEFAULT_REALTIME_MODEL = "gpt-realtime"; | |
| const DEFAULT_SUMMARY_MODEL = "gpt-4o-mini"; | |
| const DEFAULT_VOICE = "cedar"; | |
| const STORAGE_KEYS = { | |
| hfClientId: "reachyTales.hf.clientId", | |
| apiKey: "reachyTales.openai.apiKey", | |
| realtimeModel: "reachyTales.openai.realtimeModel", | |
| summaryModel: "reachyTales.openai.summaryModel", | |
| defaultVoice: "reachyTales.openai.defaultVoice", | |
| } as const; | |
| interface Settings { | |
| hfClientId: string; | |
| apiKey: string; | |
| realtimeModel: string; | |
| summaryModel: string; | |
| defaultVoice: string; | |
| } | |
| function loadSettings(): Settings { | |
| return { | |
| hfClientId: localStorage.getItem(STORAGE_KEYS.hfClientId) ?? "", | |
| apiKey: localStorage.getItem(STORAGE_KEYS.apiKey) ?? "", | |
| realtimeModel: localStorage.getItem(STORAGE_KEYS.realtimeModel) ?? DEFAULT_REALTIME_MODEL, | |
| summaryModel: localStorage.getItem(STORAGE_KEYS.summaryModel) ?? DEFAULT_SUMMARY_MODEL, | |
| defaultVoice: localStorage.getItem(STORAGE_KEYS.defaultVoice) ?? DEFAULT_VOICE, | |
| }; | |
| } | |
| function saveSettings(s: Settings): void { | |
| localStorage.setItem(STORAGE_KEYS.hfClientId, s.hfClientId); | |
| localStorage.setItem(STORAGE_KEYS.apiKey, s.apiKey); | |
| localStorage.setItem(STORAGE_KEYS.realtimeModel, s.realtimeModel); | |
| localStorage.setItem(STORAGE_KEYS.summaryModel, s.summaryModel); | |
| localStorage.setItem(STORAGE_KEYS.defaultVoice, s.defaultVoice); | |
| } | |
| // HF Spaces inject the OAuth client id at runtime via window.huggingface. | |
| // On localhost the user supplies one through the settings panel. | |
| function resolveHfClientId(saved: string): string | undefined { | |
| const injected = window.huggingface?.variables?.OAUTH_CLIENT_ID; | |
| if (injected) return injected; | |
| return saved || undefined; | |
| } | |
| // βββ State machine βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| type AppState = | |
| | "signed-out" | |
| | "authenticated" | |
| | "connecting" | |
| | "connected" | |
| | "auto-selecting" | |
| | "starting" | |
| | "listening" | |
| | "user-speaking" | |
| | "processing" | |
| | "ai-speaking" | |
| | "swapping-character" | |
| | "swapping-story" | |
| | "error"; | |
| interface StateView { | |
| caption: string; | |
| disabled: boolean; | |
| } | |
| const STATE_VIEWS: Record<AppState, StateView> = { | |
| "signed-out": { caption: "Sign in", disabled: false }, | |
| authenticated: { caption: "Tap to start", disabled: false }, | |
| connecting: { caption: "Connecting", disabled: true }, | |
| connected: { caption: "Choose a Reachy", disabled: true }, | |
| "auto-selecting": { caption: "Connecting", disabled: true }, | |
| starting: { caption: "Starting", disabled: true }, | |
| listening: { caption: "", disabled: false }, | |
| "user-speaking": { caption: "", disabled: false }, | |
| processing: { caption: "", disabled: false }, | |
| "ai-speaking": { caption: "", disabled: false }, | |
| "swapping-character": { caption: "Changing voice", disabled: true }, | |
| "swapping-story": { caption: "Loading story", disabled: true }, | |
| error: { caption: "Tap to retry", disabled: false }, | |
| }; | |
| const STATE_CLASS: Record<AppState, string> = { | |
| "signed-out": "state-signed-out", | |
| authenticated: "state-authenticated", | |
| connecting: "state-connecting", | |
| connected: "state-connected", | |
| "auto-selecting": "state-auto-selecting", | |
| starting: "state-starting", | |
| listening: "state-listening", | |
| "user-speaking": "state-user-speaking", | |
| processing: "state-processing", | |
| "ai-speaking": "state-ai-speaking", | |
| "swapping-character": "state-swapping-character", | |
| "swapping-story": "state-swapping-story", | |
| error: "state-error", | |
| }; | |
| const LIVE_STATES: ReadonlySet<AppState> = new Set([ | |
| "listening", | |
| "user-speaking", | |
| "processing", | |
| "ai-speaking", | |
| "starting", | |
| "swapping-character", | |
| "swapping-story", | |
| ]); | |
| // βββ DOM refs ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| const $ = <T extends HTMLElement>(selector: string): T => { | |
| const el = document.querySelector<T>(selector); | |
| if (!el) throw new Error(`Missing element: ${selector}`); | |
| return el; | |
| }; | |
| const circleBtn = $<HTMLButtonElement>("#main-circle"); | |
| const circleCaption = $<HTMLParagraphElement>("#circle-caption"); | |
| const orbWrap = $<HTMLElement>(".orb-wrap"); | |
| const micBtn = $<HTMLButtonElement>("#mic-btn"); | |
| const stopBtn = $<HTMLButtonElement>("#stop-btn"); | |
| const robotPicker = $<HTMLElement>("#robot-picker"); | |
| const robotList = $<HTMLElement>("#robot-list"); | |
| const hfUser = $<HTMLSpanElement>("#hf-user"); | |
| const hfAvatar = $<HTMLImageElement>("#hf-avatar"); | |
| const hfUserName = $<HTMLSpanElement>("#hf-user-name"); | |
| const settingsBtn = $<HTMLButtonElement>("#settings-btn"); | |
| const settingsModal = $<HTMLDialogElement>("#settings-modal"); | |
| const inputClientId = $<HTMLInputElement>("#hf-client-id"); | |
| const hfClientIdField = $<HTMLLabelElement>("#hf-client-id-field"); | |
| const inputApiKey = $<HTMLInputElement>("#openai-key"); | |
| const inputRealtimeModel = $<HTMLInputElement>("#openai-realtime-model"); | |
| const inputSummaryModel = $<HTMLInputElement>("#openai-summary-model"); | |
| const inputVoice = $<HTMLSelectElement>("#openai-voice"); | |
| const hfLogoutBtn = $<HTMLButtonElement>("#hf-logout"); | |
| const settingsForm = settingsModal.querySelector<HTMLFormElement>("form")!; | |
| const settingsTabs = Array.from(settingsModal.querySelectorAll<HTMLButtonElement>(".tab")); | |
| const settingsPanels = Array.from(settingsModal.querySelectorAll<HTMLElement>("[data-tab-panel]")); | |
| type SettingsTab = "access" | "story"; | |
| // βββ Runtime state ββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| let currentState: AppState = "signed-out"; | |
| let selectedRobotId: string | null = null; | |
| let settings: Settings = loadSettings(); | |
| let knownRobots: RobotInfo[] = []; | |
| let robot: ReachyMiniInstance | null = null; | |
| let bridge: StoryBridge | null = null; | |
| let openaiSink: HTMLAudioElement | null = null; | |
| let wobbler: HeadWobbler | null = null; | |
| let antennas: AntennasOscillator | null = null; | |
| let micLevel: MicLevelMonitor | null = null; | |
| let aiLevel: AiLevelMonitor | null = null; | |
| let micMuted = false; | |
| // βββ UI rendering βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| function setState(next: AppState): void { | |
| currentState = next; | |
| const view = STATE_VIEWS[next]; | |
| circleBtn.disabled = view.disabled; | |
| circleBtn.className = `circle ${STATE_CLASS[next]}`; | |
| if (next !== "error") setCaption(view.caption); | |
| const live = LIVE_STATES.has(next); | |
| orbWrap.classList.toggle("live", live); | |
| micBtn.setAttribute("aria-hidden", live ? "false" : "true"); | |
| stopBtn.setAttribute("aria-hidden", live ? "false" : "true"); | |
| micBtn.tabIndex = live ? 0 : -1; | |
| stopBtn.tabIndex = live ? 0 : -1; | |
| if (next !== "connected") showRobotPicker(false); | |
| } | |
| function setCaption(text: string, kind: "" | "error" | "muted" = ""): void { | |
| const trimmed = text.trim(); | |
| circleCaption.textContent = trimmed; | |
| circleCaption.className = `circle-caption${kind ? ` ${kind}` : ""}${trimmed ? "" : " empty"}`; | |
| } | |
| function renderRobotList(robots: RobotInfo[]): void { | |
| knownRobots = robots; | |
| robotList.innerHTML = ""; | |
| if (currentState !== "connected") return; | |
| if (!robots.length) { | |
| const empty = document.createElement("div"); | |
| empty.className = "robot-empty"; | |
| empty.textContent = "No Reachy online yet."; | |
| robotList.appendChild(empty); | |
| showRobotPicker(true); | |
| setCaption("Waiting for Reachy", "muted"); | |
| return; | |
| } | |
| if (robots.length === 1 && !selectedRobotId) { | |
| const only = robots[0]!; | |
| selectedRobotId = only.id; | |
| showRobotPicker(false); | |
| setState("auto-selecting"); | |
| window.setTimeout(() => { | |
| if (currentState === "auto-selecting") void doStart(); | |
| }, 500); | |
| return; | |
| } | |
| showRobotPicker(true); | |
| setCaption(STATE_VIEWS.connected.caption); | |
| for (const r of robots) { | |
| const card = document.createElement("div"); | |
| card.className = "robot-card"; | |
| if (r.id === selectedRobotId) card.classList.add("selected"); | |
| const name = document.createElement("div"); | |
| name.className = "name"; | |
| name.textContent = r.meta?.name || "Reachy Mini"; | |
| const idBadge = document.createElement("div"); | |
| idBadge.className = "id"; | |
| idBadge.textContent = r.id.slice(0, 12) + "β¦"; | |
| card.append(name, idBadge); | |
| card.addEventListener("click", () => { | |
| selectedRobotId = r.id; | |
| showRobotPicker(false); | |
| void doStart(); | |
| }); | |
| robotList.appendChild(card); | |
| } | |
| } | |
| function showRobotPicker(show: boolean): void { | |
| robotPicker.classList.toggle("hidden", !show); | |
| } | |
| // βββ Settings modal βββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| function openSettings(tab: SettingsTab = "access"): void { | |
| inputClientId.value = settings.hfClientId; | |
| inputApiKey.value = settings.apiKey; | |
| inputRealtimeModel.value = settings.realtimeModel; | |
| inputSummaryModel.value = settings.summaryModel; | |
| inputVoice.value = settings.defaultVoice; | |
| // Only show the HF client ID field on localhost (or when one is already | |
| // saved and might need editing). Deployed Spaces get the clientId | |
| // injected through window.huggingface so the field would be confusing. | |
| const needsClientIdField = | |
| !window.huggingface?.variables?.OAUTH_CLIENT_ID && | |
| (location.hostname === "localhost" || | |
| location.hostname === "127.0.0.1" || | |
| Boolean(settings.hfClientId)); | |
| hfClientIdField.classList.toggle("hidden", !needsClientIdField); | |
| setSettingsTab(tab); | |
| settingsModal.showModal(); | |
| } | |
| function setSettingsTab(tab: SettingsTab): void { | |
| for (const btn of settingsTabs) { | |
| const isActive = btn.dataset.tab === tab; | |
| btn.classList.toggle("active", isActive); | |
| btn.setAttribute("aria-selected", isActive ? "true" : "false"); | |
| } | |
| for (const panel of settingsPanels) { | |
| const isActive = panel.dataset.tabPanel === tab; | |
| panel.classList.toggle("active", isActive); | |
| panel.hidden = !isActive; | |
| } | |
| } | |
| for (const btn of settingsTabs) { | |
| btn.addEventListener("click", () => { | |
| const tab = btn.dataset.tab as SettingsTab | undefined; | |
| if (tab) setSettingsTab(tab); | |
| }); | |
| } | |
| settingsBtn.addEventListener("click", () => openSettings("access")); | |
| settingsForm.addEventListener("submit", (event) => { | |
| const submitter = (event as SubmitEvent).submitter as HTMLButtonElement | null; | |
| if (submitter?.value !== "save") return; | |
| const previousClientId = settings.hfClientId; | |
| settings = { | |
| hfClientId: inputClientId.value.trim(), | |
| apiKey: inputApiKey.value.trim(), | |
| realtimeModel: inputRealtimeModel.value.trim() || DEFAULT_REALTIME_MODEL, | |
| summaryModel: inputSummaryModel.value.trim() || DEFAULT_SUMMARY_MODEL, | |
| defaultVoice: inputVoice.value || DEFAULT_VOICE, | |
| }; | |
| saveSettings(settings); | |
| if (settings.hfClientId !== previousClientId) location.reload(); | |
| }); | |
| hfLogoutBtn.addEventListener("click", () => { | |
| if (!robot) return; | |
| robot.logout(); | |
| settingsModal.close(); | |
| location.reload(); | |
| }); | |
| // βββ Click handler for the central circle ββββββββββββββββββββββββββββββ | |
| circleBtn.addEventListener("click", async () => { | |
| try { | |
| switch (currentState) { | |
| case "signed-out": | |
| if (!robot) return; | |
| if (!resolveHfClientId(settings.hfClientId)) { | |
| setCaption("Add HF client ID in settings", "error"); | |
| openSettings(); | |
| return; | |
| } | |
| await robot.login(); | |
| return; | |
| case "authenticated": | |
| if (!settings.apiKey) { | |
| setCaption("Add OpenAI key in settings", "error"); | |
| openSettings(); | |
| return; | |
| } | |
| await doConnect(); | |
| return; | |
| case "error": | |
| selectedRobotId = null; | |
| if (robot?.isAuthenticated) setState("authenticated"); | |
| else setState("signed-out"); | |
| return; | |
| default: | |
| return; | |
| } | |
| } catch (err) { | |
| onFatalError(err); | |
| } | |
| }); | |
| // βββ Side controls (mic mute + stop) ββββββββββββββββββββββββββββββββββββ | |
| micBtn.addEventListener("click", () => { | |
| if (!robot) return; | |
| micMuted = !micMuted; | |
| robot.setMicMuted(micMuted); | |
| micBtn.classList.toggle("muted", micMuted); | |
| micBtn.setAttribute("aria-label", micMuted ? "Unmute" : "Mute"); | |
| micBtn.title = micMuted ? "Unmute" : "Mute"; | |
| }); | |
| stopBtn.addEventListener("click", async () => { | |
| await teardown(); | |
| selectedRobotId = null; | |
| micMuted = false; | |
| micBtn.classList.remove("muted"); | |
| if (!robot) { | |
| setState("signed-out"); | |
| } else if (robot.state !== "disconnected") { | |
| setState("connected"); | |
| renderRobotList(knownRobots); | |
| } else if (robot.isAuthenticated) { | |
| setState("authenticated"); | |
| } else { | |
| setState("signed-out"); | |
| } | |
| }); | |
| // βββ High-level flow steps ββββββββββββββββββββββββββββββββββββββββββββββ | |
| async function doConnect(): Promise<void> { | |
| if (!robot) return; | |
| setState("connecting"); | |
| try { | |
| if (robot.state === "disconnected") await robot.connect(); | |
| setState("connected"); | |
| renderRobotList(knownRobots); | |
| } catch (err) { | |
| onFatalError(err); | |
| } | |
| } | |
| async function doStart(): Promise<void> { | |
| if (!robot || !selectedRobotId) return; | |
| if (!settings.apiKey) { | |
| setCaption("Add OpenAI key in settings", "error"); | |
| openSettings(); | |
| return; | |
| } | |
| setState("starting"); | |
| try { | |
| await robot.startSession(selectedRobotId); | |
| } catch (err) { | |
| onFatalError(err); | |
| return; | |
| } | |
| const robotMicTrack = getRobotMicTrack(robot); | |
| if (!robotMicTrack) { | |
| onFatalError(new Error("Could not find the robot's microphone track")); | |
| return; | |
| } | |
| startMicLevelMonitor(robotMicTrack); | |
| startAntennas(); | |
| bridge = new StoryBridge({ | |
| content, | |
| robot, | |
| apiKey: settings.apiKey, | |
| realtimeModel: settings.realtimeModel, | |
| summaryModel: settings.summaryModel, | |
| defaultVoice: settings.defaultVoice, | |
| getRobotMicTrack: () => getRobotMicTrack(robot!), | |
| onOpenAiOutputTrack: (track) => { | |
| routeOpenaiToRobot(track); | |
| startWobbler(track); | |
| startAiLevelMonitor(track); | |
| }, | |
| onAppStateChange: (s: BridgeAppState) => { | |
| const mapped = s as AppState; | |
| // If the bridge says listening but we have audio that is still | |
| // playing out, defer the swap until the AI level monitor confirms | |
| // silence β same trick the reference uses to avoid snapping back | |
| // to bars mid-sentence. | |
| if (mapped === "listening" && currentState === "ai-speaking" && aiLevel) { | |
| aiLevel.waitForSilence(900, () => { | |
| if (currentState === "ai-speaking") setState("listening"); | |
| }); | |
| return; | |
| } | |
| if (mapped === "user-speaking") { | |
| aiLevel?.cancelSilenceWait(); | |
| wobbler?.reset(); | |
| antennas?.freeze(); | |
| } else if (mapped === "ai-speaking") { | |
| aiLevel?.cancelSilenceWait(); | |
| antennas?.resume(); | |
| } else if (mapped === "processing" || mapped === "listening") { | |
| antennas?.resume(); | |
| } | |
| setState(mapped); | |
| }, | |
| }); | |
| try { | |
| await bridge.start(); | |
| } catch (err) { | |
| onFatalError(err); | |
| return; | |
| } | |
| robot.setMicMuted(false); | |
| } | |
| function getRobotMicTrack(robotInstance: ReachyMiniInstance): MediaStreamTrack | null { | |
| const pc = robotInstance._pc; | |
| if (!pc) return null; | |
| for (const receiver of pc.getReceivers()) { | |
| if (receiver.track && receiver.track.kind === "audio") return receiver.track; | |
| } | |
| return null; | |
| } | |
| function routeOpenaiToRobot(track: MediaStreamTrack): void { | |
| if (!robot) return; | |
| const pc = robot._pc; | |
| if (!pc) return; | |
| const audioSender = pc.getSenders().find((s) => s.track && s.track.kind === "audio"); | |
| if (audioSender) { | |
| audioSender.replaceTrack(track).catch((err) => { | |
| console.error("[main] replaceTrack failed", err); | |
| }); | |
| } else { | |
| console.warn("[main] no audio sender on the robot peer β bidirectional audio missing"); | |
| } | |
| // Some browsers don't decode an inbound track unless it has a local sink. | |
| if (!openaiSink) { | |
| openaiSink = document.createElement("audio"); | |
| openaiSink.autoplay = true; | |
| openaiSink.muted = true; | |
| document.body.appendChild(openaiSink); | |
| } | |
| openaiSink.srcObject = new MediaStream([track]); | |
| } | |
| // βββ Head motion agent ββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| function startWobbler(assistantTrack: MediaStreamTrack): void { | |
| if (!robot) return; | |
| wobbler?.stop(); | |
| wobbler = new HeadWobbler({ | |
| track: assistantTrack, | |
| onOffsets: ({ roll, pitch, yaw }) => { | |
| robot?.setHeadPose(roll, pitch, yaw); | |
| }, | |
| }); | |
| wobbler.start(); | |
| } | |
| function stopWobbler(): void { | |
| wobbler?.stop(); | |
| wobbler = null; | |
| robot?.setHeadPose(0, 0, 0); | |
| } | |
| // βββ Antennas oscillator ββββββββββββββββββββββββββββββββββββββββββββββββ | |
| function startAntennas(): void { | |
| if (!robot) return; | |
| antennas?.stop(); | |
| antennas = new AntennasOscillator({ | |
| onAntennas: (right, left) => { | |
| robot?.setAntennas(right, left); | |
| }, | |
| }); | |
| antennas.start(); | |
| } | |
| function stopAntennas(): void { | |
| antennas?.stop(); | |
| antennas = null; | |
| robot?.setAntennas(0, 0); | |
| } | |
| // βββ Mic-level monitor (drives --audio-level + per-band bars) βββββββββββ | |
| class MicLevelMonitor { | |
| private ctx: AudioContext | null = null; | |
| private analyser: AnalyserNode | null = null; | |
| private source: MediaStreamAudioSourceNode | null = null; | |
| private raf = 0; | |
| private timeBuf: Float32Array<ArrayBuffer> | null = null; | |
| private freqBuf: Uint8Array<ArrayBuffer> | null = null; | |
| private level = 0; | |
| private bands = [0, 0, 0, 0, 0]; | |
| private static readonly BAND_EDGES = [4, 8, 16, 32, 64, 128]; | |
| private static readonly LOG1P_10 = Math.log1p(10); | |
| private static compress(v: number): number { | |
| return Math.log1p(v * 10) / MicLevelMonitor.LOG1P_10; | |
| } | |
| start(track: MediaStreamTrack): void { | |
| this.stop(); | |
| const ctx = new AudioContext(); | |
| const src = ctx.createMediaStreamSource(new MediaStream([track])); | |
| const analyser = ctx.createAnalyser(); | |
| analyser.fftSize = 1024; | |
| analyser.smoothingTimeConstant = 0.75; | |
| src.connect(analyser); | |
| this.ctx = ctx; | |
| this.source = src; | |
| this.analyser = analyser; | |
| this.timeBuf = new Float32Array(new ArrayBuffer(analyser.fftSize * 4)); | |
| this.freqBuf = new Uint8Array(new ArrayBuffer(analyser.frequencyBinCount)); | |
| const rootStyle = document.documentElement.style; | |
| const tick = () => { | |
| const an = this.analyser; | |
| const tbuf = this.timeBuf; | |
| const fbuf = this.freqBuf; | |
| if (!an || !tbuf || !fbuf) return; | |
| an.getFloatTimeDomainData(tbuf); | |
| let sum = 0; | |
| for (let i = 0; i < tbuf.length; i++) sum += tbuf[i]! * tbuf[i]!; | |
| const rms = Math.sqrt(sum / tbuf.length); | |
| const boosted = Math.min(1, Math.pow(rms * 6, 0.7)); | |
| const levelAttack = boosted > this.level ? 0.55 : 0.12; | |
| this.level += (boosted - this.level) * levelAttack; | |
| rootStyle.setProperty("--audio-level", this.level.toFixed(3)); | |
| an.getByteFrequencyData(fbuf); | |
| const edges = MicLevelMonitor.BAND_EDGES; | |
| for (let b = 0; b < 5; b++) { | |
| const lo = edges[b]!; | |
| const hi = edges[b + 1]!; | |
| let bandSum = 0; | |
| for (let j = lo; j < hi; j++) bandSum += fbuf[j]!; | |
| const raw = MicLevelMonitor.compress(bandSum / (hi - lo) / 255); | |
| const bandAttack = raw > this.bands[b]! ? 0.35 : 0.12; | |
| this.bands[b] = this.bands[b]! + (raw - this.bands[b]!) * bandAttack; | |
| rootStyle.setProperty(`--bar${b}`, Math.min(1, this.bands[b]!).toFixed(3)); | |
| } | |
| this.raf = requestAnimationFrame(tick); | |
| }; | |
| this.raf = requestAnimationFrame(tick); | |
| } | |
| stop(): void { | |
| cancelAnimationFrame(this.raf); | |
| this.raf = 0; | |
| try { | |
| this.source?.disconnect(); | |
| this.analyser?.disconnect(); | |
| this.ctx?.close(); | |
| } catch { | |
| // ignored | |
| } | |
| this.ctx = null; | |
| this.source = null; | |
| this.analyser = null; | |
| this.timeBuf = null; | |
| this.freqBuf = null; | |
| this.level = 0; | |
| this.bands = [0, 0, 0, 0, 0]; | |
| const rootStyle = document.documentElement.style; | |
| rootStyle.setProperty("--audio-level", "0"); | |
| for (let b = 0; b < 5; b++) rootStyle.setProperty(`--bar${b}`, "0"); | |
| } | |
| } | |
| function startMicLevelMonitor(track: MediaStreamTrack): void { | |
| micLevel ??= new MicLevelMonitor(); | |
| micLevel.start(track); | |
| } | |
| function stopMicLevelMonitor(): void { | |
| micLevel?.stop(); | |
| } | |
| // βββ AI-level monitor (drives --ai-audio-level + silence detector) ββββββ | |
| class AiLevelMonitor { | |
| private ctx: AudioContext | null = null; | |
| private analyser: AnalyserNode | null = null; | |
| private source: MediaStreamAudioSourceNode | null = null; | |
| private raf = 0; | |
| private timeBuf: Float32Array<ArrayBuffer> | null = null; | |
| private level = 0; | |
| private lastActiveTs = 0; | |
| private silenceWait: { quietMs: number; cb: () => void; maxWaitTimer: number | null } | null = null; | |
| private static readonly SILENCE_THRESHOLD = 0.006; | |
| start(track: MediaStreamTrack): void { | |
| this.stop(); | |
| const ctx = new AudioContext(); | |
| const src = ctx.createMediaStreamSource(new MediaStream([track])); | |
| const analyser = ctx.createAnalyser(); | |
| analyser.fftSize = 1024; | |
| analyser.smoothingTimeConstant = 0.75; | |
| src.connect(analyser); | |
| this.ctx = ctx; | |
| this.source = src; | |
| this.analyser = analyser; | |
| this.timeBuf = new Float32Array(new ArrayBuffer(analyser.fftSize * 4)); | |
| this.lastActiveTs = performance.now(); | |
| const rootStyle = document.documentElement.style; | |
| const tick = () => { | |
| const an = this.analyser; | |
| const buf = this.timeBuf; | |
| if (!an || !buf) return; | |
| an.getFloatTimeDomainData(buf); | |
| let sum = 0; | |
| for (let i = 0; i < buf.length; i++) sum += buf[i]! * buf[i]!; | |
| const rms = Math.sqrt(sum / buf.length); | |
| const boosted = Math.min(1, Math.pow(rms * 6, 0.7)); | |
| const levelAttack = boosted > this.level ? 0.55 : 0.12; | |
| this.level += (boosted - this.level) * levelAttack; | |
| rootStyle.setProperty("--ai-audio-level", this.level.toFixed(3)); | |
| const now = performance.now(); | |
| if (rms > AiLevelMonitor.SILENCE_THRESHOLD) { | |
| this.lastActiveTs = now; | |
| } else if (this.silenceWait) { | |
| const quietFor = now - this.lastActiveTs; | |
| if (quietFor >= this.silenceWait.quietMs) { | |
| const { cb, maxWaitTimer } = this.silenceWait; | |
| this.silenceWait = null; | |
| if (maxWaitTimer !== null) clearTimeout(maxWaitTimer); | |
| try { cb(); } catch (err) { console.warn("[ai-level] silence callback threw:", err); } | |
| } | |
| } | |
| this.raf = requestAnimationFrame(tick); | |
| }; | |
| this.raf = requestAnimationFrame(tick); | |
| } | |
| stop(): void { | |
| cancelAnimationFrame(this.raf); | |
| this.raf = 0; | |
| this.cancelSilenceWait(); | |
| try { | |
| this.source?.disconnect(); | |
| this.analyser?.disconnect(); | |
| this.ctx?.close(); | |
| } catch { | |
| // ignored | |
| } | |
| this.ctx = null; | |
| this.source = null; | |
| this.analyser = null; | |
| this.timeBuf = null; | |
| this.level = 0; | |
| document.documentElement.style.setProperty("--ai-audio-level", "0"); | |
| } | |
| waitForSilence(quietMs: number, cb: () => void, maxWaitMs = 8000): void { | |
| this.cancelSilenceWait(); | |
| const maxWaitTimer = window.setTimeout(() => { | |
| if (this.silenceWait?.cb === cb) { | |
| this.silenceWait = null; | |
| try { cb(); } catch (err) { console.warn("[ai-level] max-wait callback threw:", err); } | |
| } | |
| }, maxWaitMs); | |
| this.silenceWait = { quietMs, cb, maxWaitTimer }; | |
| } | |
| cancelSilenceWait(): void { | |
| if (!this.silenceWait) return; | |
| if (this.silenceWait.maxWaitTimer !== null) clearTimeout(this.silenceWait.maxWaitTimer); | |
| this.silenceWait = null; | |
| } | |
| } | |
| function startAiLevelMonitor(track: MediaStreamTrack): void { | |
| aiLevel ??= new AiLevelMonitor(); | |
| aiLevel.start(track); | |
| } | |
| function stopAiLevelMonitor(): void { | |
| aiLevel?.stop(); | |
| } | |
| // βββ Teardown / errors ββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| async function teardown(): Promise<void> { | |
| try { | |
| await bridge?.stop(); | |
| } catch { | |
| // ignored | |
| } | |
| bridge = null; | |
| stopWobbler(); | |
| stopAntennas(); | |
| stopMicLevelMonitor(); | |
| stopAiLevelMonitor(); | |
| if (openaiSink) { | |
| openaiSink.srcObject = null; | |
| openaiSink.remove(); | |
| openaiSink = null; | |
| } | |
| try { | |
| await robot?.stopSession(); | |
| } catch { | |
| // ignored | |
| } | |
| } | |
| async function onFatalError(err: unknown): Promise<void> { | |
| const message = err instanceof Error ? err.message : String(err); | |
| console.error("[main] error:", err); | |
| setState("error"); | |
| circleCaption.title = message; | |
| await teardown(); | |
| } | |
| // βββ Robot event wiring βββββββββββββββββββββββββββββββββββββββββββββββββ | |
| function wireRobot(): void { | |
| if (!robot) return; | |
| robot.addEventListener("robotsChanged", (event) => { | |
| const list = (event as CustomEvent<{ robots: RobotInfo[] }>).detail.robots; | |
| renderRobotList(list); | |
| }); | |
| robot.addEventListener("sessionStopped", async () => { | |
| await teardown(); | |
| selectedRobotId = null; | |
| micMuted = false; | |
| micBtn.classList.remove("muted"); | |
| if (robot?.isAuthenticated) setState("authenticated"); | |
| else setState("signed-out"); | |
| }); | |
| robot.addEventListener("disconnected", () => { | |
| showRobotPicker(false); | |
| if (robot?.isAuthenticated) setState("authenticated"); | |
| else setState("signed-out"); | |
| }); | |
| robot.addEventListener("error", (event) => { | |
| const detail = (event as CustomEvent<{ source: string; error: Error | string }>).detail; | |
| console.error(`[robot:${detail.source}]`, detail.error); | |
| }); | |
| } | |
| // βββ HF user info βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| interface HfUserInfo { | |
| handle?: string; | |
| picture?: string; | |
| } | |
| async function loadHfUserInfo(): Promise<void> { | |
| const token = sessionStorage.getItem("hf_token"); | |
| if (!token) return; | |
| const cacheKey = "reachyTales.hfUserInfo"; | |
| const cached = sessionStorage.getItem(cacheKey); | |
| if (cached) { | |
| try { | |
| applyHfUserInfo(JSON.parse(cached) as HfUserInfo); | |
| return; | |
| } catch { | |
| sessionStorage.removeItem(cacheKey); | |
| } | |
| } | |
| try { | |
| const res = await fetch("https://huggingface.co/oauth/userinfo", { | |
| headers: { Authorization: `Bearer ${token}` }, | |
| }); | |
| if (!res.ok) return; | |
| const data = (await res.json()) as { preferred_username?: string; picture?: string }; | |
| const info: HfUserInfo = { | |
| ...(data.preferred_username !== undefined && { handle: data.preferred_username }), | |
| ...(data.picture !== undefined && { picture: data.picture }), | |
| }; | |
| sessionStorage.setItem(cacheKey, JSON.stringify(info)); | |
| applyHfUserInfo(info); | |
| } catch (err) { | |
| console.warn("[main] loadHfUserInfo failed:", err); | |
| } | |
| } | |
| function applyHfUserInfo(info: HfUserInfo): void { | |
| if (info.handle) hfUserName.textContent = "@" + info.handle; | |
| if (info.picture) setHfAvatar(info.picture); | |
| } | |
| function setHfAvatar(url: string): void { | |
| hfAvatar.onload = () => hfAvatar.classList.add("loaded"); | |
| hfAvatar.onerror = () => hfAvatar.classList.remove("loaded"); | |
| hfAvatar.src = url; | |
| } | |
| function clearHfUser(): void { | |
| hfAvatar.classList.remove("loaded"); | |
| hfAvatar.removeAttribute("src"); | |
| sessionStorage.removeItem("reachyTales.hfUserInfo"); | |
| } | |
| // βββ Boot βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| async function boot(): Promise<void> { | |
| const clientId = resolveHfClientId(settings.hfClientId); | |
| robot = new window.ReachyMini({ | |
| appName: "reachy-tales", | |
| ...(clientId !== undefined && { clientId }), | |
| }); | |
| wireRobot(); | |
| let authenticated = false; | |
| try { | |
| authenticated = await robot.authenticate(); | |
| } catch (err) { | |
| console.warn("[main] authenticate() failed:", err); | |
| const message = err instanceof Error ? err.message : String(err); | |
| if (/clientId/i.test(message)) setCaption("Add HF client ID in settings", "muted"); | |
| } | |
| if (authenticated) { | |
| hfUserName.textContent = "@" + (robot.username ?? ""); | |
| hfUser.classList.remove("hidden"); | |
| void loadHfUserInfo(); | |
| setState("authenticated"); | |
| } else { | |
| hfUser.classList.add("hidden"); | |
| clearHfUser(); | |
| setState("signed-out"); | |
| } | |
| } | |
| function whenReachyReady(): Promise<void> { | |
| if (window.ReachyMini) return Promise.resolve(); | |
| return new Promise((resolve) => { | |
| window.addEventListener("reachymini:ready", () => resolve(), { once: true }); | |
| }); | |
| } | |
| // Strip `booting` after first paint so the orb's state-transition animations | |
| // don't fire on initial load. | |
| requestAnimationFrame(() => { | |
| requestAnimationFrame(() => { | |
| document.body.classList.remove("booting"); | |
| }); | |
| }); | |
| whenReachyReady().then(boot).catch(onFatalError); | |