reachy-tales / src /main.ts
Nicolas Rabault
ui: mirror minimal-conversation-app shell + HF runtime OAuth injection
fec23e5
Raw
History Blame Contribute Delete
32.4 kB
import "./style.css";
import { HeadWobbler } from "./head-wobbler.js";
import { AntennasOscillator } from "./antennas.js";
import { StoryBridge, type BridgeAppState } from "./story-bridge.js";
import { loadNarrator, loadStory } from "story-engine";
import type { LoadedContent } from "story-engine";
import type { ReachyMiniInstance, RobotInfo } from "./globals.d.ts";
// ─── Content loading (Vite bundles content/ as static assets) ────────────
const narratorJson = (await import("../content/narrator/narrator.json")).default;
const piratesManifest = (await import("../content/story_pirates/manifest.json")).default;
const piratesMap = (await import("../content/story_pirates/map.json")).default;
const piratesFlags = (await import("../content/story_pirates/flags.json")).default;
const piratesCharacters: Record<string, unknown> = {};
const characterModules = import.meta.glob("../content/story_pirates/characters/*.json", { eager: true });
for (const [path, mod] of Object.entries(characterModules)) {
const id = path.split("/").pop()!.replace(/\.json$/, "");
piratesCharacters[id] = (mod as { default: unknown }).default;
}
const piratesScenes: Record<string, unknown> = {};
const sceneModules = import.meta.glob("../content/story_pirates/scenes/*.json", { eager: true });
for (const [path, mod] of Object.entries(sceneModules)) {
const id = path.split("/").pop()!.replace(/\.json$/, "");
piratesScenes[id] = (mod as { default: unknown }).default;
}
const piratesStorylines: Record<string, unknown> = {};
const storylineModules = import.meta.glob("../content/story_pirates/storylines/chapter1/*.json", { eager: true });
for (const [path, mod] of Object.entries(storylineModules)) {
const id = path.split("/").pop()!.replace(/\.json$/, "");
piratesStorylines[id] = (mod as { default: unknown }).default;
}
const content: LoadedContent = {
narrator: loadNarrator(narratorJson),
stories: {
story_pirates: loadStory({
manifest: piratesManifest,
map: piratesMap,
flags: piratesFlags,
characters: piratesCharacters,
scenes: piratesScenes,
storylines: piratesStorylines,
}),
},
};
// ─── Settings ────────────────────────────────────────────────────────────
const DEFAULT_REALTIME_MODEL = "gpt-realtime";
const DEFAULT_SUMMARY_MODEL = "gpt-4o-mini";
const DEFAULT_VOICE = "cedar";
const STORAGE_KEYS = {
hfClientId: "reachyTales.hf.clientId",
apiKey: "reachyTales.openai.apiKey",
realtimeModel: "reachyTales.openai.realtimeModel",
summaryModel: "reachyTales.openai.summaryModel",
defaultVoice: "reachyTales.openai.defaultVoice",
} as const;
interface Settings {
hfClientId: string;
apiKey: string;
realtimeModel: string;
summaryModel: string;
defaultVoice: string;
}
function loadSettings(): Settings {
return {
hfClientId: localStorage.getItem(STORAGE_KEYS.hfClientId) ?? "",
apiKey: localStorage.getItem(STORAGE_KEYS.apiKey) ?? "",
realtimeModel: localStorage.getItem(STORAGE_KEYS.realtimeModel) ?? DEFAULT_REALTIME_MODEL,
summaryModel: localStorage.getItem(STORAGE_KEYS.summaryModel) ?? DEFAULT_SUMMARY_MODEL,
defaultVoice: localStorage.getItem(STORAGE_KEYS.defaultVoice) ?? DEFAULT_VOICE,
};
}
function saveSettings(s: Settings): void {
localStorage.setItem(STORAGE_KEYS.hfClientId, s.hfClientId);
localStorage.setItem(STORAGE_KEYS.apiKey, s.apiKey);
localStorage.setItem(STORAGE_KEYS.realtimeModel, s.realtimeModel);
localStorage.setItem(STORAGE_KEYS.summaryModel, s.summaryModel);
localStorage.setItem(STORAGE_KEYS.defaultVoice, s.defaultVoice);
}
// HF Spaces inject the OAuth client id at runtime via window.huggingface.
// On localhost the user supplies one through the settings panel.
function resolveHfClientId(saved: string): string | undefined {
const injected = window.huggingface?.variables?.OAUTH_CLIENT_ID;
if (injected) return injected;
return saved || undefined;
}
// ─── State machine ───────────────────────────────────────────────────────
type AppState =
| "signed-out"
| "authenticated"
| "connecting"
| "connected"
| "auto-selecting"
| "starting"
| "listening"
| "user-speaking"
| "processing"
| "ai-speaking"
| "swapping-character"
| "swapping-story"
| "error";
interface StateView {
caption: string;
disabled: boolean;
}
const STATE_VIEWS: Record<AppState, StateView> = {
"signed-out": { caption: "Sign in", disabled: false },
authenticated: { caption: "Tap to start", disabled: false },
connecting: { caption: "Connecting", disabled: true },
connected: { caption: "Choose a Reachy", disabled: true },
"auto-selecting": { caption: "Connecting", disabled: true },
starting: { caption: "Starting", disabled: true },
listening: { caption: "", disabled: false },
"user-speaking": { caption: "", disabled: false },
processing: { caption: "", disabled: false },
"ai-speaking": { caption: "", disabled: false },
"swapping-character": { caption: "Changing voice", disabled: true },
"swapping-story": { caption: "Loading story", disabled: true },
error: { caption: "Tap to retry", disabled: false },
};
const STATE_CLASS: Record<AppState, string> = {
"signed-out": "state-signed-out",
authenticated: "state-authenticated",
connecting: "state-connecting",
connected: "state-connected",
"auto-selecting": "state-auto-selecting",
starting: "state-starting",
listening: "state-listening",
"user-speaking": "state-user-speaking",
processing: "state-processing",
"ai-speaking": "state-ai-speaking",
"swapping-character": "state-swapping-character",
"swapping-story": "state-swapping-story",
error: "state-error",
};
const LIVE_STATES: ReadonlySet<AppState> = new Set([
"listening",
"user-speaking",
"processing",
"ai-speaking",
"starting",
"swapping-character",
"swapping-story",
]);
// ─── DOM refs ────────────────────────────────────────────────────────────
const $ = <T extends HTMLElement>(selector: string): T => {
const el = document.querySelector<T>(selector);
if (!el) throw new Error(`Missing element: ${selector}`);
return el;
};
const circleBtn = $<HTMLButtonElement>("#main-circle");
const circleCaption = $<HTMLParagraphElement>("#circle-caption");
const orbWrap = $<HTMLElement>(".orb-wrap");
const micBtn = $<HTMLButtonElement>("#mic-btn");
const stopBtn = $<HTMLButtonElement>("#stop-btn");
const robotPicker = $<HTMLElement>("#robot-picker");
const robotList = $<HTMLElement>("#robot-list");
const hfUser = $<HTMLSpanElement>("#hf-user");
const hfAvatar = $<HTMLImageElement>("#hf-avatar");
const hfUserName = $<HTMLSpanElement>("#hf-user-name");
const settingsBtn = $<HTMLButtonElement>("#settings-btn");
const settingsModal = $<HTMLDialogElement>("#settings-modal");
const inputClientId = $<HTMLInputElement>("#hf-client-id");
const hfClientIdField = $<HTMLLabelElement>("#hf-client-id-field");
const inputApiKey = $<HTMLInputElement>("#openai-key");
const inputRealtimeModel = $<HTMLInputElement>("#openai-realtime-model");
const inputSummaryModel = $<HTMLInputElement>("#openai-summary-model");
const inputVoice = $<HTMLSelectElement>("#openai-voice");
const hfLogoutBtn = $<HTMLButtonElement>("#hf-logout");
const settingsForm = settingsModal.querySelector<HTMLFormElement>("form")!;
const settingsTabs = Array.from(settingsModal.querySelectorAll<HTMLButtonElement>(".tab"));
const settingsPanels = Array.from(settingsModal.querySelectorAll<HTMLElement>("[data-tab-panel]"));
type SettingsTab = "access" | "story";
// ─── Runtime state ──────────────────────────────────────────────────────
let currentState: AppState = "signed-out";
let selectedRobotId: string | null = null;
let settings: Settings = loadSettings();
let knownRobots: RobotInfo[] = [];
let robot: ReachyMiniInstance | null = null;
let bridge: StoryBridge | null = null;
let openaiSink: HTMLAudioElement | null = null;
let wobbler: HeadWobbler | null = null;
let antennas: AntennasOscillator | null = null;
let micLevel: MicLevelMonitor | null = null;
let aiLevel: AiLevelMonitor | null = null;
let micMuted = false;
// ─── UI rendering ───────────────────────────────────────────────────────
function setState(next: AppState): void {
currentState = next;
const view = STATE_VIEWS[next];
circleBtn.disabled = view.disabled;
circleBtn.className = `circle ${STATE_CLASS[next]}`;
if (next !== "error") setCaption(view.caption);
const live = LIVE_STATES.has(next);
orbWrap.classList.toggle("live", live);
micBtn.setAttribute("aria-hidden", live ? "false" : "true");
stopBtn.setAttribute("aria-hidden", live ? "false" : "true");
micBtn.tabIndex = live ? 0 : -1;
stopBtn.tabIndex = live ? 0 : -1;
if (next !== "connected") showRobotPicker(false);
}
function setCaption(text: string, kind: "" | "error" | "muted" = ""): void {
const trimmed = text.trim();
circleCaption.textContent = trimmed;
circleCaption.className = `circle-caption${kind ? ` ${kind}` : ""}${trimmed ? "" : " empty"}`;
}
function renderRobotList(robots: RobotInfo[]): void {
knownRobots = robots;
robotList.innerHTML = "";
if (currentState !== "connected") return;
if (!robots.length) {
const empty = document.createElement("div");
empty.className = "robot-empty";
empty.textContent = "No Reachy online yet.";
robotList.appendChild(empty);
showRobotPicker(true);
setCaption("Waiting for Reachy", "muted");
return;
}
if (robots.length === 1 && !selectedRobotId) {
const only = robots[0]!;
selectedRobotId = only.id;
showRobotPicker(false);
setState("auto-selecting");
window.setTimeout(() => {
if (currentState === "auto-selecting") void doStart();
}, 500);
return;
}
showRobotPicker(true);
setCaption(STATE_VIEWS.connected.caption);
for (const r of robots) {
const card = document.createElement("div");
card.className = "robot-card";
if (r.id === selectedRobotId) card.classList.add("selected");
const name = document.createElement("div");
name.className = "name";
name.textContent = r.meta?.name || "Reachy Mini";
const idBadge = document.createElement("div");
idBadge.className = "id";
idBadge.textContent = r.id.slice(0, 12) + "…";
card.append(name, idBadge);
card.addEventListener("click", () => {
selectedRobotId = r.id;
showRobotPicker(false);
void doStart();
});
robotList.appendChild(card);
}
}
function showRobotPicker(show: boolean): void {
robotPicker.classList.toggle("hidden", !show);
}
// ─── Settings modal ─────────────────────────────────────────────────────
function openSettings(tab: SettingsTab = "access"): void {
inputClientId.value = settings.hfClientId;
inputApiKey.value = settings.apiKey;
inputRealtimeModel.value = settings.realtimeModel;
inputSummaryModel.value = settings.summaryModel;
inputVoice.value = settings.defaultVoice;
// Only show the HF client ID field on localhost (or when one is already
// saved and might need editing). Deployed Spaces get the clientId
// injected through window.huggingface so the field would be confusing.
const needsClientIdField =
!window.huggingface?.variables?.OAUTH_CLIENT_ID &&
(location.hostname === "localhost" ||
location.hostname === "127.0.0.1" ||
Boolean(settings.hfClientId));
hfClientIdField.classList.toggle("hidden", !needsClientIdField);
setSettingsTab(tab);
settingsModal.showModal();
}
function setSettingsTab(tab: SettingsTab): void {
for (const btn of settingsTabs) {
const isActive = btn.dataset.tab === tab;
btn.classList.toggle("active", isActive);
btn.setAttribute("aria-selected", isActive ? "true" : "false");
}
for (const panel of settingsPanels) {
const isActive = panel.dataset.tabPanel === tab;
panel.classList.toggle("active", isActive);
panel.hidden = !isActive;
}
}
for (const btn of settingsTabs) {
btn.addEventListener("click", () => {
const tab = btn.dataset.tab as SettingsTab | undefined;
if (tab) setSettingsTab(tab);
});
}
settingsBtn.addEventListener("click", () => openSettings("access"));
settingsForm.addEventListener("submit", (event) => {
const submitter = (event as SubmitEvent).submitter as HTMLButtonElement | null;
if (submitter?.value !== "save") return;
const previousClientId = settings.hfClientId;
settings = {
hfClientId: inputClientId.value.trim(),
apiKey: inputApiKey.value.trim(),
realtimeModel: inputRealtimeModel.value.trim() || DEFAULT_REALTIME_MODEL,
summaryModel: inputSummaryModel.value.trim() || DEFAULT_SUMMARY_MODEL,
defaultVoice: inputVoice.value || DEFAULT_VOICE,
};
saveSettings(settings);
if (settings.hfClientId !== previousClientId) location.reload();
});
hfLogoutBtn.addEventListener("click", () => {
if (!robot) return;
robot.logout();
settingsModal.close();
location.reload();
});
// ─── Click handler for the central circle ──────────────────────────────
circleBtn.addEventListener("click", async () => {
try {
switch (currentState) {
case "signed-out":
if (!robot) return;
if (!resolveHfClientId(settings.hfClientId)) {
setCaption("Add HF client ID in settings", "error");
openSettings();
return;
}
await robot.login();
return;
case "authenticated":
if (!settings.apiKey) {
setCaption("Add OpenAI key in settings", "error");
openSettings();
return;
}
await doConnect();
return;
case "error":
selectedRobotId = null;
if (robot?.isAuthenticated) setState("authenticated");
else setState("signed-out");
return;
default:
return;
}
} catch (err) {
onFatalError(err);
}
});
// ─── Side controls (mic mute + stop) ────────────────────────────────────
micBtn.addEventListener("click", () => {
if (!robot) return;
micMuted = !micMuted;
robot.setMicMuted(micMuted);
micBtn.classList.toggle("muted", micMuted);
micBtn.setAttribute("aria-label", micMuted ? "Unmute" : "Mute");
micBtn.title = micMuted ? "Unmute" : "Mute";
});
stopBtn.addEventListener("click", async () => {
await teardown();
selectedRobotId = null;
micMuted = false;
micBtn.classList.remove("muted");
if (!robot) {
setState("signed-out");
} else if (robot.state !== "disconnected") {
setState("connected");
renderRobotList(knownRobots);
} else if (robot.isAuthenticated) {
setState("authenticated");
} else {
setState("signed-out");
}
});
// ─── High-level flow steps ──────────────────────────────────────────────
async function doConnect(): Promise<void> {
if (!robot) return;
setState("connecting");
try {
if (robot.state === "disconnected") await robot.connect();
setState("connected");
renderRobotList(knownRobots);
} catch (err) {
onFatalError(err);
}
}
async function doStart(): Promise<void> {
if (!robot || !selectedRobotId) return;
if (!settings.apiKey) {
setCaption("Add OpenAI key in settings", "error");
openSettings();
return;
}
setState("starting");
try {
await robot.startSession(selectedRobotId);
} catch (err) {
onFatalError(err);
return;
}
const robotMicTrack = getRobotMicTrack(robot);
if (!robotMicTrack) {
onFatalError(new Error("Could not find the robot's microphone track"));
return;
}
startMicLevelMonitor(robotMicTrack);
startAntennas();
bridge = new StoryBridge({
content,
robot,
apiKey: settings.apiKey,
realtimeModel: settings.realtimeModel,
summaryModel: settings.summaryModel,
defaultVoice: settings.defaultVoice,
getRobotMicTrack: () => getRobotMicTrack(robot!),
onOpenAiOutputTrack: (track) => {
routeOpenaiToRobot(track);
startWobbler(track);
startAiLevelMonitor(track);
},
onAppStateChange: (s: BridgeAppState) => {
const mapped = s as AppState;
// If the bridge says listening but we have audio that is still
// playing out, defer the swap until the AI level monitor confirms
// silence β€” same trick the reference uses to avoid snapping back
// to bars mid-sentence.
if (mapped === "listening" && currentState === "ai-speaking" && aiLevel) {
aiLevel.waitForSilence(900, () => {
if (currentState === "ai-speaking") setState("listening");
});
return;
}
if (mapped === "user-speaking") {
aiLevel?.cancelSilenceWait();
wobbler?.reset();
antennas?.freeze();
} else if (mapped === "ai-speaking") {
aiLevel?.cancelSilenceWait();
antennas?.resume();
} else if (mapped === "processing" || mapped === "listening") {
antennas?.resume();
}
setState(mapped);
},
});
try {
await bridge.start();
} catch (err) {
onFatalError(err);
return;
}
robot.setMicMuted(false);
}
function getRobotMicTrack(robotInstance: ReachyMiniInstance): MediaStreamTrack | null {
const pc = robotInstance._pc;
if (!pc) return null;
for (const receiver of pc.getReceivers()) {
if (receiver.track && receiver.track.kind === "audio") return receiver.track;
}
return null;
}
function routeOpenaiToRobot(track: MediaStreamTrack): void {
if (!robot) return;
const pc = robot._pc;
if (!pc) return;
const audioSender = pc.getSenders().find((s) => s.track && s.track.kind === "audio");
if (audioSender) {
audioSender.replaceTrack(track).catch((err) => {
console.error("[main] replaceTrack failed", err);
});
} else {
console.warn("[main] no audio sender on the robot peer β€” bidirectional audio missing");
}
// Some browsers don't decode an inbound track unless it has a local sink.
if (!openaiSink) {
openaiSink = document.createElement("audio");
openaiSink.autoplay = true;
openaiSink.muted = true;
document.body.appendChild(openaiSink);
}
openaiSink.srcObject = new MediaStream([track]);
}
// ─── Head motion agent ──────────────────────────────────────────────────
function startWobbler(assistantTrack: MediaStreamTrack): void {
if (!robot) return;
wobbler?.stop();
wobbler = new HeadWobbler({
track: assistantTrack,
onOffsets: ({ roll, pitch, yaw }) => {
robot?.setHeadPose(roll, pitch, yaw);
},
});
wobbler.start();
}
function stopWobbler(): void {
wobbler?.stop();
wobbler = null;
robot?.setHeadPose(0, 0, 0);
}
// ─── Antennas oscillator ────────────────────────────────────────────────
function startAntennas(): void {
if (!robot) return;
antennas?.stop();
antennas = new AntennasOscillator({
onAntennas: (right, left) => {
robot?.setAntennas(right, left);
},
});
antennas.start();
}
function stopAntennas(): void {
antennas?.stop();
antennas = null;
robot?.setAntennas(0, 0);
}
// ─── Mic-level monitor (drives --audio-level + per-band bars) ───────────
class MicLevelMonitor {
private ctx: AudioContext | null = null;
private analyser: AnalyserNode | null = null;
private source: MediaStreamAudioSourceNode | null = null;
private raf = 0;
private timeBuf: Float32Array<ArrayBuffer> | null = null;
private freqBuf: Uint8Array<ArrayBuffer> | null = null;
private level = 0;
private bands = [0, 0, 0, 0, 0];
private static readonly BAND_EDGES = [4, 8, 16, 32, 64, 128];
private static readonly LOG1P_10 = Math.log1p(10);
private static compress(v: number): number {
return Math.log1p(v * 10) / MicLevelMonitor.LOG1P_10;
}
start(track: MediaStreamTrack): void {
this.stop();
const ctx = new AudioContext();
const src = ctx.createMediaStreamSource(new MediaStream([track]));
const analyser = ctx.createAnalyser();
analyser.fftSize = 1024;
analyser.smoothingTimeConstant = 0.75;
src.connect(analyser);
this.ctx = ctx;
this.source = src;
this.analyser = analyser;
this.timeBuf = new Float32Array(new ArrayBuffer(analyser.fftSize * 4));
this.freqBuf = new Uint8Array(new ArrayBuffer(analyser.frequencyBinCount));
const rootStyle = document.documentElement.style;
const tick = () => {
const an = this.analyser;
const tbuf = this.timeBuf;
const fbuf = this.freqBuf;
if (!an || !tbuf || !fbuf) return;
an.getFloatTimeDomainData(tbuf);
let sum = 0;
for (let i = 0; i < tbuf.length; i++) sum += tbuf[i]! * tbuf[i]!;
const rms = Math.sqrt(sum / tbuf.length);
const boosted = Math.min(1, Math.pow(rms * 6, 0.7));
const levelAttack = boosted > this.level ? 0.55 : 0.12;
this.level += (boosted - this.level) * levelAttack;
rootStyle.setProperty("--audio-level", this.level.toFixed(3));
an.getByteFrequencyData(fbuf);
const edges = MicLevelMonitor.BAND_EDGES;
for (let b = 0; b < 5; b++) {
const lo = edges[b]!;
const hi = edges[b + 1]!;
let bandSum = 0;
for (let j = lo; j < hi; j++) bandSum += fbuf[j]!;
const raw = MicLevelMonitor.compress(bandSum / (hi - lo) / 255);
const bandAttack = raw > this.bands[b]! ? 0.35 : 0.12;
this.bands[b] = this.bands[b]! + (raw - this.bands[b]!) * bandAttack;
rootStyle.setProperty(`--bar${b}`, Math.min(1, this.bands[b]!).toFixed(3));
}
this.raf = requestAnimationFrame(tick);
};
this.raf = requestAnimationFrame(tick);
}
stop(): void {
cancelAnimationFrame(this.raf);
this.raf = 0;
try {
this.source?.disconnect();
this.analyser?.disconnect();
this.ctx?.close();
} catch {
// ignored
}
this.ctx = null;
this.source = null;
this.analyser = null;
this.timeBuf = null;
this.freqBuf = null;
this.level = 0;
this.bands = [0, 0, 0, 0, 0];
const rootStyle = document.documentElement.style;
rootStyle.setProperty("--audio-level", "0");
for (let b = 0; b < 5; b++) rootStyle.setProperty(`--bar${b}`, "0");
}
}
function startMicLevelMonitor(track: MediaStreamTrack): void {
micLevel ??= new MicLevelMonitor();
micLevel.start(track);
}
function stopMicLevelMonitor(): void {
micLevel?.stop();
}
// ─── AI-level monitor (drives --ai-audio-level + silence detector) ──────
class AiLevelMonitor {
private ctx: AudioContext | null = null;
private analyser: AnalyserNode | null = null;
private source: MediaStreamAudioSourceNode | null = null;
private raf = 0;
private timeBuf: Float32Array<ArrayBuffer> | null = null;
private level = 0;
private lastActiveTs = 0;
private silenceWait: { quietMs: number; cb: () => void; maxWaitTimer: number | null } | null = null;
private static readonly SILENCE_THRESHOLD = 0.006;
start(track: MediaStreamTrack): void {
this.stop();
const ctx = new AudioContext();
const src = ctx.createMediaStreamSource(new MediaStream([track]));
const analyser = ctx.createAnalyser();
analyser.fftSize = 1024;
analyser.smoothingTimeConstant = 0.75;
src.connect(analyser);
this.ctx = ctx;
this.source = src;
this.analyser = analyser;
this.timeBuf = new Float32Array(new ArrayBuffer(analyser.fftSize * 4));
this.lastActiveTs = performance.now();
const rootStyle = document.documentElement.style;
const tick = () => {
const an = this.analyser;
const buf = this.timeBuf;
if (!an || !buf) return;
an.getFloatTimeDomainData(buf);
let sum = 0;
for (let i = 0; i < buf.length; i++) sum += buf[i]! * buf[i]!;
const rms = Math.sqrt(sum / buf.length);
const boosted = Math.min(1, Math.pow(rms * 6, 0.7));
const levelAttack = boosted > this.level ? 0.55 : 0.12;
this.level += (boosted - this.level) * levelAttack;
rootStyle.setProperty("--ai-audio-level", this.level.toFixed(3));
const now = performance.now();
if (rms > AiLevelMonitor.SILENCE_THRESHOLD) {
this.lastActiveTs = now;
} else if (this.silenceWait) {
const quietFor = now - this.lastActiveTs;
if (quietFor >= this.silenceWait.quietMs) {
const { cb, maxWaitTimer } = this.silenceWait;
this.silenceWait = null;
if (maxWaitTimer !== null) clearTimeout(maxWaitTimer);
try { cb(); } catch (err) { console.warn("[ai-level] silence callback threw:", err); }
}
}
this.raf = requestAnimationFrame(tick);
};
this.raf = requestAnimationFrame(tick);
}
stop(): void {
cancelAnimationFrame(this.raf);
this.raf = 0;
this.cancelSilenceWait();
try {
this.source?.disconnect();
this.analyser?.disconnect();
this.ctx?.close();
} catch {
// ignored
}
this.ctx = null;
this.source = null;
this.analyser = null;
this.timeBuf = null;
this.level = 0;
document.documentElement.style.setProperty("--ai-audio-level", "0");
}
waitForSilence(quietMs: number, cb: () => void, maxWaitMs = 8000): void {
this.cancelSilenceWait();
const maxWaitTimer = window.setTimeout(() => {
if (this.silenceWait?.cb === cb) {
this.silenceWait = null;
try { cb(); } catch (err) { console.warn("[ai-level] max-wait callback threw:", err); }
}
}, maxWaitMs);
this.silenceWait = { quietMs, cb, maxWaitTimer };
}
cancelSilenceWait(): void {
if (!this.silenceWait) return;
if (this.silenceWait.maxWaitTimer !== null) clearTimeout(this.silenceWait.maxWaitTimer);
this.silenceWait = null;
}
}
function startAiLevelMonitor(track: MediaStreamTrack): void {
aiLevel ??= new AiLevelMonitor();
aiLevel.start(track);
}
function stopAiLevelMonitor(): void {
aiLevel?.stop();
}
// ─── Teardown / errors ──────────────────────────────────────────────────
async function teardown(): Promise<void> {
try {
await bridge?.stop();
} catch {
// ignored
}
bridge = null;
stopWobbler();
stopAntennas();
stopMicLevelMonitor();
stopAiLevelMonitor();
if (openaiSink) {
openaiSink.srcObject = null;
openaiSink.remove();
openaiSink = null;
}
try {
await robot?.stopSession();
} catch {
// ignored
}
}
async function onFatalError(err: unknown): Promise<void> {
const message = err instanceof Error ? err.message : String(err);
console.error("[main] error:", err);
setState("error");
circleCaption.title = message;
await teardown();
}
// ─── Robot event wiring ─────────────────────────────────────────────────
function wireRobot(): void {
if (!robot) return;
robot.addEventListener("robotsChanged", (event) => {
const list = (event as CustomEvent<{ robots: RobotInfo[] }>).detail.robots;
renderRobotList(list);
});
robot.addEventListener("sessionStopped", async () => {
await teardown();
selectedRobotId = null;
micMuted = false;
micBtn.classList.remove("muted");
if (robot?.isAuthenticated) setState("authenticated");
else setState("signed-out");
});
robot.addEventListener("disconnected", () => {
showRobotPicker(false);
if (robot?.isAuthenticated) setState("authenticated");
else setState("signed-out");
});
robot.addEventListener("error", (event) => {
const detail = (event as CustomEvent<{ source: string; error: Error | string }>).detail;
console.error(`[robot:${detail.source}]`, detail.error);
});
}
// ─── HF user info ───────────────────────────────────────────────────────
interface HfUserInfo {
handle?: string;
picture?: string;
}
async function loadHfUserInfo(): Promise<void> {
const token = sessionStorage.getItem("hf_token");
if (!token) return;
const cacheKey = "reachyTales.hfUserInfo";
const cached = sessionStorage.getItem(cacheKey);
if (cached) {
try {
applyHfUserInfo(JSON.parse(cached) as HfUserInfo);
return;
} catch {
sessionStorage.removeItem(cacheKey);
}
}
try {
const res = await fetch("https://huggingface.co/oauth/userinfo", {
headers: { Authorization: `Bearer ${token}` },
});
if (!res.ok) return;
const data = (await res.json()) as { preferred_username?: string; picture?: string };
const info: HfUserInfo = {
...(data.preferred_username !== undefined && { handle: data.preferred_username }),
...(data.picture !== undefined && { picture: data.picture }),
};
sessionStorage.setItem(cacheKey, JSON.stringify(info));
applyHfUserInfo(info);
} catch (err) {
console.warn("[main] loadHfUserInfo failed:", err);
}
}
function applyHfUserInfo(info: HfUserInfo): void {
if (info.handle) hfUserName.textContent = "@" + info.handle;
if (info.picture) setHfAvatar(info.picture);
}
function setHfAvatar(url: string): void {
hfAvatar.onload = () => hfAvatar.classList.add("loaded");
hfAvatar.onerror = () => hfAvatar.classList.remove("loaded");
hfAvatar.src = url;
}
function clearHfUser(): void {
hfAvatar.classList.remove("loaded");
hfAvatar.removeAttribute("src");
sessionStorage.removeItem("reachyTales.hfUserInfo");
}
// ─── Boot ───────────────────────────────────────────────────────────────
async function boot(): Promise<void> {
const clientId = resolveHfClientId(settings.hfClientId);
robot = new window.ReachyMini({
appName: "reachy-tales",
...(clientId !== undefined && { clientId }),
});
wireRobot();
let authenticated = false;
try {
authenticated = await robot.authenticate();
} catch (err) {
console.warn("[main] authenticate() failed:", err);
const message = err instanceof Error ? err.message : String(err);
if (/clientId/i.test(message)) setCaption("Add HF client ID in settings", "muted");
}
if (authenticated) {
hfUserName.textContent = "@" + (robot.username ?? "");
hfUser.classList.remove("hidden");
void loadHfUserInfo();
setState("authenticated");
} else {
hfUser.classList.add("hidden");
clearHfUser();
setState("signed-out");
}
}
function whenReachyReady(): Promise<void> {
if (window.ReachyMini) return Promise.resolve();
return new Promise((resolve) => {
window.addEventListener("reachymini:ready", () => resolve(), { once: true });
});
}
// Strip `booting` after first paint so the orb's state-transition animations
// don't fire on initial load.
requestAnimationFrame(() => {
requestAnimationFrame(() => {
document.body.classList.remove("booting");
});
});
whenReachyReady().then(boot).catch(onFatalError);