feat(app): copy plumbing (openai-realtime, antennas, head-wobbler, move-player, robot-tools) from reference app

Relax noUncheckedIndexedAccess in tsconfig.app.json only (upstream reference files
use array indexing without undefined guards; base tsconfig is unchanged).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

src/antennas.ts

@@ -0,0 +1,106 @@
+/**
+ * Antennas oscillator - mirrors the `BreathingMove` antenna behaviour from
+ * `reachy_mini_conversation_app/moves.py`: a gentle sinusoid in opposition
+ * (right ↔ −left) at ~0.5 Hz with a ±15° amplitude. The two ears "breathe"
+ * asymmetrically which makes an idle robot feel alive.
+ *
+ * When the user starts speaking we freeze the antennas on their current
+ * position (just like the Python `set_listening(True)` path), and smoothly
+ * blend back into the oscillation once they stop.
+ */
+
+const FREQ_HZ = 0.5;
+const AMPLITUDE_DEG = 15;
+const BLEND_MS = 400; // time to smoothly ramp back after a freeze
+
+export interface AntennasOscillatorOptions {
+  onAntennas: (right: number, left: number) => void;
+  /** Update rate in Hz. 30 is plenty for smooth motion. Defaults to 30. */
+  updateHz?: number;
+}
+
+export class AntennasOscillator {
+  private readonly onAntennas: (right: number, left: number) => void;
+  private readonly periodMs: number;
+
+  private timer: number | null = null;
+  private t = 0;
+  private frozen = false;
+  private frozenRight = 0;
+  private frozenLeft = 0;
+  // 0..1 crossfader between frozen pose and live oscillation.
+  private blend = 1;
+
+  constructor(options: AntennasOscillatorOptions) {
+    this.onAntennas = options.onAntennas;
+    this.periodMs = 1000 / (options.updateHz ?? 30);
+  }
+
+  start(): void {
+    if (this.timer !== null) return;
+    this.t = 0;
+    this.blend = 1;
+    this.frozen = false;
+    this.timer = window.setInterval(() => this.tick(), this.periodMs);
+  }
+
+  stop(): void {
+    if (this.timer !== null) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    // Return to neutral so the ears don't stay stuck at a random angle.
+    this.onAntennas(0, 0);
+  }
+
+  /**
+   * Freeze the antennas on their current position. Used when the user is
+   * speaking: the robot "listens attentively" instead of keeping its
+   * idle animation.
+   */
+  freeze(): void {
+    if (this.frozen) return;
+    this.frozen = true;
+    // Snapshot the current oscillation output so the freeze is seamless.
+    const { right, left } = this.currentOscillation();
+    this.frozenRight = right;
+    this.frozenLeft = left;
+    this.blend = 0; // fully on the frozen values
+  }
+
+  /**
+   * Release the freeze; antennas ramp back into the oscillation over
+   * ~BLEND_MS ms.
+   */
+  resume(): void {
+    if (!this.frozen) return;
+    this.frozen = false;
+    // `blend` will rise back to 1 via tick().
+  }
+
+  // ─── Internals ───────────────────────────────────────────────────────
+
+  private tick(): void {
+    this.t += this.periodMs / 1000;
+
+    // Update blend towards its target (0 when frozen, 1 when live).
+    const target = this.frozen ? 0 : 1;
+    const step = this.periodMs / BLEND_MS;
+    if (this.blend < target) this.blend = Math.min(1, this.blend + step);
+    else if (this.blend > target) this.blend = Math.max(0, this.blend - step);
+
+    const { right: liveR, left: liveL } = this.currentOscillation();
+    const right = liveR * this.blend + this.frozenRight * (1 - this.blend);
+    const left = liveL * this.blend + this.frozenLeft * (1 - this.blend);
+
+    this.onAntennas(right, left);
+  }
+
+  private currentOscillation(): { right: number; left: number } {
+    const phase = 2 * Math.PI * FREQ_HZ * this.t;
+    const right = AMPLITUDE_DEG * Math.sin(phase);
+    // Opposition: when right antenna goes "up", left goes "down".
+    const left = -AMPLITUDE_DEG * Math.sin(phase);
+    return { right, left };
+  }
+}

src/globals.d.ts

@@ -0,0 +1,75 @@
+/**
+ * Type declarations for the ReachyMini SDK loaded from a CDN script tag in
+ * index.html. We only expose what we actually consume here.
+ */
+
+export interface RobotInfo {
+  id: string;
+  meta?: { name?: string };
+}
+
+export interface RobotState {
+  head: { roll: number; pitch: number; yaw: number };
+  antennas: { right: number; left: number };
+}
+
+export interface ReachyMiniOptions {
+  signalingUrl?: string;
+  enableMicrophone?: boolean;
+  clientId?: string;
+  appName?: string;
+}
+
+export interface ReachyMiniInstance extends EventTarget {
+  readonly state: "disconnected" | "connected" | "streaming";
+  readonly robots: RobotInfo[];
+  readonly username: string | null;
+  readonly isAuthenticated: boolean;
+  readonly micSupported: boolean;
+  readonly micMuted: boolean;
+  readonly audioMuted: boolean;
+
+  /**
+   * Exposed by the SDK as an internal field but used by the webrtc_example
+   * reference to pull stats; we read it to get the audio receiver/senders.
+   */
+  _pc: RTCPeerConnection | null;
+
+  authenticate(): Promise<boolean>;
+  login(): Promise<void>;
+  logout(): void;
+
+  connect(token?: string): Promise<void>;
+  disconnect(): void;
+
+  startSession(robotId: string): Promise<void>;
+  stopSession(): Promise<void>;
+
+  attachVideo(el: HTMLVideoElement): () => void;
+
+  setHeadPose(roll: number, pitch: number, yaw: number): boolean;
+  setAntennas(right: number, left: number): boolean;
+  playSound(file: string): boolean;
+
+  /**
+   * Send an arbitrary JSON message on the robot data channel. Useful for
+   * wire-format commands not exposed by dedicated helpers (e.g.
+   * `set_full_target` to stream dance frames).
+   */
+  sendRaw(data: unknown): boolean;
+
+  setAudioMuted(muted: boolean): void;
+  setMicMuted(muted: boolean): void;
+}
+
+export type ReachyMiniConstructor = new (
+  options?: ReachyMiniOptions,
+) => ReachyMiniInstance;
+
+declare global {
+  interface Window {
+    ReachyMini: ReachyMiniConstructor;
+  }
+}
+
+export {};

src/head-wobbler.ts

@@ -0,0 +1,256 @@
+/**
+ * Head wobble generator - JS port of the Python
+ * `reachy_mini_conversation_app/audio/speech_tapper.py` + `head_wobbler.py`.
+ *
+ * Purpose: take the assistant's outgoing voice (the audio OpenAI plays back
+ * through the robot) and drive small, organic head sways + nods in sync with
+ * the speech, so Reachy "comes alive" while talking.
+ *
+ * The Python version sampled raw PCM base64 deltas over the OpenAI WebSocket
+ * API; we instead hook a Web Audio `AnalyserNode` to the remote
+ * `MediaStreamTrack` (which is how audio reaches us in WebRTC mode). Both
+ * yield the same loudness envelope we need to modulate the sine oscillators.
+ *
+ * The SDK only exposes rotational `setHeadPose(roll, pitch, yaw)` in degrees,
+ * so we drop the small (±mm-range) translations from the original Python
+ * pipeline - they're a cherry-on-top that isn't reachable through the public
+ * JS API anyway.
+ */
+
+// ─── Constants ported from speech_tapper.py ─────────────────────────────
+// Frame / hop timing. Hop drives how often we update offsets.
+const HOP_MS = 50; // produce one pose target every 50ms (20Hz)
+const FRAME_MS = 20; // RMS window in ms
+
+// Loudness mapping: RMS in dBFS is remapped to [0..1] with a slight gamma.
+const SWAY_DB_LOW = -46;
+const SWAY_DB_HIGH = -18;
+const SWAY_DB_GAMMA = 0.9;
+const SWAY_MASTER = 1.5;
+const SENS_DB_OFFSET = 4;
+
+// Hysteretic VAD thresholds on the sliding-frame dBFS envelope.
+const VAD_DB_ON = -35;
+const VAD_DB_OFF = -45;
+// Attack / release for the 0→1 envelope follower that multiplies the sway.
+const ENV_FOLLOW_GAIN = 0.65;
+
+// Sinusoid frequencies (Hz) + peak amplitudes (degrees).
+// All three axes use independent initial phases so the motion doesn't feel
+// mechanical when they align.
+const SWAY_F_PITCH = 2.2;
+const SWAY_A_PITCH_DEG = 4.5;
+
+const SWAY_F_YAW = 0.6;
+const SWAY_A_YAW_DEG = 7.5;
+
+const SWAY_F_ROLL = 1.3;
+const SWAY_A_ROLL_DEG = 2.25;
+
+// Deliberate delay between the audio we analyse and the matching head
+// motion. A touch of latency makes the motion feel slaved to the voice
+// (reactive, not predictive). Matches `MOVEMENT_LATENCY_S = 0.2` in Python.
+const MOVEMENT_LATENCY_S = 0.2;
+
+// ─── Types ──────────────────────────────────────────────────────────────
+
+export interface HeadOffsetsDeg {
+  roll: number;
+  pitch: number;
+  yaw: number;
+}
+
+export interface HeadWobblerOptions {
+  track: MediaStreamTrack;
+  onOffsets: (offsets: HeadOffsetsDeg) => void;
+  /**
+   * Master scalar applied on top of the per-axis amplitudes. Lets callers
+   * tone the whole effect down without recompiling constants. Defaults to 1.
+   */
+  gain?: number;
+}
+
+// ─── Implementation ─────────────────────────────────────────────────────
+
+/**
+ * Consumes a `MediaStreamTrack` (assistant voice) and emits head rotation
+ * offsets in degrees every hop (~50 ms), smoothly ramping in and out with
+ * the speech envelope.
+ *
+ * Call `start()` to begin, `stop()` to tear down, `reset()` to zero the
+ * internal sway state (useful when the user barges in on the assistant).
+ */
+export class HeadWobbler {
+  private readonly track: MediaStreamTrack;
+  private readonly onOffsets: (offsets: HeadOffsetsDeg) => void;
+  private readonly gain: number;
+
+  private audioCtx: AudioContext | null = null;
+  private analyser: AnalyserNode | null = null;
+  private sourceNode: MediaStreamAudioSourceNode | null = null;
+  private frameBuf: Float32Array<ArrayBuffer> | null = null;
+
+  private timer: number | null = null;
+
+  // Sway state
+  private t = 0; // seconds since start() for the sine oscillators
+  private swayEnv = 0; // 0..1 envelope follower on VAD-on/off
+  private vadOn = false;
+  private pendingOffsets: HeadOffsetsDeg[] = [];
+
+  // Per-axis phase offsets so the three sines don't line up on the downbeat.
+  private readonly phasePitch = 0.13 * Math.PI;
+  private readonly phaseYaw = 0.48 * Math.PI;
+  private readonly phaseRoll = 1.21 * Math.PI;
+
+  constructor(options: HeadWobblerOptions) {
+    this.track = options.track;
+    this.onOffsets = options.onOffsets;
+    this.gain = options.gain ?? 1.0;
+  }
+
+  start(): void {
+    if (this.audioCtx) return; // already running
+
+    // A dedicated AudioContext keeps the robot mic path untouched. We don't
+    // need the browser to actually play this back (the audio is also piped
+    // straight to the robot speaker via replaceTrack), we just want samples.
+    const ctx = new AudioContext();
+    const sourceStream = new MediaStream([this.track]);
+    const source = ctx.createMediaStreamSource(sourceStream);
+    const analyser = ctx.createAnalyser();
+    analyser.fftSize = 1024;
+    analyser.smoothingTimeConstant = 0;
+    source.connect(analyser);
+
+    const frameSamples = Math.max(
+      128,
+      Math.round((ctx.sampleRate * FRAME_MS) / 1000),
+    );
+    this.frameBuf = new Float32Array(new ArrayBuffer(frameSamples * 4));
+
+    this.audioCtx = ctx;
+    this.sourceNode = source;
+    this.analyser = analyser;
+
+    // Kick the tick loop on setInterval so the rate stays aligned with
+    // hop_ms regardless of requestAnimationFrame throttling when the tab
+    // is backgrounded. 20 Hz is low enough not to spam the robot.
+    this.timer = window.setInterval(() => this.tick(), HOP_MS);
+  }
+
+  stop(): void {
+    if (this.timer !== null) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    try {
+      this.sourceNode?.disconnect();
+      this.analyser?.disconnect();
+      this.audioCtx?.close();
+    } catch {
+      // ignored
+    }
+    this.sourceNode = null;
+    this.analyser = null;
+    this.audioCtx = null;
+    this.frameBuf = null;
+    this.pendingOffsets = [];
+
+    // Return to neutral head pose on teardown so the robot doesn't freeze
+    // mid-motion.
+    this.onOffsets({ roll: 0, pitch: 0, yaw: 0 });
+  }
+
+  /**
+   * Re-activate the internal AudioContext after a visibility change. Some
+   * browsers (Safari, iOS) suspend contexts when the tab is hidden and
+   * don't auto-resume; without this call the RMS meter reads zero forever
+   * after the first background trip.
+   */
+  resumeAudio(): void {
+    const ctx = this.audioCtx;
+    if (!ctx) return;
+    if (ctx.state === "suspended") {
+      ctx.resume().catch((err) => {
+        console.warn("[head-wobbler] audioCtx resume failed:", err);
+      });
+    }
+  }
+
+  /**
+   * Reset the sway envelope + latency queue. Call this when the user starts
+   * speaking while the assistant is still talking: we want motion to fade
+   * out smoothly instead of holding the last sine value.
+   */
+  reset(): void {
+    this.swayEnv = 0;
+    this.vadOn = false;
+    this.pendingOffsets = [];
+    this.onOffsets({ roll: 0, pitch: 0, yaw: 0 });
+  }
+
+  // ─── Internals ───────────────────────────────────────────────────────
+
+  private tick(): void {
+    if (!this.analyser || !this.frameBuf) return;
+
+    // 1. Measure current loudness in dBFS on the most recent ~20 ms.
+    this.analyser.getFloatTimeDomainData(this.frameBuf);
+    const db = rmsDbfs(this.frameBuf) + SENS_DB_OFFSET;
+
+    // 2. Hysteretic VAD on the dB envelope.
+    if (this.vadOn) {
+      if (db < VAD_DB_OFF) this.vadOn = false;
+    } else {
+      if (db > VAD_DB_ON) this.vadOn = true;
+    }
+
+    // 3. Smooth 0..1 envelope the sines are modulated by.
+    const target = this.vadOn ? 1 : 0;
+    this.swayEnv += (target - this.swayEnv) * ENV_FOLLOW_GAIN;
+
+    // 4. Loudness gain - how much of the max amplitude we reach on this hop.
+    const loud = loudnessGain(db);
+
+    // 5. Time-evolve the oscillators.
+    this.t += HOP_MS / 1000;
+    const twoPiT = 2 * Math.PI * this.t;
+    const mod = loud * this.swayEnv * this.gain * SWAY_MASTER;
+
+    const pitchDeg =
+      SWAY_A_PITCH_DEG * mod * Math.sin(twoPiT * SWAY_F_PITCH + this.phasePitch);
+    const yawDeg =
+      SWAY_A_YAW_DEG * mod * Math.sin(twoPiT * SWAY_F_YAW + this.phaseYaw);
+    const rollDeg =
+      SWAY_A_ROLL_DEG * mod * Math.sin(twoPiT * SWAY_F_ROLL + this.phaseRoll);
+
+    // 6. Enqueue with latency, then emit the oldest ready sample.
+    this.pendingOffsets.push({ roll: rollDeg, pitch: pitchDeg, yaw: yawDeg });
+    const maxQueue = Math.ceil((MOVEMENT_LATENCY_S * 1000) / HOP_MS);
+    while (this.pendingOffsets.length > maxQueue) {
+      const next = this.pendingOffsets.shift();
+      if (next) this.onOffsets(next);
+    }
+  }
+}
+
+// ─── Pure helpers ───────────────────────────────────────────────────────
+
+function rmsDbfs(samples: Float32Array<ArrayBuffer>): number {
+  let sum = 0;
+  for (let i = 0; i < samples.length; i++) {
+    const s = samples[i];
+    sum += s * s;
+  }
+  const rms = Math.sqrt(sum / Math.max(1, samples.length));
+  if (rms <= 1e-8) return -120;
+  return 20 * Math.log10(rms);
+}
+
+function loudnessGain(db: number): number {
+  // Linear remap [SWAY_DB_LOW..SWAY_DB_HIGH] -> [0..1] with a gamma.
+  const norm = (db - SWAY_DB_LOW) / (SWAY_DB_HIGH - SWAY_DB_LOW);
+  const clamped = Math.max(0, Math.min(1, norm));
+  return Math.pow(clamped, SWAY_DB_GAMMA);
+}

src/move-player.ts

@@ -0,0 +1,504 @@
+/**
+ * Move player - streams Reachy Mini recorded choreographies on the WebRTC
+ * data channel as `set_full_target` frames.
+ *
+ * Why not just ask the daemon to play a move by name?
+ *   The daemon *does* expose `POST /api/move/play/recorded-move-dataset/{ds}/
+ *   {move}` which plays a RecordedMove internally (see `move.py` on the
+ *   daemon, and `pollen-robotics/reachy-mini-{dances,emotions}-library` HF
+ *   datasets it preloads at boot). But that endpoint is HTTP on the robot's
+ *   LAN, which a browser running on `https://*.hf.space` can't reach (mixed
+ *   content, no DNS). The WebRTC data channel protocol does not have a
+ *   `play_dance` command either (`PlayMoveTaskRequest not yet implemented
+ *   over WS`). So we replicate what the daemon does internally: fetch the
+ *   move JSON from the public HF CDN, evaluate it client-side at 100 Hz,
+ *   and stream each frame as `set_full_target`.
+ *
+ * What the JSON looks like (recorded move, see `RecordedMove` in
+ * `reachy_mini/motion/recorded_move.py`):
+ *   {
+ *     description: string,
+ *     time: number[],                          // seconds, monotonically increasing
+ *     set_target_data: Array<{
+ *       head: number[4][4],                    // 4x4 pose (last row may be [0,0,0,0] in some files)
+ *       antennas: [right_rad, left_rad],
+ *       body_yaw: number,
+ *       check_collision: boolean | null,
+ *     }>
+ *   }
+ *
+ * Two gotchas this module handles:
+ *   1. The datasets in the wild have `[0,0,0,0]` as the last row of `head`
+ *      instead of `[0,0,0,1]`. We clean that up before sending so the wire
+ *      payload is a proper homogeneous matrix.
+ *   2. Rotations must be interpolated on the rotation group (SE(3)), not
+ *      elementwise - otherwise the intermediate matrices aren't orthogonal
+ *      and the IK picks up weird shear. We port the daemon's
+ *      `linear_pose_interpolation` (axis-angle slerp) to TypeScript.
+ */
+
+import type { ReachyMiniInstance } from "./globals.d.ts";
+
+const HF_DATASET_BASE = "https://huggingface.co/datasets";
+const STREAM_HZ = 100;
+
+// ─── Curated catalog ────────────────────────────────────────────────────
+
+/**
+ * A single entry the LLM can pick from. `id` is the opaque identifier used
+ * in tool calls; we map it to a dataset + file stem at playback time.
+ */
+export interface MoveCatalogEntry {
+  readonly id: string;
+  readonly kind: "dance" | "emotion";
+  readonly dataset: string;
+  readonly file: string;
+  readonly description: string;
+}
+
+/**
+ * Curated selection exposed to the model: a handful of dances for
+ * rhythmic/punctuating moments, and a handful of emotions for reactive
+ * body language. Ordered roughly by how often we expect them to be useful.
+ */
+export const MOVE_CATALOG: readonly MoveCatalogEntry[] = [
+  // Dances - rhythmic, expressive, ~1-2s
+  {
+    id: "yeah_nod",
+    kind: "dance",
+    dataset: "pollen-robotics/reachy-mini-dances-library",
+    file: "yeah_nod",
+    description: "A confident, rhythmic nod - agreement, affirmation.",
+  },
+  {
+    id: "uh_huh_tilt",
+    kind: "dance",
+    dataset: "pollen-robotics/reachy-mini-dances-library",
+    file: "uh_huh_tilt",
+    description: "Curious acknowledgement, small sidelong tilt.",
+  },
+  {
+    id: "side_peekaboo",
+    kind: "dance",
+    dataset: "pollen-robotics/reachy-mini-dances-library",
+    file: "side_peekaboo",
+    description: "Playful peek from the side - teasing moments.",
+  },
+  {
+    id: "dizzy_spin",
+    kind: "dance",
+    dataset: "pollen-robotics/reachy-mini-dances-library",
+    file: "dizzy_spin",
+    description: "Dizzy spin - for lighthearted, silly replies.",
+  },
+  {
+    id: "groovy_sway",
+    kind: "dance",
+    dataset: "pollen-robotics/reachy-mini-dances-library",
+    file: "groovy_sway_and_roll",
+    description: "Grooving sway and roll - when the conversation gets musical.",
+  },
+  {
+    id: "chicken_peck",
+    kind: "dance",
+    dataset: "pollen-robotics/reachy-mini-dances-library",
+    file: "chicken_peck",
+    description: "Sharp forward pecks - lively and percussive.",
+  },
+
+  // Emotions - reactive, affective, 1-4s
+  {
+    id: "cheerful",
+    kind: "emotion",
+    dataset: "pollen-robotics/reachy-mini-emotions-library",
+    file: "cheerful1",
+    description: "Bright, upbeat body language - praise, small wins.",
+  },
+  {
+    id: "surprised",
+    kind: "emotion",
+    dataset: "pollen-robotics/reachy-mini-emotions-library",
+    file: "surprised1",
+    description: "Startled, a tiny recoil - when something unexpected happens.",
+  },
+  {
+    id: "curious",
+    kind: "emotion",
+    dataset: "pollen-robotics/reachy-mini-emotions-library",
+    file: "curious1",
+    description: "Inquisitive gaze shift - investigating, asking.",
+  },
+  {
+    id: "amazed",
+    kind: "emotion",
+    dataset: "pollen-robotics/reachy-mini-emotions-library",
+    file: "amazed1",
+    description: "In awe, long upward look - genuine wonder.",
+  },
+  {
+    id: "confused",
+    kind: "emotion",
+    dataset: "pollen-robotics/reachy-mini-emotions-library",
+    file: "confused1",
+    description: "Hesitant head tilt - when something doesn't compute.",
+  },
+  {
+    id: "proud",
+    kind: "emotion",
+    dataset: "pollen-robotics/reachy-mini-emotions-library",
+    file: "proud1",
+    description: "Chin up, chest out - taking credit, feeling good.",
+  },
+  {
+    id: "shy",
+    kind: "emotion",
+    dataset: "pollen-robotics/reachy-mini-emotions-library",
+    file: "shy1",
+    description: "Small look-away - embarrassed, modest.",
+  },
+  {
+    id: "sad",
+    kind: "emotion",
+    dataset: "pollen-robotics/reachy-mini-emotions-library",
+    file: "sad1",
+    description: "Subtle downward look - sympathy, bad news.",
+  },
+];
+
+export type MoveId = (typeof MOVE_CATALOG)[number]["id"];
+
+export const MOVE_IDS: readonly MoveId[] = MOVE_CATALOG.map((m) => m.id);
+
+function findMove(id: string): MoveCatalogEntry | undefined {
+  return MOVE_CATALOG.find((m) => m.id === id);
+}
+
+// ─── File format & loading ──────────────────────────────────────────────
+
+interface RawFrame {
+  head: number[][];
+  antennas: [number, number];
+  body_yaw: number;
+}
+
+interface MoveFile {
+  description: string;
+  time: number[];
+  set_target_data: RawFrame[];
+}
+
+interface LoadedMove {
+  readonly id: MoveId;
+  readonly description: string;
+  readonly duration: number;
+  readonly times: readonly number[];
+  readonly frames: readonly RawFrame[];
+}
+
+// ─── SE(3) interpolation ────────────────────────────────────────────────
+//
+// Ported from `linear_pose_interpolation` in
+// reachy_mini/utils/interpolation.py. Rotations are slerp'd via quaternions;
+// translations are linearly interpolated.
+
+type Quat = readonly [number, number, number, number]; // [x, y, z, w]
+
+/** Extract a unit quaternion from the top-left 3x3 of a 4x4 homogeneous matrix. */
+function quatFromMatrix(m: readonly number[][]): Quat {
+  const m00 = m[0][0], m01 = m[0][1], m02 = m[0][2];
+  const m10 = m[1][0], m11 = m[1][1], m12 = m[1][2];
+  const m20 = m[2][0], m21 = m[2][1], m22 = m[2][2];
+  const trace = m00 + m11 + m22;
+  let x: number, y: number, z: number, w: number;
+  if (trace > 0) {
+    const s = 0.5 / Math.sqrt(trace + 1.0);
+    w = 0.25 / s;
+    x = (m21 - m12) * s;
+    y = (m02 - m20) * s;
+    z = (m10 - m01) * s;
+  } else if (m00 > m11 && m00 > m22) {
+    const s = 2.0 * Math.sqrt(1.0 + m00 - m11 - m22);
+    w = (m21 - m12) / s;
+    x = 0.25 * s;
+    y = (m01 + m10) / s;
+    z = (m02 + m20) / s;
+  } else if (m11 > m22) {
+    const s = 2.0 * Math.sqrt(1.0 + m11 - m00 - m22);
+    w = (m02 - m20) / s;
+    x = (m01 + m10) / s;
+    y = 0.25 * s;
+    z = (m12 + m21) / s;
+  } else {
+    const s = 2.0 * Math.sqrt(1.0 + m22 - m00 - m11);
+    w = (m10 - m01) / s;
+    x = (m02 + m20) / s;
+    y = (m12 + m21) / s;
+    z = 0.25 * s;
+  }
+  const len = Math.sqrt(x * x + y * y + z * z + w * w) || 1;
+  return [x / len, y / len, z / len, w / len];
+}
+
+/** Convert a unit quaternion to a 3x3 rotation matrix (flat row-major). */
+function quatToRot3(q: Quat): number[] {
+  const [x, y, z, w] = q;
+  const xx = x * x, yy = y * y, zz = z * z;
+  const xy = x * y, xz = x * z, yz = y * z;
+  const wx = w * x, wy = w * y, wz = w * z;
+  return [
+    1 - 2 * (yy + zz), 2 * (xy - wz), 2 * (xz + wy),
+    2 * (xy + wz), 1 - 2 * (xx + zz), 2 * (yz - wx),
+    2 * (xz - wy), 2 * (yz + wx), 1 - 2 * (xx + yy),
+  ];
+}
+
+/** Spherical linear interpolation between two unit quaternions. */
+function quatSlerp(a: Quat, b: Quat, t: number): Quat {
+  let [ax, ay, az, aw] = a;
+  let [bx, by, bz, bw] = b;
+  // Pick the shorter arc by flipping b if needed.
+  let dot = ax * bx + ay * by + az * bz + aw * bw;
+  if (dot < 0) {
+    bx = -bx; by = -by; bz = -bz; bw = -bw;
+    dot = -dot;
+  }
+  // Fall back to lerp when quaternions are nearly colinear, to avoid div/0.
+  if (dot > 0.9995) {
+    const x = ax + (bx - ax) * t;
+    const y = ay + (by - ay) * t;
+    const z = az + (bz - az) * t;
+    const w = aw + (bw - aw) * t;
+    const len = Math.sqrt(x * x + y * y + z * z + w * w) || 1;
+    return [x / len, y / len, z / len, w / len];
+  }
+  const theta0 = Math.acos(Math.min(1, Math.max(-1, dot)));
+  const sinTheta0 = Math.sin(theta0);
+  const theta = theta0 * t;
+  const s1 = Math.sin(theta) / sinTheta0;
+  const s0 = Math.cos(theta) - dot * s1;
+  return [
+    ax * s0 + bx * s1,
+    ay * s0 + by * s1,
+    az * s0 + bz * s1,
+    aw * s0 + bw * s1,
+  ];
+}
+
+/**
+ * Interpolate a full 4x4 pose at parameter `t` in [0, 1] between two
+ * recorded frames. Output is a flat 16-float row-major homogeneous matrix
+ * with a well-formed `[0,0,0,1]` last row.
+ */
+function interpolateHeadFlat(
+  a: readonly number[][],
+  b: readonly number[][],
+  t: number,
+): number[] {
+  const rot = quatToRot3(quatSlerp(quatFromMatrix(a), quatFromMatrix(b), t));
+  // Translation is always 0 in the recorded dances, but we still lerp in
+  // case a future dataset uses it.
+  const tx = a[0][3] + (b[0][3] - a[0][3]) * t;
+  const ty = a[1][3] + (b[1][3] - a[1][3]) * t;
+  const tz = a[2][3] + (b[2][3] - a[2][3]) * t;
+  return [
+    rot[0], rot[1], rot[2], tx,
+    rot[3], rot[4], rot[5], ty,
+    rot[6], rot[7], rot[8], tz,
+    0, 0, 0, 1,
+  ];
+}
+
+/** Same cleanup applied to a frame served as-is (no interpolation). */
+function headFlatFromFrame(m: readonly number[][]): number[] {
+  // Re-normalize the rotation block via quat roundtrip - this also scrubs
+  // the degenerate `[0,0,0,0]` last row found in the dataset.
+  const rot = quatToRot3(quatFromMatrix(m));
+  return [
+    rot[0], rot[1], rot[2], m[0][3] ?? 0,
+    rot[3], rot[4], rot[5], m[1][3] ?? 0,
+    rot[6], rot[7], rot[8], m[2][3] ?? 0,
+    0, 0, 0, 1,
+  ];
+}
+
+// ─── Player ─────────────────────────────────────────────────────────────
+
+/**
+ * Streams moves to the robot one at a time. Load-once / play-many: fetched
+ * JSON is cached in memory for the session.
+ */
+export class MovePlayer {
+  private readonly robot: ReachyMiniInstance;
+  private readonly cache = new Map<string, LoadedMove>();
+  private timer: number | null = null;
+  private current: { move: LoadedMove; t0: number } | null = null;
+  private onFinish: (() => void) | null = null;
+
+  constructor(robot: ReachyMiniInstance) {
+    this.robot = robot;
+  }
+
+  /** True while a move is actively streaming. */
+  get isPlaying(): boolean {
+    return this.current !== null;
+  }
+
+  /**
+   * Fetch (or hit cache) the trajectory JSON and return the parsed move.
+   * Safe to call ahead of time to prewarm a move that's likely to play.
+   */
+  async load(id: MoveId): Promise<LoadedMove> {
+    const cached = this.cache.get(id);
+    if (cached) return cached;
+
+    const entry = findMove(id);
+    if (!entry) throw new Error(`Unknown move id '${id}'`);
+
+    const url = `${HF_DATASET_BASE}/${entry.dataset}/resolve/main/${encodeURIComponent(entry.file)}.json`;
+    const response = await fetch(url);
+    if (!response.ok) {
+      throw new Error(
+        `Failed to fetch move '${id}' from ${entry.dataset}: ${response.status} ${response.statusText}`,
+      );
+    }
+    const raw = (await response.json()) as MoveFile;
+    if (!Array.isArray(raw.time) || !Array.isArray(raw.set_target_data)) {
+      throw new Error(`Malformed move file '${id}'`);
+    }
+    if (raw.time.length !== raw.set_target_data.length) {
+      throw new Error(`Move '${id}' has mismatched time/frame lengths`);
+    }
+
+    const move: LoadedMove = {
+      id,
+      description: raw.description ?? entry.description,
+      times: raw.time.slice(),
+      frames: raw.set_target_data,
+      duration: raw.time[raw.time.length - 1] ?? 0,
+    };
+    this.cache.set(id, move);
+    return move;
+  }
+
+  /**
+   * Play a move. If another move is already streaming it is cancelled
+   * first. Resolves when the move finishes (or is cancelled via `stop()`).
+   */
+  async play(id: MoveId): Promise<void> {
+    const move = await this.load(id);
+    this.stop();
+
+    return new Promise<void>((resolve) => {
+      this.onFinish = resolve;
+      this.current = { move, t0: performance.now() / 1000 };
+      this.timer = window.setInterval(() => this.tick(), 1000 / STREAM_HZ);
+    });
+  }
+
+  /** Abort playback immediately. Safe to call when idle. */
+  stop(): void {
+    if (this.timer !== null) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    const cb = this.onFinish;
+    this.current = null;
+    this.onFinish = null;
+    if (cb) cb();
+  }
+
+  // ─── internals ──────────────────────────────────────────────────────
+
+  private tick(): void {
+    if (!this.current) return;
+    const { move, t0 } = this.current;
+    const t = performance.now() / 1000 - t0;
+
+    if (t >= move.duration) {
+      // Snap to the final frame for a clean landing before stopping.
+      const last = move.frames[move.frames.length - 1];
+      this.sendFrame(headFlatFromFrame(last.head), last.antennas, last.body_yaw);
+      this.stop();
+      return;
+    }
+
+    const sample = sampleAt(move, t);
+    this.sendFrame(sample.head, sample.antennas, sample.body_yaw);
+  }
+
+  private sendFrame(
+    headFlat: number[],
+    antennas: readonly [number, number],
+    bodyYaw: number,
+  ): void {
+    this.robot.sendRaw({
+      type: "set_full_target",
+      head: headFlat,
+      antennas: [antennas[0], antennas[1]],
+      body_yaw: bodyYaw,
+    });
+  }
+}
+
+// ─── Sampling ───────────────────────────────────────────────────────────
+
+interface SampledFrame {
+  head: number[]; // flat row-major 4x4
+  antennas: [number, number];
+  body_yaw: number;
+}
+
+function sampleAt(move: LoadedMove, t: number): SampledFrame {
+  const { times, frames } = move;
+  const n = times.length;
+  if (n === 0) {
+    return { head: identity4Flat(), antennas: [0, 0], body_yaw: 0 };
+  }
+  if (t <= times[0]) {
+    return {
+      head: headFlatFromFrame(frames[0].head),
+      antennas: frames[0].antennas,
+      body_yaw: frames[0].body_yaw,
+    };
+  }
+  if (t >= times[n - 1]) {
+    const last = frames[n - 1];
+    return {
+      head: headFlatFromFrame(last.head),
+      antennas: last.antennas,
+      body_yaw: last.body_yaw,
+    };
+  }
+
+  // Binary search for the bracket [i, i+1] with times[i] <= t < times[i+1].
+  let lo = 0;
+  let hi = n - 1;
+  while (lo + 1 < hi) {
+    const mid = (lo + hi) >> 1;
+    if (times[mid] <= t) lo = mid;
+    else hi = mid;
+  }
+  const ta = times[lo];
+  const tb = times[lo + 1];
+  const alpha = tb > ta ? (t - ta) / (tb - ta) : 0;
+
+  const a = frames[lo];
+  const b = frames[lo + 1];
+  return {
+    head: interpolateHeadFlat(a.head, b.head, alpha),
+    antennas: [
+      a.antennas[0] + (b.antennas[0] - a.antennas[0]) * alpha,
+      a.antennas[1] + (b.antennas[1] - a.antennas[1]) * alpha,
+    ],
+    body_yaw: a.body_yaw + (b.body_yaw - a.body_yaw) * alpha,
+  };
+}
+
+function identity4Flat(): number[] {
+  return [
+    1, 0, 0, 0,
+    0, 1, 0, 0,
+    0, 0, 1, 0,
+    0, 0, 0, 1,
+  ];
+}

src/openai-realtime.ts

@@ -0,0 +1,611 @@
+/**
+ * Minimal OpenAI Realtime WebRTC client.
+ *
+ * We let the caller provide the input audio `MediaStreamTrack` (the robot's
+ * microphone, received on a separate RTCPeerConnection) and expose the remote
+ * track produced by OpenAI (synthesized voice) via the `outputTrack` event so
+ * it can be routed to the robot's speakers with `sender.replaceTrack()`.
+ *
+ * References:
+ *   https://platform.openai.com/docs/guides/realtime-webrtc
+ *   https://platform.openai.com/docs/api-reference/realtime-client-events
+ *   https://platform.openai.com/docs/api-reference/realtime-server-events
+ */
+
+// OpenAI blocks direct browser calls to their Realtime endpoints (no CORS
+// headers for user API keys). We therefore always hit a same-origin path
+// (`/openai/v1/realtime`) which is forwarded to `api.openai.com` by:
+//   - Vite dev server (see vite.config.ts `server.proxy`) during `npm run dev`
+//   - nginx on HF Spaces (see nginx.conf `location /openai/`) in production.
+// The Bearer token travels through the proxy untouched.
+const REALTIME_BASE_URL = "/openai/v1/realtime";
+
+// ─── WiFi robustness knobs ──────────────────────────────────────────────
+// Peak bitrate we allow Opus to use on our uplink. 32 kbps is plenty for
+// clean voice; capping avoids saturating a weak WiFi AP which causes
+// bursty packet loss.
+const MAX_AUDIO_BITRATE_BPS = 32_000;
+// Hint to the browser's jitter buffer: we'd rather play audio ~150 ms
+// late than chop it up on jitter spikes. Voice assistants are forgiving
+// of a bit of latency.
+const PLAYOUT_DELAY_HINT_S = 0.15;
+// Opus fmtp knobs forced via SDP munging. FEC inline gives us ~5-15%
+// loss resilience "for free", DTX off keeps the comfort-noise packets
+// arriving continuously so our own VAD/wobbler never thinks the line
+// went silent.
+const OPUS_FMTP_OVERRIDES: Record<string, string> = {
+  minptime: "10",
+  useinbandfec: "1",
+  usedtx: "0",
+  stereo: "0",
+  "sprop-stereo": "0",
+  maxaveragebitrate: String(MAX_AUDIO_BITRATE_BPS),
+};
+// How long we tolerate an ICE `disconnected` state before giving up.
+// WiFi blips often self-heal within ~2 s; we budget a bit more than that.
+const ICE_DISCONNECT_GRACE_MS = 5_000;
+
+export type RealtimeStatus =
+  | "idle"
+  | "connecting"
+  | "connected"
+  | "user-speaking"
+  | "processing"
+  | "ai-speaking"
+  | "closed"
+  | "error";
+
+/**
+ * OpenAI Realtime tool definition (a.k.a. function calling). We only expose
+ * the subset of fields we actually populate; the API accepts more.
+ */
+export interface RealtimeTool {
+  name: string;
+  description: string;
+  parameters: Record<string, unknown>;
+}
+
+export interface RealtimeToolCall {
+  callId: string;
+  name: string;
+  arguments: Record<string, unknown>;
+}
+
+export interface RealtimeOptions {
+  apiKey: string;
+  model: string;
+  voice: string;
+  instructions: string;
+  inputTrack: MediaStreamTrack;
+  /** Optional list of tools exposed to the model. */
+  tools?: RealtimeTool[];
+}
+
+type EventMap = {
+  status: { status: RealtimeStatus };
+  outputTrack: { track: MediaStreamTrack };
+  transcript: { role: "user" | "assistant"; text: string; partial: boolean };
+  toolCall: RealtimeToolCall;
+  error: { error: unknown };
+};
+
+type Listener<K extends keyof EventMap> = (detail: EventMap[K]) => void;
+
+export class OpenaiRealtimeClient {
+  private pc: RTCPeerConnection | null = null;
+  private dc: RTCDataChannel | null = null;
+  private listeners: { [K in keyof EventMap]?: Set<Listener<K>> } = {};
+  private status: RealtimeStatus = "idle";
+  // Debounces transient ICE disconnects so a brief WiFi blip doesn't
+  // bubble up as an error all the way to the UI.
+  private iceGraceTimer: number | null = null;
+  // One-shot visibilitychange listener used when we disconnect while the
+  // tab is hidden: we re-arm the grace timer only once the tab is active
+  // so throttled timers can't cause spurious errors.
+  private pendingVisibilityHandler: (() => void) | null = null;
+
+  readonly options: RealtimeOptions;
+
+  constructor(options: RealtimeOptions) {
+    this.options = options;
+  }
+
+  on<K extends keyof EventMap>(event: K, listener: Listener<K>): () => void {
+    let set = this.listeners[event] as Set<Listener<K>> | undefined;
+    if (!set) {
+      set = new Set<Listener<K>>();
+      (this.listeners as Record<string, Set<Listener<K>>>)[event] = set;
+    }
+    set.add(listener);
+    return () => set?.delete(listener);
+  }
+
+  private emit<K extends keyof EventMap>(event: K, detail: EventMap[K]): void {
+    const set = this.listeners[event] as Set<Listener<K>> | undefined;
+    if (!set) return;
+    for (const listener of set) {
+      try {
+        listener(detail);
+      } catch (err) {
+        console.error("[openai-realtime] listener error:", err);
+      }
+    }
+  }
+
+  private setStatus(status: RealtimeStatus): void {
+    if (this.status === status) return;
+    this.status = status;
+    this.emit("status", { status });
+  }
+
+  /**
+   * Bump the state to "ai-speaking" from any pre-speaking phase. Called
+   * from whichever server event lands first to flag that Reachy has
+   * started talking. Deliberately permissive (processing → ai-speaking
+   * AND connected → ai-speaking) because depending on the transport we
+   * might skip the intermediate processing phase entirely.
+   */
+  private markAudible(): void {
+    if (this.status === "ai-speaking") return;
+    if (this.status === "closed" || this.status === "error") return;
+    this.setStatus("ai-speaking");
+  }
+
+  /**
+   * Establish the WebRTC session with OpenAI. Resolves once the SDP exchange
+   * completes; the data channel may still be opening asynchronously.
+   */
+  async connect(): Promise<void> {
+    if (this.pc) throw new Error("Already connected");
+    this.setStatus("connecting");
+
+    const pc = new RTCPeerConnection({
+      iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
+    });
+    this.pc = pc;
+
+    pc.addTransceiver(this.options.inputTrack, {
+      direction: "sendrecv",
+      streams: [new MediaStream([this.options.inputTrack])],
+    });
+
+    pc.ontrack = (event) => {
+      const [stream] = event.streams;
+      const track = event.track.kind === "audio" ? event.track : stream?.getAudioTracks()[0];
+      if (track) {
+        // Ask the browser to buffer a bit of playout. This costs latency
+        // but trades it for resilience on jittery WiFi links.
+        try {
+          // `playoutDelayHint` is a non-standard but widely-shipped property.
+          (event.receiver as RTCRtpReceiver & { playoutDelayHint?: number }).playoutDelayHint =
+            PLAYOUT_DELAY_HINT_S;
+        } catch {
+          // Not fatal - older browsers just ignore the hint.
+        }
+        this.emit("outputTrack", { track });
+      }
+    };
+
+    pc.oniceconnectionstatechange = () => {
+      const s = pc.iceConnectionState;
+      if (s === "connected" || s === "completed") {
+        // We recovered before the grace timer fired - good.
+        this.clearIceGrace();
+        return;
+      }
+      if (s === "failed") {
+        this.clearIceGrace();
+        this.emit("error", { error: new Error("ICE failed") });
+        this.setStatus("error");
+        return;
+      }
+      if (s === "disconnected") {
+        // Wait a bit - WebRTC flips to `disconnected` on WiFi blips and
+        // self-heals once ICE finds packets again. Only escalate if it
+        // stays that way past the grace window.
+        //
+        // Background tabs: browsers throttle JS timers heavily so a 5 s
+        // setTimeout can fire minutes late, AND the native WebRTC stack
+        // keeps running so the connection often recovers by then. We
+        // pause the grace timer while the tab is hidden and wait for
+        // visibilitychange before deciding anything.
+        if (this.iceGraceTimer !== null) return;
+        if (typeof document !== "undefined" && document.hidden) {
+          console.warn(
+            "[openai-realtime] ICE disconnected while tab hidden — deferring grace timer until visibility returns.",
+          );
+          this.armIceGraceOnVisibility();
+          return;
+        }
+        this.scheduleIceGrace();
+      }
+    };
+
+    const dc = pc.createDataChannel("oai-events");
+    this.dc = dc;
+
+    dc.addEventListener("open", () => {
+      this.setStatus("connected");
+      const tools = (this.options.tools ?? []).map((t) => ({
+        type: "function" as const,
+        name: t.name,
+        description: t.description,
+        parameters: t.parameters,
+      }));
+      this.sendEvent({
+        type: "session.update",
+        session: {
+          modalities: ["audio", "text"],
+          voice: this.options.voice,
+          instructions: this.options.instructions,
+          input_audio_format: "pcm16",
+          output_audio_format: "pcm16",
+          input_audio_transcription: { model: "whisper-1" },
+          turn_detection: {
+            type: "server_vad",
+            threshold: 0.5,
+            prefix_padding_ms: 300,
+            silence_duration_ms: 500,
+            create_response: true,
+            interrupt_response: true,
+          },
+          ...(tools.length ? { tools, tool_choice: "auto" } : {}),
+        },
+      });
+    });
+
+    dc.addEventListener("message", (event) => this.handleEvent(event.data));
+    dc.addEventListener("error", (event) => {
+      console.error("[openai-realtime] data channel error:", event);
+    });
+
+    const rawOffer = await pc.createOffer();
+    // Patch the offer SDP to force Opus FEC + a sane maxaveragebitrate.
+    // This tells OpenAI how to decode our uplink AND advertises our
+    // decoder's capabilities for the answer it sends back.
+    const offerSdp = patchOpusFmtp(rawOffer.sdp ?? "");
+    await pc.setLocalDescription({ type: "offer", sdp: offerSdp });
+
+    // Enforce the bitrate cap on our sender too - many impls ignore the
+    // SDP `maxaveragebitrate` without this.
+    capSenderBitrate(pc, MAX_AUDIO_BITRATE_BPS).catch((err) => {
+      console.warn("[openai-realtime] failed to cap sender bitrate:", err);
+    });
+
+    const url = `${REALTIME_BASE_URL}?model=${encodeURIComponent(this.options.model)}`;
+    const response = await fetch(url, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${this.options.apiKey}`,
+        "Content-Type": "application/sdp",
+      },
+      body: offerSdp,
+    });
+
+    if (!response.ok) {
+      const text = await response.text().catch(() => "");
+      throw new Error(`OpenAI Realtime handshake failed (${response.status}): ${text}`);
+    }
+
+    const rawAnswer = await response.text();
+    // Patch the answer too so our *decoder* honours the same FEC knobs.
+    const answerSdp = patchOpusFmtp(rawAnswer);
+    await pc.setRemoteDescription({ type: "answer", sdp: answerSdp });
+  }
+
+  private clearIceGrace(): void {
+    if (this.iceGraceTimer !== null) {
+      clearTimeout(this.iceGraceTimer);
+      this.iceGraceTimer = null;
+    }
+    if (this.pendingVisibilityHandler) {
+      document.removeEventListener("visibilitychange", this.pendingVisibilityHandler);
+      this.pendingVisibilityHandler = null;
+    }
+  }
+
+  /**
+   * Arm the ICE grace timer proper. Safe to call once we know the tab
+   * is in the foreground (timers aren't throttled).
+   */
+  private scheduleIceGrace(): void {
+    if (this.iceGraceTimer !== null) return;
+    console.warn(
+      `[openai-realtime] ICE disconnected, waiting ${ICE_DISCONNECT_GRACE_MS}ms before giving up`,
+    );
+    this.iceGraceTimer = window.setTimeout(() => {
+      this.iceGraceTimer = null;
+      if (!this.pc) return;
+      const still = this.pc.iceConnectionState;
+      if (still === "disconnected" || still === "failed") {
+        this.emit("error", {
+          error: new Error(`ICE stuck in '${still}' for > ${ICE_DISCONNECT_GRACE_MS}ms`),
+        });
+        this.setStatus("error");
+      }
+    }, ICE_DISCONNECT_GRACE_MS);
+  }
+
+  /**
+   * We're disconnected AND the tab is hidden. Wait for visibility to
+   * return before deciding: the connection often self-heals while we're
+   * in the background, and JS timers there fire unpredictably late.
+   */
+  private armIceGraceOnVisibility(): void {
+    if (this.pendingVisibilityHandler) return;
+    const handler = () => {
+      if (document.hidden) return;
+      document.removeEventListener("visibilitychange", handler);
+      this.pendingVisibilityHandler = null;
+      if (!this.pc) return;
+      const s = this.pc.iceConnectionState;
+      if (s === "connected" || s === "completed") return; // healed itself
+      if (s === "failed") {
+        this.emit("error", { error: new Error("ICE failed") });
+        this.setStatus("error");
+        return;
+      }
+      // Still disconnected when we came back - now give it the normal
+      // grace window with a foreground timer that'll actually fire.
+      this.scheduleIceGrace();
+    };
+    document.addEventListener("visibilitychange", handler);
+    this.pendingVisibilityHandler = handler;
+  }
+
+  private handleEvent(raw: string): void {
+    let event: { type?: string; [key: string]: unknown };
+    try {
+      event = JSON.parse(raw);
+    } catch {
+      return;
+    }
+
+    // Lightweight trace of every server event. Makes it trivial to see
+    // in DevTools whether the state-triggering events (speech_started,
+    // output_audio.delta, audio_transcript.delta, response.done, ...)
+    // actually reach the data channel for the current transport.
+    if (typeof event.type === "string") {
+      console.debug("[openai-realtime] event:", event.type);
+    }
+
+    switch (event.type) {
+      case "input_audio_buffer.speech_started":
+        this.setStatus("user-speaking");
+        break;
+
+      case "input_audio_buffer.speech_stopped":
+        // User stopped talking; the model is computing its reply and will
+        // start audio shortly. We surface this as `processing` so the UI
+        // can show a "thinking" visual until audio actually arrives.
+        if (this.status === "user-speaking") {
+          this.setStatus("processing");
+        }
+        break;
+
+      // ─── "Assistant is speaking" triggers ─────────────────────────
+      //
+      // Multiple events can mark the start of an audible reply, and
+      // which ones actually fire depends on the transport:
+      //
+      //   - WebSocket: the audio itself streams as `response.output_audio.delta`
+      //     events on the same channel.
+      //   - WebRTC: the audio goes through the media track, not the
+      //     data channel, so `output_audio.delta` frames may be sparse
+      //     or missing entirely. But `output_audio_transcript.delta`
+      //     and `content_part.added` with an output_audio part DO
+      //     fire reliably over the data channel, so we listen to all
+      //     of them and whichever lands first flips us to ai-speaking.
+      //
+      // Transcript deltas are handled in their own case below (they
+      // also need to emit a `transcript` event), so we duplicate the
+      // `markAudible()` call there rather than listing them here.
+      case "response.audio.delta":
+      case "response.output_audio.delta":
+        this.markAudible();
+        break;
+
+      case "response.content_part.added": {
+        const part = event.part as { type?: string } | undefined;
+        if (part?.type === "audio" || part?.type === "output_audio") {
+          this.markAudible();
+        }
+        break;
+      }
+
+      case "response.created":
+      case "response.output_item.added":
+        // No audio yet, just metadata; keep us in `processing` until
+        // we get a clearer "speaking" signal above.
+        if (this.status === "connected" || this.status === "user-speaking") {
+          this.setStatus("processing");
+        }
+        break;
+
+      case "response.done":
+      case "response.cancelled":
+        if (this.status === "ai-speaking" || this.status === "processing") {
+          this.setStatus("connected");
+        }
+        break;
+
+      case "conversation.item.input_audio_transcription.delta": {
+        const delta = typeof event.delta === "string" ? event.delta : "";
+        if (delta) {
+          this.emit("transcript", { role: "user", text: delta, partial: true });
+        }
+        break;
+      }
+
+      case "conversation.item.input_audio_transcription.completed": {
+        const transcript = typeof event.transcript === "string" ? event.transcript : "";
+        if (transcript) {
+          this.emit("transcript", { role: "user", text: transcript, partial: false });
+        }
+        break;
+      }
+
+      case "response.audio_transcript.delta":
+      case "response.output_audio_transcript.delta": {
+        // Transcript deltas fire continuously while the model streams
+        // audio. Over WebRTC they're our most reliable "Reachy is
+        // actually talking right now" signal because `output_audio.delta`
+        // metadata events don't always reach the data channel.
+        this.markAudible();
+        const delta = typeof event.delta === "string" ? event.delta : "";
+        if (delta) {
+          this.emit("transcript", { role: "assistant", text: delta, partial: true });
+        }
+        break;
+      }
+
+      case "response.audio_transcript.done":
+      case "response.output_audio_transcript.done": {
+        const transcript = typeof event.transcript === "string" ? event.transcript : "";
+        if (transcript) {
+          this.emit("transcript", { role: "assistant", text: transcript, partial: false });
+        }
+        break;
+      }
+
+      case "response.function_call_arguments.done": {
+        // The model finished streaming tool arguments; execute & reply.
+        const callId = typeof event.call_id === "string" ? event.call_id : "";
+        const name = typeof event.name === "string" ? event.name : "";
+        const argsRaw = typeof event.arguments === "string" ? event.arguments : "{}";
+        let args: Record<string, unknown> = {};
+        try {
+          args = JSON.parse(argsRaw);
+        } catch {
+          args = {};
+        }
+        if (callId && name) {
+          this.emit("toolCall", { callId, name, arguments: args });
+        }
+        break;
+      }
+
+      case "error": {
+        const err = event.error as { message?: string } | undefined;
+        this.emit("error", { error: new Error(err?.message ?? "OpenAI error") });
+        break;
+      }
+    }
+  }
+
+  private sendEvent(event: Record<string, unknown>): void {
+    if (!this.dc || this.dc.readyState !== "open") return;
+    this.dc.send(JSON.stringify(event));
+  }
+
+  /**
+   * Send the result of a previously received `toolCall` back to the model
+   * so it can continue the conversation. `output` will be JSON-stringified
+   * if it isn't a string already.
+   */
+  sendToolResponse(callId: string, output: unknown): void {
+    const outputStr = typeof output === "string" ? output : JSON.stringify(output);
+    this.sendEvent({
+      type: "conversation.item.create",
+      item: {
+        type: "function_call_output",
+        call_id: callId,
+        output: outputStr,
+      },
+    });
+    // Ask the model to pick up from here and speak the reply.
+    this.sendEvent({ type: "response.create" });
+  }
+
+  async close(): Promise<void> {
+    this.clearIceGrace();
+
+    try {
+      this.dc?.close();
+    } catch {
+      // ignored
+    }
+    this.dc = null;
+
+    try {
+      this.pc?.close();
+    } catch {
+      // ignored
+    }
+    this.pc = null;
+
+    this.setStatus("closed");
+  }
+}
+
+// ─── Helpers ────────────────────────────────────────────────────────────
+
+/**
+ * Rewrite all `a=fmtp:<pt> ...` lines for Opus payload types so that they
+ * include our required parameters (FEC on, DTX off, bitrate cap). Missing
+ * fmtp lines are synthesized next to the matching `a=rtpmap` line.
+ */
+function patchOpusFmtp(sdp: string): string {
+  if (!sdp) return sdp;
+
+  // Find every Opus payload type via its rtpmap entry.
+  const rtpmapRe = /^a=rtpmap:(\d+)\s+opus\/48000/gim;
+  const payloadTypes: string[] = [];
+  let m: RegExpExecArray | null;
+  while ((m = rtpmapRe.exec(sdp)) !== null) {
+    payloadTypes.push(m[1]);
+  }
+  if (payloadTypes.length === 0) return sdp;
+
+  let patched = sdp;
+  for (const pt of payloadTypes) {
+    const fmtpLineRe = new RegExp(`^a=fmtp:${pt}\\s+([^\\r\\n]*)`, "m");
+    const existing = fmtpLineRe.exec(patched);
+    if (existing) {
+      const merged = mergeFmtpParams(existing[1], OPUS_FMTP_OVERRIDES);
+      patched = patched.replace(fmtpLineRe, `a=fmtp:${pt} ${merged}`);
+    } else {
+      const merged = mergeFmtpParams("", OPUS_FMTP_OVERRIDES);
+      // Insert the new fmtp line right after the matching rtpmap.
+      const rtpmapLineRe = new RegExp(`(^a=rtpmap:${pt}\\s+opus/48000[^\\r\\n]*)`, "m");
+      patched = patched.replace(rtpmapLineRe, `$1\r\na=fmtp:${pt} ${merged}`);
+    }
+  }
+  return patched;
+}
+
+function mergeFmtpParams(existing: string, overrides: Record<string, string>): string {
+  const params = new Map<string, string>();
+  for (const chunk of existing.split(";")) {
+    const kv = chunk.trim();
+    if (!kv) continue;
+    const eq = kv.indexOf("=");
+    if (eq < 0) params.set(kv, "");
+    else params.set(kv.slice(0, eq).trim(), kv.slice(eq + 1).trim());
+  }
+  for (const [k, v] of Object.entries(overrides)) {
+    params.set(k, v);
+  }
+  return Array.from(params.entries())
+    .map(([k, v]) => (v ? `${k}=${v}` : k))
+    .join(";");
+}
+
+/**
+ * Cap the outbound audio sender's bitrate. The spec recommends setting
+ * `maxBitrate` on the first (and typically only) encoding.
+ */
+async function capSenderBitrate(pc: RTCPeerConnection, maxBitrate: number): Promise<void> {
+  const sender = pc
+    .getSenders()
+    .find((s) => s.track && s.track.kind === "audio");
+  if (!sender) return;
+  const params = sender.getParameters();
+  if (!params.encodings || params.encodings.length === 0) {
+    params.encodings = [{}];
+  }
+  for (const enc of params.encodings) {
+    enc.maxBitrate = maxBitrate;
+  }
+  await sender.setParameters(params);
+}

src/robot-tools.ts

@@ -0,0 +1,71 @@
+// src/robot-tools.ts — extracted from the reference app's main.ts
+import {
+  MovePlayer,
+  MOVE_CATALOG,
+  MOVE_IDS,
+  type MoveId,
+} from "./move-player.js";
+import type { ReachyMiniInstance } from "./globals.d.ts";
+import type { RealtimeTool } from "./openai-realtime.js";
+
+const HEAD_POSES = {
+  center: { roll: 0, pitch: 0, yaw: 0 },
+  up: { roll: 0, pitch: -18, yaw: 0 },
+  down: { roll: 0, pitch: 18, yaw: 0 },
+  left: { roll: 0, pitch: 0, yaw: 25 },
+  right: { roll: 0, pitch: 0, yaw: -25 },
+  tilt_left: { roll: -15, pitch: 0, yaw: 0 },
+  tilt_right: { roll: 15, pitch: 0, yaw: 0 },
+} as const;
+
+export const ROBOT_TOOLS: RealtimeTool[] = [
+  {
+    name: "move_head",
+    description:
+      "Point the robot's head in a named direction. Use to accompany speech with a tiny gesture.",
+    parameters: {
+      type: "object",
+      properties: {
+        direction: { type: "string", enum: Object.keys(HEAD_POSES),
+                     description: "Named head pose to assume." },
+      },
+      required: ["direction"],
+    },
+  },
+  {
+    name: "play_move",
+    description:
+      "Trigger a short pre-recorded body-language move (1-4s) from the Reachy library.\n" +
+      MOVE_CATALOG.map((m) => `  - ${m.id} | ${m.kind} | ${m.description}`).join("\n"),
+    parameters: {
+      type: "object",
+      properties: {
+        name: { type: "string", enum: [...MOVE_IDS],
+                description: "Catalog id." },
+      },
+      required: ["name"],
+    },
+  },
+];
+
+export class RobotToolDispatcher {
+  private movePlayer: MovePlayer;
+  constructor(robot: ReachyMiniInstance) {
+    this.movePlayer = new MovePlayer(robot);
+  }
+  async handle(robot: ReachyMiniInstance, name: string, args: Record<string, unknown>): Promise<string> {
+    if (name === "move_head") {
+      const dir = args.direction as keyof typeof HEAD_POSES;
+      const pose = HEAD_POSES[dir];
+      if (!pose) return `Unknown direction: ${args.direction}`;
+      robot.setHeadPose(pose.roll, pose.pitch, pose.yaw);
+      return `Head moved to ${dir}.`;
+    }
+    if (name === "play_move") {
+      const moveId = args.name as MoveId;
+      await this.movePlayer.play(moveId);
+      return `Played move: ${moveId}.`;
+    }
+    return `Unknown robot tool: ${name}`;
+  }
+}

tsconfig.app.json

@@ -2,7 +2,8 @@
   "extends": "./tsconfig.base.json",
   "compilerOptions": {
     "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "types": []
+    "types": [],
+    "noUncheckedIndexedAccess": false
   },
   "include": ["src/**/*", "index.html"]
 }