import { StoryEngine, type TransitionResult, type LoadedContent } from "story-engine"; import { OpenaiRealtimeClient, type RealtimeTool } from "./openai-realtime.js"; import { SummaryBuilder } from "./summary-builder.js"; import { ROBOT_TOOLS, RobotToolDispatcher } from "./robot-tools.js"; import { toRealtimeTools, isStoryTool } from "./story-tools.js"; import type { ReachyMiniInstance } from "./globals.d.ts"; export interface StoryBridgeOptions { content: LoadedContent; robot: ReachyMiniInstance; apiKey: string; realtimeModel: string; summaryModel: string; defaultVoice: string; getRobotMicTrack: () => MediaStreamTrack | null; onOpenAiOutputTrack: (track: MediaStreamTrack) => void; onAppStateChange: (state: BridgeAppState) => void; } export type BridgeAppState = | "starting" | "listening" | "user-speaking" | "processing" | "ai-speaking" | "swapping-character" | "swapping-story" | "error"; export class StoryBridge { private engine: StoryEngine; private realtime: OpenaiRealtimeClient | null = null; private summary: SummaryBuilder; private robotDispatcher: RobotToolDispatcher; private currentVoice: string | null = null; private opts: StoryBridgeOptions; constructor(opts: StoryBridgeOptions) { this.opts = opts; this.engine = new StoryEngine(opts.content); this.summary = new SummaryBuilder({ apiKey: opts.apiKey, model: opts.summaryModel }); this.robotDispatcher = new RobotToolDispatcher(opts.robot); } async start(): Promise { this.opts.onAppStateChange("starting"); const narrator = this.engine.getCurrentCharacter(); const voice = narrator?.voice ?? this.opts.defaultVoice; await this.openSession(voice); this.opts.onAppStateChange("listening"); } async stop(): Promise { if (this.realtime) { await this.realtime.close(); this.realtime = null; } } private buildTools(): RealtimeTool[] { return [...toRealtimeTools(this.engine.buildToolDefinitions()), ...ROBOT_TOOLS]; } private attachListeners(client: OpenaiRealtimeClient): void { client.on("status", ({ status }) => this.translateRealtimeState(status)); client.on("outputTrack", ({ track }) => this.opts.onOpenAiOutputTrack(track)); client.on("toolCall", ({ callId, name, arguments: args }) => { this.handleToolCall(name, args) .then((result) => client.sendToolResponse(callId, result)) .catch((err) => { console.error("[story-bridge] tool call error:", err); client.sendToolResponse(callId, `Error: ${String(err)}`); }); }); client.on("error", ({ error }) => { console.error("[story-bridge] realtime error:", error); this.opts.onAppStateChange("error"); }); } private async openSession(voice: string): Promise { const inputTrack = this.opts.getRobotMicTrack(); if (!inputTrack) throw new Error("No mic track available"); this.realtime = new OpenaiRealtimeClient({ apiKey: this.opts.apiKey, model: this.opts.realtimeModel, voice, instructions: this.engine.buildSystemPrompt(), tools: this.buildTools(), inputTrack, }); this.attachListeners(this.realtime); await this.realtime.connect(); this.currentVoice = voice; } private translateRealtimeState(rtState: string): void { switch (rtState) { case "connected": this.opts.onAppStateChange("listening"); break; case "user-speaking": case "processing": case "ai-speaking": this.opts.onAppStateChange(rtState as BridgeAppState); break; case "error": this.opts.onAppStateChange("error"); break; } } async handleToolCall(name: string, args: Record): Promise { if (!isStoryTool(name)) { return this.robotDispatcher.handle(this.opts.robot, name, args); } let result: TransitionResult; switch (name) { case "select_story": result = this.engine.enterStoryMode(args.storyId as string); return this.applyTransition(result, "swapping-story"); case "advance_storyline": result = this.engine.advanceStoryline(args.actionId as string); return this.applyTransition(result); case "navigate_scene": result = this.engine.navigateScene(args.sceneId as string); return this.applyTransition(result); case "perform_scene_action": result = this.engine.performSceneAction(args.actionId as string); return this.applyTransition(result); case "request_options": return this.engine.formatOptionsRecap(); case "exit_story": result = this.engine.exitStoryMode(); return this.applyTransition(result, "swapping-story"); default: return `Unknown tool: ${name}`; } } private async applyTransition( result: TransitionResult, transientState: "swapping-character" | "swapping-story" = "swapping-character", ): Promise { const blocked = result.events.find((e) => e.type === "blocked"); if (blocked) return `Cannot do that: ${(blocked as Extract).reason}`; const speakerChange = result.events.find((e) => e.type === "speakerChanged") as | Extract | undefined; if (speakerChange) { const newVoice = speakerChange.newCharacter.voice ?? this.opts.defaultVoice; if (newVoice !== this.currentVoice) { this.opts.onAppStateChange(transientState); await this.swapSession(newVoice); this.opts.onAppStateChange("ai-speaking"); return ""; } } // Same voice — session already has current state; no updateSession available, // so we close and reopen to push fresh instructions + tools. await this.swapSession(this.currentVoice ?? this.opts.defaultVoice); return this.engine.getCurrentNode()?.text ?? ""; } private async swapSession(newVoice: string): Promise { let summaryText = "(no summary)"; try { summaryText = await this.summary.summarize(this.engine.getRecentTransitionLog()); } catch (e) { console.warn("[story-bridge] Summary failed, continuing with no summary:", e); } if (this.realtime) { await this.realtime.close(); this.realtime = null; } const inputTrack = this.opts.getRobotMicTrack(); if (!inputTrack) { console.error("[story-bridge] No mic track available for new session"); this.opts.onAppStateChange("error"); return; } const instructions = this.engine.buildSystemPrompt() + `\n\n# RECENT STORY CONTEXT\n${summaryText}\n\nSpeak the next scripted line now, in character.`; this.realtime = new OpenaiRealtimeClient({ apiKey: this.opts.apiKey, model: this.opts.realtimeModel, voice: newVoice, instructions, tools: this.buildTools(), inputTrack, }); this.attachListeners(this.realtime); await this.realtime.connect(); this.currentVoice = newVoice; } }