"""Assistant tool surface for the local photo/video immersive viewer. The viewer itself lives in the WebAR client (`ar-core/ar-media-viewer.js`), which is what actually holds the decoded frame and the projection geometry. These tools are the assistant's way in: each one broadcasts a `media:control` event over the existing agent WebSocket, which the client picks up and applies to the live session — the same mechanism `ar_control_tool` already uses for AR state, rather than a second, parallel channel invented for this feature. Honesty rules encoded here rather than left to the model's discretion, because the governing brief requires them and a prompt is easier to drift from than code: - `experimental_surround` is never selected implicitly. Asking for "360" gets the real projection path; the generative mode has to be named explicitly, and the return text says plainly that what it produces is invented. - Spatial audio is never called "Dolby Atmos". If a caller passes that string the tool corrects it in the response instead of quietly accepting the wording. """ from __future__ import annotations PROJECTIONS = ("flat", "180", "360", "fisheye") PACKINGS = ("mono", "sbs-full", "sbs-half", "tb-full", "tb-half") ENVIRONMENTS = ("passthrough", "mixed", "theater", "inox", "imax", "pvr") # What each environment actually changes, so the assistant can answer truthfully # when asked. These are room and screen presets -- they never touch decode # quality, resolution, or bitrate. ENVIRONMENT_FACTS = { "passthrough": "the real room stays visible; the picture is composited into it as a screen", "mixed": "passthrough plus a virtual bezel and ambient light response", "theater": "a standard cinema hall, 36 degrees subtended from the seat, gently curved screen", "pvr": "a mainstream multiplex auditorium, 34 degrees, further back, flatter screen", "inox": "a premium large-screen auditorium, 40 degrees, between Theater and IMAX in scale", "imax": "large-format geometry, 70 degrees subtended, curvature radius 1.5x the screen width", } _DOLBY_WORDS = ("dolby", "atmos") async def _dispatch(action: str, data: dict) -> None: from backend.ws.agent_ws import ws_manager await ws_manager.broadcast({ "event": "media:control", "payload": {"action": action, "data": data}, }) async def media_open_tool(file: str, projection: str = "", packing: str = "", environment: str = "", *args, **kwargs): """Open a local photo or video file in the immersive viewer. `file` is a path or name on the user's own device. Nothing is uploaded. `projection` (flat/180/360/fisheye) and `packing` are optional overrides -- left blank, the viewer auto-detects them from the file and tells the user why. """ if projection and projection not in PROJECTIONS: return f"Unknown projection '{projection}'. Valid: {', '.join(PROJECTIONS)}." if packing and packing not in PACKINGS: return f"Unknown stereo packing '{packing}'. Valid: {', '.join(PACKINGS)}." if environment and environment not in ENVIRONMENTS: return f"Unknown environment '{environment}'. Valid: {', '.join(ENVIRONMENTS)}." await _dispatch("open", { "file": file, "projection": projection or None, "packing": packing or None, "environment": environment or None, }) detail = [] if projection: detail.append(f"as {projection}") if packing: detail.append(f"packed {packing}") if environment: detail.append(f"in {environment} ({ENVIRONMENT_FACTS[environment]})") suffix = (" " + ", ".join(detail)) if detail else " with format auto-detected" return f"Opening {file} in the immersive viewer{suffix}." async def media_set_environment_tool(environment: str, *args, **kwargs): """Switch the viewing mode: passthrough, mixed, theater, inox, imax or pvr.""" environment = (environment or "").strip().lower() if environment not in ENVIRONMENTS: return (f"'{environment}' is not one of the viewing modes. " f"Valid: {', '.join(ENVIRONMENTS)}.") await _dispatch("environment", {"environment": environment}) return (f"Switched to {environment} — {ENVIRONMENT_FACTS[environment]}. " "This changes the room and screen only; the picture is decoded exactly the same way.") async def media_set_format_tool(projection: str = "", packing: str = "", *args, **kwargs): """Override the projection format and/or the stereo packing.""" out = [] if projection: if projection not in PROJECTIONS: return f"Unknown projection '{projection}'. Valid: {', '.join(PROJECTIONS)}." await _dispatch("projection", {"projection": projection}) out.append(f"format set to {projection}") if packing: if packing not in PACKINGS: return f"Unknown stereo packing '{packing}'. Valid: {', '.join(PACKINGS)}." await _dispatch("packing", {"packing": packing}) out.append(f"stereo packing set to {packing}") if not out: return "Nothing to change — give a projection, a packing, or both." return "Done: " + " and ".join(out) + "." async def media_adjust_screen_tool(width: float = 0, distance: float = 0, elevation: float = 0, curvature_radius: float = 0, *args, **kwargs): """Resize, move or re-curve the virtual screen. Values are in metres.""" patch = {} for key, value in (("width", width), ("distance", distance), ("elevation", elevation), ("curvatureRadius", curvature_radius)): try: v = float(value) except (TypeError, ValueError): continue if v: patch[key] = v if not patch: return "Nothing to adjust — give a width, distance, elevation or curvature radius in metres." await _dispatch("adjust_screen", patch) parts = ", ".join(f"{k} {v}m" for k, v in patch.items()) return f"Screen adjusted: {parts}. It moves live in the running session." async def media_spatial_audio_tool(enabled: bool = True, *args, **kwargs): """Turn head-tracked spatial audio on or off. This is real HRTF binaural rendering. It is NOT Dolby Atmos, and must never be described as such -- that is a licensed technology this product does not carry. """ want = str(enabled).strip().lower() not in ("false", "0", "off", "no", "") await _dispatch("spatial_audio", {"enabled": want}) if not want: return "Spatial audio off — back to normal stereo output." return ("Spatial audio on. This is real HRTF binaural rendering: a genuinely ambisonic " "source is decoded as recorded, and an ordinary stereo track is widened. " "It is not Dolby Atmos — that is a separate licensed technology this product " "does not include. It also only applies to headphones, so it stays off on speakers.") async def media_ai_depth_tool(enabled: bool = True, *args, **kwargs): """Toggle the AI depth/stereo enhancement for flat video.""" want = str(enabled).strip().lower() not in ("false", "0", "off", "no", "") await _dispatch("ai_depth", {"enabled": want}) if not want: return "AI 3D depth off." return ("AI 3D depth requested. This estimates per-pixel depth and synthesises a second eye " "from a flat video — the small regions revealed behind foreground objects are filled " "in, not captured. Note: the depth model is not installed on the Space yet, so the " "control will report that rather than pretending to work.") async def media_experimental_surround_tool(enabled: bool = True, confirm: bool = False, *args, **kwargs): """Toggle the EXPERIMENTAL generative surround mode. Deliberately requires being named explicitly. Asking to "play this in 360" must never land here -- that request is served by the real 360 projection path. """ want = str(enabled).strip().lower() not in ("false", "0", "off", "no", "") if not want: await _dispatch("experimental_surround", {"enabled": False}) return "Experimental AI surround off." await _dispatch("experimental_surround", {"enabled": True}) return ("Experimental AI-generated surround requested. Be clear with the user about what this " "is: it invents the roughly 300 degrees of scene the camera never captured. Most of " "what they would see in this mode is generated, not filmed, and it must never be " "described as a real 360 video or as 'flawless'. A persistent on-screen label saying " "'Experimental: AI-generated surround — not real footage' stays visible the whole " "time. Note: the generative model is not installed on the Space yet, so the control " "will say so rather than pretending to work.") async def media_describe_tool(topic: str = "", *args, **kwargs): """Answer honestly about what the viewer can and cannot do.""" t = (topic or "").strip().lower() if any(w in t for w in _DOLBY_WORDS): return ("That is not Dolby Atmos. Dolby Atmos is a specific licensed technology, and this " "product does not carry that licence. What it does have is real HRTF binaural " "spatial audio — the same underlying mechanism, head-tracked — which I can turn on.") if "360" in t and ("convert" in t or "make" in t or "turn" in t): return ("A normal video cannot honestly be made into a real 360 video — a flat camera never " "recorded the other 300 degrees. What I can do is play it at full original quality " "on a virtual screen inside an immersive environment (theater, IMAX, passthrough " "and so on), and optionally add an AI depth effect for a stereo sense of depth. " "There is also a clearly-labelled experimental mode that generates the surround, " "but that content is invented, not filmed.") return ("The immersive viewer plays local photos and videos in four formats (flat, 180, 360, " "fisheye), five stereo packings, and six viewing modes (" + ", ".join(ENVIRONMENTS) + "). Format and packing are auto-detected and overridable. Switching viewing mode " "changes the room and screen geometry only — never the decode path, resolution or " "bitrate.")