from __future__ import annotations """ backend/routes/mobile_bridge_routes.py — S4 Guardian ⇄ cloud REST compatibility. The JARVIS Mobile Guardian APK (phone/jarvis-mobile-guardian) speaks a fixed `/api/*` REST contract (see network/ApiClient.kt). On the local PC that contract is served by phone/local_server.py on :7474; on the cloud it was only partially served — xr_router@/api covers ar_task/ar_scene/ar_game_state/ar_game_builder, but eight endpoints (pair_confirm, status, caps, link_info, command, phone_observe, jarvis/mobile_event, max_autonomy/task) had no cloud handler, so Guardian's status card, capability grid, AR-URL fetch, command box, telemetry sync and autonomy tasks all 404'd against jarvis-cloud.hf.space. This router fills exactly those gaps, mounted at /api behind verify_token (Guardian sends `Authorization: Bearer ` on every call). It delegates to the real cross-device coordinator (modules/max_autonomy.py — the module explicitly built as "the single backend surface for APK, PC AR, browser panels and Guardian heartbeats") and the shared AES-GCM helpers (phone/crypto.py) rather than re-implementing behaviour, so cloud and PC stay in lock-step. Encrypted /api/command: Guardian derives its AES key as SHA-256(passphrase) (CryptoBox.deriveKey). phone/crypto.LinkConfig.key_bytes() derives the same way, so a cloud pairing whose passphrase is the JARVIS_CLOUD_TOKEN lets the Space decrypt and route the command through the safety-gated autonomy engine. """ import base64 import hashlib import logging import os import secrets import time from typing import Any from fastapi import APIRouter, Request, Response router = APIRouter() log = logging.getLogger(__name__) _BOOT_TS = time.time() # --------------------------------------------------------------------------- # # helpers # --------------------------------------------------------------------------- # def _master_token() -> str: try: from backend.dependencies.auth import get_or_create_master_token return get_or_create_master_token() except Exception: return os.environ.get("JARVIS_CLOUD_TOKEN", "").strip() def _uptime_str() -> str: secs = int(time.time() - _BOOT_TS) h, rem = divmod(secs, 3600) m, s = divmod(rem, 60) if h: return f"{h}h {m}m" if m: return f"{m}m {s}s" return f"{s}s" def _cpu_ram() -> tuple[float | None, float | None]: try: import psutil # type: ignore return ( float(psutil.cpu_percent(interval=None)), float(psutil.virtual_memory().percent), ) except Exception: return (None, None) def _base_url(request: Request) -> str: # Honour reverse-proxy host so the AR URL points at the public Space, not the # internal 127.0.0.1:7860 uvicorn bind. host = request.headers.get("x-forwarded-host") or request.headers.get("host") if host: proto = request.headers.get("x-forwarded-proto") or request.url.scheme return f"{proto}://{host}".rstrip("/") return str(request.base_url).rstrip("/") async def _json_body(request: Request) -> dict[str, Any]: try: data = await request.json() return data if isinstance(data, dict) else {} except Exception: return {} def _candidate_keys() -> list[bytes]: """AES keys to try when decrypting a Guardian command envelope, best-first.""" keys: list[bytes] = [] token = _master_token() if token: keys.append(hashlib.sha256(token.encode("utf-8")).digest()) try: from phone.crypto import load_or_create_link keys.append(load_or_create_link().key_bytes()) except Exception as _exc: log.debug("link key unavailable: %s", _exc) return keys # --------------------------------------------------------------------------- # # pairing / status / capabilities / link info # --------------------------------------------------------------------------- # @router.post("/pair_confirm") async def pair_confirm(request: Request) -> dict[str, Any]: body = await _json_body(request) device_name = str(body.get("device_name") or "android").strip()[:64] passphrase = str(body.get("passphrase") or "").strip() # The Bearer token already authenticated the request; pairing simply records # the device. If the passphrase matches the cloud token, encrypted commands # will also decrypt cleanly (see /command). token = _master_token() keyed = bool(token) and secrets.compare_digest(passphrase, token) try: from phone.pairing import mark_paired mark_paired(device_name=device_name or "android") except Exception as _exc: log.debug("mark_paired skipped on cloud: %s", _exc) return {"ok": True, "paired": True, "keyed": keyed, "device_name": device_name} @router.get("/status") async def status() -> dict[str, Any]: cpu, ram = _cpu_ram() location = "cloud" if os.environ.get("SPACE_ID") else "pc" return { "ok": True, "status": "online", "version": "1.0.0", "uptime": _uptime_str(), "mode": location, "activated": True, "assistant_name": "JARVIS", "cpu": cpu, "ram": ram, } @router.get("/caps") async def caps() -> dict[str, Any]: """Flatten the capability catalogue to the {capabilities:[str]} shape the APK deserializes (CapabilityList).""" labels: list[str] = [] try: from phone.capabilities import capabilities_payload for category in capabilities_payload().get("categories", []): for item in category.get("items", []): label = str(item.get("label") or item.get("cmd") or "").strip() if label: labels.append(label) except Exception as _exc: log.debug("capabilities unavailable: %s", _exc) return {"ok": True, "capabilities": labels} @router.get("/link_info") async def link_info(request: Request) -> dict[str, Any]: base = _base_url(request) ws_base = base.replace("https://", "wss://").replace("http://", "ws://") ar_url = f"{base}/webar/" return { "ok": True, "assistant": "JARVIS", "host": base, "pwa": ar_url, "pwa_https": ar_url, "ar": ar_url, "ar_https": ar_url, "ar_engine": "8thwall", "ar_runtime": "webxr", "scene_ws": f"{ws_base}/scene/ws", } # --------------------------------------------------------------------------- # # command (encrypted) — routed through the safety-gated autonomy engine # --------------------------------------------------------------------------- # @router.post("/command") async def command(request: Request) -> dict[str, Any]: envelope = await _json_body(request) if not (envelope.get("nonce") and envelope.get("ciphertext")): return {"ok": False, "error": "expected encrypted envelope {nonce, ciphertext}"} plaintext: dict[str, Any] | None = None try: from phone.crypto import decrypt_json for key in _candidate_keys(): try: plaintext = decrypt_json(envelope, key) break except Exception: continue except Exception as _exc: log.debug("decrypt_json import failed: %s", _exc) if not isinstance(plaintext, dict): # Wrong pairing passphrase for the cloud: pair with the cloud access key # so the Space can derive the same AES key. return {"ok": False, "error": "decrypt failed — pair using your cloud access key"} cmd = str(plaintext.get("cmd") or "").strip() if not cmd: return {"ok": False, "error": "missing cmd"} device_name = str(plaintext.get("device_name") or "").strip()[:80] try: from modules.max_autonomy import execute_task # This router only ever serves the *cloud* path (on the LAN, Guardian hits # phone/local_server.py:7474, which executes for real against the PC). The # cloud container has no PC to drive, so the honest behaviour is to run the # command through the safety-gated coordinator in record+sync mode: it is # logged, risk-assessed and pushed to the AR HUD, without faking hardware # control. Real desktop actions stay on the paired-PC channel. result = execute_task( cmd, source="guardian_command", device_name=device_name, context={"record_only": True}, ) spoken = list(result.response or []) return {"ok": bool(result.ok), "spoken": spoken, "status": result.status, "action": result.action} except Exception as exc: log.warning("guardian command routing failed: %s", exc) return {"ok": False, "spoken": [f"Command failed: {str(exc)[:160]}"]} # --------------------------------------------------------------------------- # # chat — the on-device JARVIS brain (Gemini-native response shape) # --------------------------------------------------------------------------- # def _gemini_envelope(text: str) -> dict[str, Any]: """Wrap plain text in the Gemini generateContent response shape the APK parses (candidates[].content.parts[].text + finishReason).""" return { "candidates": [ { "content": {"parts": [{"text": text or ""}], "role": "model"}, "finishReason": "STOP", } ] } @router.post("/chat") async def chat(request: Request) -> dict[str, Any]: """Synchronous LLM turn for the mobile app. Both the on-device autonomous automation loop (MobileAutomationEngine — reads the live screen via the Accessibility Service, asks JARVIS for the next CLICK/INPUT/SCROLL/DONE step) and the mobile JARVIS chat/decision engine (MobileJarvisEngine) POST {text, context} here and read a Gemini-native response (candidates[].content.parts[].text). The backend only ever served conversational chat at /agent/chat (async, {status:ok} over WebSocket), so /api/chat 404'd and the entire on-device brain was dead on the cloud. This runs the prompt synchronously through the token manager's full Gemini→NVIDIA fallback chain and returns the Gemini envelope, so both callers work with no APK change. """ body = await _json_body(request) text = str(body.get("text") or "").strip() context = body.get("context") if isinstance(body.get("context"), dict) else {} persona = str(context.get("ai") or "jarvis").strip().lower() or "jarvis" if persona not in ("jarvis", "friday"): persona = "jarvis" if not text: return _gemini_envelope("Standing by.") try: from backend.services.token_manager import gemini_call_with_checkpoint reply = await gemini_call_with_checkpoint(text, task_type="general", persona=persona) return _gemini_envelope(reply or "") except Exception as exc: log.warning("mobile /api/chat LLM call failed: %s", exc) return _gemini_envelope("") # --------------------------------------------------------------------------- # # voice — TTS for the mobile assistant (WAV bytes) # --------------------------------------------------------------------------- # @router.get("/voice/speak") async def voice_speak(text: str = "", voice: str = "friday", format: str = "wav") -> Response: """Mobile TTS. ContinuousVoiceRelay fetches `GET /api/voice/speak?text=&voice= &format=wav` and injects the raw PCM into its live WAV log. The backend only served TTS at `POST /voice/speak` (JSON body {text, agent}), so the mobile call 404'd on both path and method — JARVIS had no spoken voice on the cloud. Same XTTS-v2 pipeline as /voice/speak; `voice` maps to the persona.""" text = (text or "").strip() if not text: return Response(status_code=204) try: from backend.voice.tts import TTSPipeline tts = TTSPipeline() audio = await tts.synthesize(text, personality=(voice or "friday")) return Response(content=audio, media_type="audio/wav") except Exception as exc: log.warning("mobile /api/voice/speak failed: %s", exc) return Response(status_code=500) # --------------------------------------------------------------------------- # # telemetry / events / autonomy # --------------------------------------------------------------------------- # @router.post("/phone_observe") async def phone_observe(request: Request) -> dict[str, Any]: body = await _json_body(request) device_name = str(body.get("device_name") or "").strip()[:80] payload = {k: v for k, v in body.items() if k != "device_name"} try: from modules.max_autonomy import record_mobile_event record_mobile_event("phone_observe", payload, device_name=device_name) except Exception as exc: log.debug("phone_observe record failed: %s", exc) return {"ok": False, "error": str(exc)[:160]} return {"ok": True} @router.post("/jarvis/mobile_event") async def mobile_event(request: Request) -> dict[str, Any]: body = await _json_body(request) kind = str(body.get("kind") or "event").strip()[:80] device_name = str(body.get("device_name") or "").strip()[:80] payload = body.get("payload") if not isinstance(payload, dict): payload = {k: v for k, v in body.items() if k not in ("kind", "device_name", "ts")} try: from modules.max_autonomy import record_mobile_event record_mobile_event(kind, payload, device_name=device_name) except Exception as exc: log.debug("mobile_event record failed: %s", exc) return {"ok": False, "error": str(exc)[:160]} return {"ok": True} @router.post("/max_autonomy/task") async def max_autonomy_task(request: Request) -> dict[str, Any]: body = await _json_body(request) task = str(body.get("task") or "").strip() source = str(body.get("source") or "jarvis-native-android").strip()[:80] device_name = str(body.get("device_name") or "").strip()[:80] confirmed = bool(body.get("confirmed") or False) context = dict(body.get("context")) if isinstance(body.get("context"), dict) else {} # Cloud boundary (same rationale as /command): the headless Space has no PC to # drive and does not ship phone/command_runner (a Windows-PC command surface), # so autonomy runs in record+sync mode here — logged, risk-assessed and pushed # to the AR HUD. Real execution happens on the paired-PC :7474 channel. This is # also the safer default: dangerous "confirmed" actions record rather than fire. if os.environ.get("SPACE_ID"): context.setdefault("record_only", "true") try: from modules.max_autonomy import execute_task result = execute_task( task, source=source, device_name=device_name, context=context, confirmed=confirmed, ) return result.as_dict() except Exception as exc: log.warning("max_autonomy task failed: %s", exc) return { "ok": False, "status": "error", "task": task, "action": "error", "message": f"Autonomy task failed: {str(exc)[:160]}", "requires_confirmation": False, "blocked": False, "risk": {}, "plan": [], "response": [f"Autonomy task failed: {str(exc)[:160]}"], }