Spaces:
Running
Running
| from fastapi import HTTPException, APIRouter, Request, Depends | |
| from pydantic import BaseModel | |
| # Server-side plan gating + per-user usage metering for the company-provisioned | |
| # 3D-model-generation routes (spawn/builder/forge). AR_MODEL_GEN is only granted on | |
| # paid plans and is metered against the plan's cap; the frontend renders the locked | |
| # state but enforcement lives here. | |
| from backend.billing.gating import gate_and_meter | |
| from backend.billing.plans import Feature | |
| _AR_MODEL_GEN_GATE = Depends(gate_and_meter(Feature.AR_MODEL_GEN)) | |
| # AR-FIX-SESSION: import canonical AR scene store to mirror anchor writes. | |
| # Guarded by try/except — safe in frozen PyInstaller context if path not set up. | |
| try: | |
| from modules.ar_scene import apply_patch as _ar_scene_apply_patch | |
| except ImportError: | |
| _ar_scene_apply_patch = None | |
| router = APIRouter() | |
| class SpatialAnchor(BaseModel): | |
| id: str | |
| x: float | |
| y: float | |
| z: float | |
| session_id: str = "default" | |
| ANCHOR_DB = {} | |
| async def save_anchor(a: SpatialAnchor): | |
| if a.session_id not in ANCHOR_DB: | |
| ANCHOR_DB[a.session_id] = [] | |
| # Update or insert into ephemeral in-memory store | |
| status = "saved" | |
| for i, existing in enumerate(ANCHOR_DB[a.session_id]): | |
| if existing.id == a.id: | |
| ANCHOR_DB[a.session_id][i] = a | |
| status = "updated" | |
| break | |
| else: | |
| ANCHOR_DB[a.session_id].append(a) | |
| # AR-FIX-SESSION: mirror anchor to canonical ar_scene.json via apply_patch. | |
| # Constructs 4x4 row-major translation matrix (identity rotation) from x/y/z. | |
| if _ar_scene_apply_patch is not None: | |
| _ar_scene_apply_patch({ | |
| "anchors": { | |
| a.id: [1.0, 0.0, 0.0, a.x, | |
| 0.0, 1.0, 0.0, a.y, | |
| 0.0, 0.0, 1.0, a.z, | |
| 0.0, 0.0, 0.0, 1.0] | |
| } | |
| }) | |
| return {"status": status} | |
| async def get_anchors(session_id: str): | |
| return ANCHOR_DB.get(session_id, []) | |
| async def get_session_state(): | |
| # AR-FIX-S2: was unconditionally {state: "active"} — now reflects real WS client count. | |
| # ws_manager.active_connections is a list[WebSocket] maintained by ConnectionManager.connect() | |
| # and ConnectionManager.disconnect() in backend/ws/agent_ws.py. | |
| # Import inside function body — same pattern as /ar_task, /ar_scene_patch, etc. | |
| from backend.ws.agent_ws import ws_manager | |
| connected = len(ws_manager.active_connections) | |
| total_anchors = sum(len(v) for v in ANCHOR_DB.values()) | |
| return { | |
| "state": "active" if connected > 0 else "idle", | |
| "connected_clients": connected, | |
| "total_anchors": total_anchors, | |
| } | |
| class ShaderPayload(BaseModel): | |
| shader_code: str | |
| target: str = "webgl2" | |
| async def compile_shader(payload: ShaderPayload): | |
| """ | |
| Validate a WebGL/WebXR shader server-side and return a content hash. | |
| There is no server-side binary compilation here by design: the AR runtime is | |
| WebGL/Three.js, so the shader is actually compiled client-side by the GPU | |
| driver at runtime. This endpoint therefore does real, honest work — a | |
| lightweight static validation pass + a stable content hash the client can use | |
| for caching — rather than returning a fake `mock://` blob URL (the previous | |
| behaviour), which read like a real artifact link but resolved to nothing. | |
| """ | |
| import hashlib | |
| code = payload.shader_code or "" | |
| shader_hash = hashlib.md5(code.encode()).hexdigest() | |
| warnings, errors = [], [] | |
| if not code.strip(): | |
| errors.append("empty shader source") | |
| if code.count("{") != code.count("}"): | |
| errors.append("unbalanced braces") | |
| if code.count("(") != code.count(")"): | |
| errors.append("unbalanced parentheses") | |
| if "void main" not in code and code.strip(): | |
| warnings.append("no main() entry point found") | |
| return { | |
| "status": "validated" if not errors else "error", | |
| "hash": shader_hash, | |
| "target": payload.target, | |
| "warnings": warnings, | |
| "errors": errors, | |
| # Compilation is client-side (WebGL); stated honestly, not a fake URL. | |
| "compilation": "client_side_webgl", | |
| } | |
| class ArTaskPayload(BaseModel): | |
| task: str | |
| mode: str = "task" | |
| device_name: str = "unknown" | |
| async def handle_ar_task(payload: ArTaskPayload): | |
| """ | |
| Receives AR tasks from Android phone or WebAR frontend, | |
| and broadcasts them to all connected XR clients (e.g. headsets, Desktop UI). | |
| """ | |
| from backend.ws.agent_ws import ws_manager | |
| await ws_manager.broadcast({ | |
| "event": "ar:task", | |
| "payload": payload.dict() | |
| }) | |
| return {"ok": True, "status": "dispatched", "message": f"Task '{payload.task}' sent to AR bus."} | |
| async def get_ar_config(request: Request): | |
| """ | |
| Returns base configuration for the WebAR engine. | |
| Everything here is derived from the *actual request*, never hardcoded. The | |
| previous version returned ``ws://{HOST or 127.0.0.1}:5050`` and | |
| ``http://127.0.0.1:7474/...`` unconditionally, which is wrong for every | |
| client that is not on the same machine as the backend: a phone or headset | |
| resolves 127.0.0.1 to itself, so the scene socket and game URLs could never | |
| connect from the cloud deployment. Port 5050 was wrong too — the socket is | |
| mounted at ``/scene-ws`` on this same app (backend/main.py), and a hosted | |
| Space only exposes 443. | |
| """ | |
| # Trust X-Forwarded-Proto before request.url.scheme. HuggingFace terminates | |
| # TLS at its proxy and forwards plain HTTP to the container, so base_url | |
| # reports "http" even though the browser loaded the page over https. Deriving | |
| # the socket scheme from it produced `ws://…` on an `https://` page, which | |
| # browsers block outright as mixed content — trading the old 127.0.0.1 bug | |
| # for a subtler one that only shows up in a real browser. | |
| forwarded = request.headers.get("x-forwarded-proto", "").split(",")[0].strip().lower() | |
| scheme = forwarded or (request.url.scheme or "http") | |
| host = str(request.base_url).split("://", 1)[1].rstrip("/") | |
| base = f"{scheme}://{host}" | |
| ws_scheme = "wss" if scheme == "https" else "ws" | |
| return { | |
| "ok": True, | |
| "deviceName": "OMEGA-CORE", | |
| "wsBase": f"{ws_scheme}://{host}/scene-ws", | |
| "gameUrls": { | |
| "eternum": f"{base}/games/eternum", | |
| "ripples": f"{base}/games/ripples", | |
| }, | |
| } | |
| async def handle_ar_scene_patch(patch: dict): | |
| """ | |
| Syncs WebAR component states across the network. | |
| """ | |
| from backend.ws.agent_ws import ws_manager | |
| await ws_manager.broadcast({ | |
| "event": "ar:patch", | |
| "payload": patch | |
| }) | |
| return {"ok": True, "status": "synced"} | |
| async def get_ar_scene_state(): | |
| # Reflect the authoritative object list so a fresh client can hydrate over | |
| # REST (the WS scene bus also delivers it via hello/state). | |
| if _ar_scene_apply_patch is not None: | |
| try: | |
| from modules.ar_scene import load_state | |
| st = load_state() | |
| return { | |
| "ok": True, | |
| "entities": st.get("ar_objects", []), | |
| "environment": st.get("preset", "default"), | |
| } | |
| except Exception: | |
| pass | |
| return {"ok": True, "entities": [], "environment": "default"} | |
| class ArSceneSyncPayload(BaseModel): | |
| device_name: str = "unknown" | |
| objects: list[dict] = [] | |
| async def sync_ar_scene(payload: ArSceneSyncPayload): | |
| """ | |
| Full-scene sync from the mobile AR Lab. Guardian POSTs {device_name, objects} | |
| on every spawn / manipulate / delete / room-scan. Previously the backend only | |
| exposed GET /ar_scene, so this POST returned 405 on cloud and the authoritative | |
| scene never persisted. | |
| Persists the object list into the canonical store and fans it out to every | |
| connected scene-bus client (phone reads state.ar_objects) plus the agent WS | |
| event bus (desktop / headset listeners), mirroring the model-push path. | |
| """ | |
| objects = payload.objects if isinstance(payload.objects, list) else [] | |
| # Durable authoritative write (idempotent full-list replace). | |
| if _ar_scene_apply_patch is not None: | |
| _ar_scene_apply_patch({"ar_objects": objects}) | |
| # Fan out to scene-bus clients — the bus re-applies + broadcasts {type:state}. | |
| try: | |
| from phone import ws_scene_bus | |
| await ws_scene_bus.broadcast_to_clients_async({ | |
| "type": "patch", | |
| "patch": {"ar_objects": objects}, | |
| }) | |
| except Exception: | |
| pass | |
| # Parity with the other AR handlers: notify the agent WS event bus too. | |
| try: | |
| from backend.ws.agent_ws import ws_manager | |
| await ws_manager.broadcast({ | |
| "event": "ar:scene", | |
| "payload": {"device_name": payload.device_name, "objects": objects}, | |
| }) | |
| except Exception: | |
| pass | |
| return {"ok": True, "status": "synced", "count": len(objects)} | |
| async def update_ar_game_state(state: dict): | |
| from backend.ws.agent_ws import ws_manager | |
| await ws_manager.broadcast({"event": "ar:game_state", "payload": state}) | |
| return {"ok": True} | |
| async def sync_ar_game_builder(graph: dict): | |
| from backend.ws.agent_ws import ws_manager | |
| await ws_manager.broadcast({"event": "ar:game_builder", "payload": graph}) | |
| return {"ok": True} | |
| class SpawnModelRequest(BaseModel): | |
| description: str | |
| persona: str = "JARVIS" | |
| # Without this the endpoint was a dead end: spawn_3d_model() returns an | |
| # "ask the user, then call again with use_cloud_fallback=True" instruction | |
| # whenever local SF3D weights are absent — which is ALWAYS true on the | |
| # CPU-only cloud Space — but the request model had no way to express it, | |
| # so no client could ever take the cloud path. | |
| use_cloud_fallback: bool = False | |
| async def spawn_model(request: SpawnModelRequest, _user: str = _AR_MODEL_GEN_GATE): | |
| import logging | |
| logger = logging.getLogger(__name__) | |
| from backend.tools import xr_tools | |
| try: | |
| # On the cloud Space the 10GB local weights are never present, so honour | |
| # an explicit cloud-fallback request instead of returning the interactive | |
| # prompt string as if it were a model path. | |
| model_path = await xr_tools.spawn_3d_model( | |
| request.description, request.persona, request.use_cloud_fallback | |
| ) | |
| if isinstance(model_path, str) and model_path.startswith("STATUS: LOCAL_MODEL_MISSING"): | |
| # Surface this as a real, actionable API state — not a fake success | |
| # whose "model_path" is a paragraph of instructions. | |
| return { | |
| "status": "needs_confirmation", | |
| "reason": "local_sf3d_missing", | |
| "message": "Local Stable-Fast-3D weights are not installed. Retry with " | |
| "use_cloud_fallback=true to generate via the hosted API.", | |
| } | |
| return {"status": "ok", "model_path": model_path} | |
| except xr_tools.StableFast3DNotProvisionedError as e: | |
| logger.error(f"SF3D not provisioned: {e}") | |
| raise HTTPException(status_code=503, detail=str(e)) | |
| except Exception as e: | |
| # A core feature must not hard-fail because a THIRD-PARTY space is down. | |
| # Verified 2026-07-20: stabilityai/stable-fast-3d's /run_button raises an | |
| # opaque AppError for every parameter combination, reproduced directly | |
| # from a client outside this backend — their GPU inference is the fault, | |
| # not our request (signature confirmed against their own view_api). | |
| # Our procedural Blender Builder is fully operational, so degrade to it | |
| # instead of returning a 500: the user still gets a model in the scene. | |
| logger.warning(f"SF3D generation failed ({e}); falling back to procedural Builder.") | |
| try: | |
| result = await builder_generate( | |
| BuilderRequest(description=request.description, persona=request.persona) | |
| ) | |
| if isinstance(result, dict): | |
| result = dict(result) | |
| result["fallback"] = "procedural_builder" | |
| result["fallback_reason"] = f"stable-fast-3d unavailable: {str(e)[:160]}" | |
| return result | |
| except HTTPException: | |
| raise | |
| except Exception as fallback_error: | |
| logger.exception("Procedural Builder fallback also failed") | |
| raise HTTPException( | |
| status_code=502, | |
| detail=f"SF3D unavailable ({str(e)[:120]}) and Builder fallback failed: {fallback_error}", | |
| ) | |
| async def check_health(): | |
| from backend.services.connectors import stable_fast_3d_local | |
| return await stable_fast_3d_local.health_check() | |
| # ─── S4: Builder + Forge Blender pipeline ──────────────────────────────────── | |
| BLUEPRINT_PROMPT = """Create a compact JSON blueprint for a fully interactive 3D model. | |
| Return JSON only, no markdown, no prose. | |
| Schema: | |
| {{ | |
| "name": "string", | |
| "kit": "house|room|device|vehicle|character|scene", | |
| "parts": [ | |
| {{ | |
| "name": "string", | |
| "type": "base|wall|door|window|roof|drawer|panel|screen|wheel|prop|light|arm|leg|head|detail", | |
| "geometry": "box|sphere|cylinder|cone|torus|ring|capsule|octahedron|icosahedron", | |
| "material": "matte|metal|glass|glow|wood", | |
| "position": [0,0,0], | |
| "rotation": [0,0,0], | |
| "scale": [1,1,1], | |
| "actions": ["open","close","toggle","rotate","reveal","hide","shatter","reconfigure"] | |
| }} | |
| ], | |
| "behaviors": [ | |
| {{ "gesture": "pinch|fist|open-palm|swipe-left|swipe-right|two-hand-scale", "action": "string", "target": "part name" }} | |
| ] | |
| }} | |
| Prompt: {description}""" | |
| def _parse_blueprint_json(text: str) -> dict | None: | |
| """Extract the blueprint object from LLM output. | |
| S4: the NVIDIA fallback (glm-5.1 and friends) wraps JSON in <think> blocks | |
| and prose, and may emit several JSON-ish fragments. A first-{ to last-} | |
| slice fails whenever any prose contains a stray brace — scan for the first | |
| *balanced* object that parses and looks like a blueprint instead. | |
| """ | |
| import json as _json | |
| import re as _re | |
| raw = str(text or "") | |
| raw = _re.sub(r"<think>.*?</think>", "", raw, flags=_re.DOTALL) | |
| raw = raw.replace("```json", "```").strip() | |
| candidates = [] | |
| # Fenced blocks first — models that follow instructions put the JSON there. | |
| for block in raw.split("```")[1::2]: | |
| candidates.append(block.strip()) | |
| candidates.append(raw.replace("```", "")) | |
| for candidate in candidates: | |
| idx = candidate.find("{") | |
| while idx >= 0: | |
| depth = 0 | |
| for end in range(idx, len(candidate)): | |
| ch = candidate[end] | |
| if ch == "{": | |
| depth += 1 | |
| elif ch == "}": | |
| depth -= 1 | |
| if depth == 0: | |
| try: | |
| obj = _json.loads(candidate[idx:end + 1]) | |
| except Exception: | |
| obj = None | |
| if isinstance(obj, dict) and isinstance(obj.get("parts"), list) and obj["parts"]: | |
| return obj | |
| break | |
| idx = candidate.find("{", idx + 1) | |
| return None | |
| class BuilderRequest(BaseModel): | |
| description: str | |
| persona: str = "JARVIS" | |
| name: str = "" | |
| async def builder_generate(request: BuilderRequest, _user: str = _AR_MODEL_GEN_GATE): | |
| """"Builder <description>" — blueprint via the token manager (it selects the | |
| right vault credential for the generation task, NVIDIA fallback included), | |
| then a real headless-Blender procedural build, then serve + durable hub push.""" | |
| import logging | |
| logger = logging.getLogger(__name__) | |
| from backend.services import blender_processor | |
| from backend.services.token_manager import gemini_call_with_checkpoint | |
| if not blender_processor.blender_available(): | |
| raise HTTPException(status_code=503, detail="Blender is not installed in this runtime.") | |
| description = request.description.strip() | |
| if not description: | |
| raise HTTPException(status_code=422, detail="A model description is required.") | |
| # Token manager owns credential selection: task_type="image" routes to the | |
| # IMAGE_GENERATION_* key domain (cloud/pc aware), with NVIDIA vault fallback. | |
| raw = await gemini_call_with_checkpoint( | |
| BLUEPRINT_PROMPT.format(description=description), | |
| task_type="image", persona=request.persona.lower(), | |
| ) | |
| blueprint = _parse_blueprint_json(raw) | |
| if not blueprint: | |
| head = str(raw or "")[:500] | |
| logger.error(f"Builder blueprint unparseable; raw head: {head!r}") | |
| raise HTTPException( | |
| status_code=502, | |
| detail=f"Blueprint generation returned no parseable JSON. Model said: {head[:160]!r}", | |
| ) | |
| name = request.name or blueprint.get("name") or description[:24] | |
| try: | |
| result = await blender_processor.build_from_blueprint(blueprint, name) | |
| except Exception as e: | |
| logger.exception("Builder Blender pass failed") | |
| raise HTTPException(status_code=500, detail=f"Blender build failed: {e}") | |
| from backend.tools.xr_tools import push_model_to_ar_scene | |
| await push_model_to_ar_scene(result["model_path"], name) | |
| return {"status": "ok", "name": name, "blueprint": blueprint, **result} | |
| class ForgeProcessRequest(BaseModel): | |
| model_url: str | |
| name: str = "Sourced Model" | |
| kit: str = "scene" | |
| behaviors: list | None = None | |
| async def forge_process(request: ForgeProcessRequest, _user: str = _AR_MODEL_GEN_GATE): | |
| """Forge handoff for sourced models (Sketchfab/TurboSquid/...). Reuse-check runs | |
| FIRST against saved templates (pure-python GLB part scan — no Blender launch); | |
| Blender only on a miss, and the fresh result becomes a new reusable template.""" | |
| import logging | |
| import os | |
| import tempfile | |
| logger = logging.getLogger(__name__) | |
| from backend.services import blender_processor | |
| url = request.model_url.strip() | |
| if not url.lower().startswith(("http://", "https://")): | |
| raise HTTPException(status_code=422, detail="model_url must be an http(s) URL.") | |
| import asyncio as _asyncio | |
| import requests as _requests | |
| def _download() -> str: | |
| with _requests.get(url, stream=True, timeout=60) as resp: | |
| resp.raise_for_status() | |
| fd, path = tempfile.mkstemp(suffix=".glb") | |
| written = 0 | |
| with os.fdopen(fd, "wb") as fh: | |
| for chunk in resp.iter_content(1024 * 256): | |
| written += len(chunk) | |
| if written > 200 * 1024 * 1024: | |
| raise RuntimeError("Model exceeds the 200MB processing limit.") | |
| fh.write(chunk) | |
| return path | |
| try: | |
| local_path = await _asyncio.to_thread(_download) | |
| except Exception as e: | |
| raise HTTPException(status_code=502, detail=f"Model download failed: {e}") | |
| template = {"behaviors": request.behaviors} if request.behaviors else None | |
| try: | |
| result = await blender_processor.forge_process_glb(local_path, request.name, request.kit, template) | |
| except Exception as e: | |
| logger.exception("Forge Blender pass failed") | |
| raise HTTPException(status_code=500, detail=f"Forge processing failed: {e}") | |
| finally: | |
| # The temp download only feeds the part scan / Blender import; the processed | |
| # output is a separate file in storage/models3d. Always clean up. | |
| if os.path.exists(local_path): | |
| os.unlink(local_path) | |
| if result.get("reused"): | |
| # model_path was the (now deleted) temp file — the client keeps its own copy | |
| # of the sourced model and only needs the template to apply. | |
| result.pop("model_path", None) | |
| return {"status": "ok", **result} | |
| async def blender_health(): | |
| from backend.services import blender_processor | |
| return { | |
| "blender_available": blender_processor.blender_available(), | |
| "blender_path": blender_processor.find_blender() or None, | |
| "templates": len(blender_processor.load_templates()), | |
| "models_dataset": blender_processor.MODELS_DATASET, | |
| } | |