import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") # Load models locally (no external text-encoder service) and prefer the HF cache. os.environ.setdefault("TEXT_ENCODER_MODE", "local") import base64 import gzip import json import random import sys import time import xml.etree.ElementTree as ET from pathlib import Path import spaces # must precede torch / CUDA-touching imports import torch import numpy as np import trimesh import gradio as gr # The vendored ARDY package lives next to this file. sys.path.insert(0, str(Path(__file__).resolve().parent)) from ardy.model import load_model # noqa: E402 from ardy.model.load_model import load_text_encoder # noqa: E402 from ardy.motion_rep.tools import length_to_mask # noqa: E402 from ardy.tools import seed_everything, to_numpy # noqa: E402 # Two rigs are offered in the playground: # "human" -> ARDY-Core-RP-20FPS-Horizon40, 27-joint skeleton @ 20 fps # "robot" -> ARDY-G1-RP-25FPS-Horizon52, 34-joint Unitree G1 robot @ 25 fps # The robot rig matches the sibling Space hugging-apps/ardy-g1-motion-generation. DEFAULT_RIG = "human" MAX_SEED = 2**31 - 1 # ----------------------------------------------------------------------------- # Model loading (module scope). # # ZeroGPU has no live GPU at startup: it intercepts .to("cuda") *placement* but # NOT arbitrary CUDA compute. LLM2Vec's PEFT load runs a LoRA `merge_and_unload` # (real matmuls), so we build everything on CPU first, then .to("cuda") — the # placement call is what the ZeroGPU hijack packs to disk and streams into VRAM # on the first @spaces.GPU request. # ----------------------------------------------------------------------------- # transformers / PEFT / safetensors infer their load device from # torch.cuda.is_available(), which ZeroGPU reports True at startup even though no # real GPU is attached — so a plain load tries to place weights on cuda and dies # with "No CUDA GPUs are available". Force every loader onto CPU by masking # is_available() during construction, then restore it and .to("cuda") (which the # ZeroGPU hijack packs to disk + streams into VRAM on the first request). # # A single text encoder is built once and shared across both motion models # (load_text_encoder is explicitly designed for this — see its docstring). _real_cuda_available = torch.cuda.is_available torch.cuda.is_available = lambda: False try: print("Loading ARDY text encoder (LLM2Vec-Llama-3-8B) on CPU…", flush=True) _text_encoder = load_text_encoder(mode="local", device="cpu") print("Loading ARDY human (core) motion model on CPU…", flush=True) MODEL_HUMAN = load_model("core", device="cpu", text_encoder=_text_encoder) print("Loading ARDY robot (G1) motion model on CPU…", flush=True) MODEL_ROBOT = load_model("g1", device="cpu", text_encoder=_text_encoder) finally: torch.cuda.is_available = _real_cuda_available print("Moving models to CUDA (ZeroGPU-intercepted placement)…", flush=True) _text_encoder = _text_encoder.to("cuda") for _m in (MODEL_HUMAN, MODEL_ROBOT): _m.to("cuda") # self.device was captured at construction (device="cpu"); generation creates # many tensors on it, so retarget it to cuda to match the moved weights. _m.device = "cuda" _m.eval() def _rig_params(model): fps = float(model.motion_rep.fps) skeleton = model.skeleton parents = skeleton.joint_parents.cpu().numpy().astype(int).tolist() patch = model.num_frames_per_token gen_horizon = model.gen_horizon_len num_base_steps = int(model.diffusion.num_base_steps) # History carried between autoregressive windows. The reference streaming demo # keeps this SHORT so a new prompt takes effect within a window. hist_crop = max(patch, (4 // patch) * patch) root_idx = parents.index(-1) if -1 in parents else 0 return { "model": model, "fps": fps, "skeleton": skeleton, "parents": parents, "patch": patch, "gen_horizon": gen_horizon, "num_base_steps": num_base_steps, "hist_crop": hist_crop, "root_idx": root_idx, } RIGS = { "human": _rig_params(MODEL_HUMAN), "robot": _rig_params(MODEL_ROBOT), } # The largest base-step count across rigs bounds the diffusion-steps slider. NUM_BASE_STEPS = max(r["num_base_steps"] for r in RIGS.values()) for _name, _r in RIGS.items(): print( f"[{_name}] rig ready: {_r['skeleton'].nbjoints} joints, {_r['fps']} fps, " f"horizon {_r['gen_horizon']}, patch {_r['patch']}, " f"hist_crop {_r['hist_crop']}, base_steps {_r['num_base_steps']}", flush=True, ) def _normalize_rig(rig) -> str: """Map any UI/API rig value onto a valid RIGS key ('human' | 'robot').""" if rig is None: return DEFAULT_RIG key = str(rig).strip().lower() if key in RIGS: return key if key.startswith("hum") or "core" in key or "person" in key: return "human" if key.startswith("rob") or "g1" in key or "unitree" in key: return "robot" return DEFAULT_RIG # ----------------------------------------------------------------------------- # HUMAN rig: skinned body mesh (ARDY "CoreSkin" linear-blend skinning). # # The reference viz (ardy/viz/viser_utils.py) renders a smooth humanoid body by # skinning a bind mesh with the per-frame *global* joint transforms: # verts = CoreSkin.skin(global_rot_mats, posed_joints, rot_is_global=True) # The browser holds the static skin data (bind vertices / faces / LBS # indices+weights) and does the per-vertex blend, while the server sends only the # tiny per-frame joint affine matrices # A[f,j] = fk[f,j] @ bind_rig_transform_inv[j] (fk = [R_global | pos]) # so the payload stays small (~0.1 MB/clip). # ----------------------------------------------------------------------------- _HUMAN_SKEL = RIGS["human"]["skeleton"] _SKIN_PATH = Path(_HUMAN_SKEL.folder) / "skin_standard.npz" _skin = np.load(_SKIN_PATH) BIND_RIG_INV = np.linalg.inv( np.asarray(_skin["bind_rig_transform"], dtype=np.float64) ).astype(np.float32) # [J, 4, 4] def _build_skin_blob(): """Pack the static skin data into one gzip+base64 blob (loaded once by the browser). Layout: bind_vertices f32[V,3] | faces u32[F,3] | lbs_idx u8[V,W] | lbs_wt f32[V,W].""" bind_v = np.asarray(_skin["bind_vertices"], dtype=np.float32) faces = np.asarray(_skin["faces"], dtype=np.uint32) idx = np.asarray(_skin["lbs_indices"], dtype=np.uint8) wt = np.asarray(_skin["lbs_weights"], dtype=np.float32) raw = ( np.ascontiguousarray(bind_v).tobytes() + np.ascontiguousarray(faces).tobytes() + np.ascontiguousarray(idx).tobytes() + np.ascontiguousarray(wt).tobytes() ) meta = {"V": int(bind_v.shape[0]), "F": int(faces.shape[0]), "W": int(idx.shape[1])} return base64.b64encode(gzip.compress(raw, 6)).decode("ascii"), meta SKIN_B64, SKIN_META = _build_skin_blob() print(f"CoreSkin ready: {SKIN_META['V']} verts / {SKIN_META['F']} faces, " f"blob {len(SKIN_B64) // 1024} KB", flush=True) def _joint_affines_human(global_rot_mats: np.ndarray, posed_joints: np.ndarray) -> str: """Per-frame joint affine matrices A = fk @ bind_rig_inv, base64 f32 [T,J,12]. global_rot_mats: [T, J, 3, 3]; posed_joints: [T, J, 3].""" T, J = posed_joints.shape[:2] fk = np.tile(np.eye(4, dtype=np.float32), (T, J, 1, 1)) fk[..., :3, :3] = global_rot_mats.astype(np.float32) fk[..., :3, 3] = posed_joints.astype(np.float32) A = (fk @ BIND_RIG_INV)[..., :3, :] # [T, J, 3, 4] A = np.ascontiguousarray(A.reshape(T, J, 12).astype(np.float32)) return base64.b64encode(A.tobytes()).decode("ascii") # ----------------------------------------------------------------------------- # ROBOT rig: G1 robot mesh rig (rigid per-joint STL meshes). # # Unlike the human skeleton (rendered with one skinned body mesh via LBS), the # Unitree G1 robot is rendered by attaching a rigid STL mesh to each articulated # joint. This mirrors ardy/viz/g1_rig.py (G1MeshRig): each mesh has a local # transform (geom_pos, geom_rot) relative to its joint, read from the MuJoCo # g1.xml, plus a coordinate change from MuJoCo to ARDY axes. We precompute — for # each mesh — its geometry PRE-TRANSFORMED into the joint-local frame # (v' = geom_rot @ v + geom_pos), so at render time the browser just applies the # per-frame joint transform: world_v = joint_pos + joint_rot @ v'. # ----------------------------------------------------------------------------- # G1 joint -> STL mesh mapping (mirrors ardy/viz/g1_rig.py G1_MESH_JOINT_MAP). G1_MESH_JOINT_MAP = { "pelvis_skel": ["pelvis.STL", "pelvis_contour_link.STL"], "left_hip_pitch_skel": ["left_hip_pitch_link.STL"], "left_hip_roll_skel": ["left_hip_roll_link.STL"], "left_hip_yaw_skel": ["left_hip_yaw_link.STL"], "left_knee_skel": ["left_knee_link.STL"], "left_ankle_pitch_skel": ["left_ankle_pitch_link.STL"], "left_ankle_roll_skel": ["left_ankle_roll_link.STL"], "right_hip_pitch_skel": ["right_hip_pitch_link.STL"], "right_hip_roll_skel": ["right_hip_roll_link.STL"], "right_hip_yaw_skel": ["right_hip_yaw_link.STL"], "right_knee_skel": ["right_knee_link.STL"], "right_ankle_pitch_skel": ["right_ankle_pitch_link.STL"], "right_ankle_roll_skel": ["right_ankle_roll_link.STL"], "waist_yaw_skel": ["waist_yaw_link_rev_1_0.STL", "waist_yaw_link.STL"], "waist_roll_skel": ["waist_roll_link_rev_1_0.STL", "waist_roll_link.STL"], "waist_pitch_skel": [ "torso_link_rev_1_0.STL", "torso_link.STL", "logo_link.STL", "head_link.STL", ], "left_shoulder_pitch_skel": ["left_shoulder_pitch_link.STL"], "left_shoulder_roll_skel": ["left_shoulder_roll_link.STL"], "left_shoulder_yaw_skel": ["left_shoulder_yaw_link.STL"], "left_elbow_skel": ["left_elbow_link.STL"], "left_wrist_roll_skel": ["left_wrist_roll_link.STL"], "left_wrist_pitch_skel": ["left_wrist_pitch_link.STL"], "left_wrist_yaw_skel": ["left_wrist_yaw_link.STL", "left_rubber_hand.STL"], "right_shoulder_pitch_skel": ["right_shoulder_pitch_link.STL"], "right_shoulder_roll_skel": ["right_shoulder_roll_link.STL"], "right_shoulder_yaw_skel": ["right_shoulder_yaw_link.STL"], "right_elbow_skel": ["right_elbow_link.STL"], "right_wrist_roll_skel": ["right_wrist_roll_link.STL"], "right_wrist_pitch_skel": ["right_wrist_pitch_link.STL"], "right_wrist_yaw_skel": ["right_wrist_yaw_link.STL", "right_rubber_hand.STL"], } _ROBOT_SKEL = RIGS["robot"]["skeleton"] _MUJOCO_TO_ARDY = np.array( [[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 0.0, 0.0]], dtype=np.float64 ) _G1_SKEL_DIR = Path(_ROBOT_SKEL.folder) _G1_MESH_DIR = _G1_SKEL_DIR / "meshes" / "g1" _G1_XML = _G1_SKEL_DIR / "xml" / "g1.xml" def _quat_wxyz_to_matrix(wxyz: np.ndarray) -> np.ndarray: w, x, y, z = wxyz n = np.sqrt(w * w + x * x + y * y + z * z) if n < 1e-12: return np.eye(3) w, x, y, z = w / n, x / n, y / n, z / n return np.array( [ [1 - 2 * (y * y + z * z), 2 * (x * y - z * w), 2 * (x * z + y * w)], [2 * (x * y + z * w), 1 - 2 * (x * x + z * z), 2 * (y * z - x * w)], [2 * (x * z - y * w), 2 * (y * z + x * w), 1 - 2 * (x * x + y * y)], ], dtype=np.float64, ) def _mesh_local_transforms() -> dict: """mesh_file -> (geom_pos[3], geom_rot[3x3]) in ARDY axes, parsed from g1.xml.""" if not _G1_XML.exists(): return {} root = ET.parse(_G1_XML).getroot() file_to_name = {} for mesh in root.findall(".//asset/mesh"): name, file = mesh.get("name"), mesh.get("file") if name and file: file_to_name[file] = name name_to_tf = {} for geom in root.findall(".//geom"): name = geom.get("mesh") if name is None: continue pos = geom.get("pos") quat = geom.get("quat") gp = np.zeros(3) if pos is None else np.array([float(v) for v in pos.split()]) gr_ = np.eye(3) if quat is None else _quat_wxyz_to_matrix( np.array([float(v) for v in quat.split()]) ) name_to_tf[name] = (gp, gr_) out = {} for file, name in file_to_name.items(): gp, gr_ = name_to_tf.get(name, (np.zeros(3), np.eye(3))) gp = _MUJOCO_TO_ARDY @ gp gr_ = _MUJOCO_TO_ARDY @ gr_ @ _MUJOCO_TO_ARDY.T out[file] = (gp, gr_) return out def _build_g1_mesh_blob(): """Pack all rigid G1 meshes, pre-transformed into joint-local frame, into one gzip+base64 blob. Returns (b64, meta). meta.parts lists per-mesh {joint, v_off, v_cnt}. Layout: all verts f32[Vtot,3] then all faces u32[Ftot,3] (face indices are GLOBAL into the concatenated vertex array).""" skeleton = _ROBOT_SKEL local_tf = _mesh_local_transforms() all_v = [] all_f = [] parts = [] v_cursor = 0 for joint_name, mesh_files in G1_MESH_JOINT_MAP.items(): if joint_name not in skeleton.bone_index: continue joint_idx = int(skeleton.bone_index[joint_name]) for mesh_file in mesh_files: mp = _G1_MESH_DIR / mesh_file if not mp.exists(): continue mesh = trimesh.load_mesh(str(mp), process=True) if isinstance(mesh, trimesh.Scene): mesh = trimesh.util.concatenate(mesh.dump()) verts = np.asarray(mesh.vertices, dtype=np.float64) @ _MUJOCO_TO_ARDY.T faces = np.asarray(mesh.faces, dtype=np.int64) gp, gr_ = local_tf.get(mesh_file, (np.zeros(3), np.eye(3))) # Pre-apply the mesh's joint-local transform: v' = geom_rot @ v + geom_pos. verts = (verts @ gr_.T) + gp vcnt = verts.shape[0] parts.append({"joint": joint_idx, "v_off": v_cursor, "v_cnt": vcnt}) all_v.append(verts.astype(np.float32)) all_f.append((faces + v_cursor).astype(np.uint32)) # global vertex indices v_cursor += vcnt V = np.concatenate(all_v, axis=0) if all_v else np.zeros((0, 3), np.float32) F = np.concatenate(all_f, axis=0) if all_f else np.zeros((0, 3), np.uint32) raw = np.ascontiguousarray(V).tobytes() + np.ascontiguousarray(F).tobytes() meta = {"Vtot": int(V.shape[0]), "Ftot": int(F.shape[0]), "parts": parts} b64 = base64.b64encode(gzip.compress(raw, 6)).decode("ascii") return b64, meta G1_MESH_B64, G1_MESH_META = _build_g1_mesh_blob() print( f"G1 rig ready: {len(G1_MESH_META['parts'])} meshes, " f"{G1_MESH_META['Vtot']} verts / {G1_MESH_META['Ftot']} faces, " f"blob {len(G1_MESH_B64) // 1024} KB", flush=True, ) def _joint_transforms_robot(global_rot_mats: np.ndarray, posed_joints: np.ndarray) -> str: """Per-frame joint affine matrices [R_global | pos], base64 f32 [T,J,12]. The browser applies world_v = pos + R_global @ v' per mesh (v' already in joint-local frame).""" T, J = posed_joints.shape[:2] A = np.zeros((T, J, 3, 4), dtype=np.float32) A[..., :3, :3] = global_rot_mats.astype(np.float32) A[..., :3, 3] = posed_joints.astype(np.float32) A = np.ascontiguousarray(A.reshape(T, J, 12).astype(np.float32)) return base64.b64encode(A.tobytes()).decode("ascii") # --- Autoregressive generation with a persistable latent state --------------- # ARDY is autoregressive: it generates one `gen_horizon_len`-frame window at a # time, conditioned on a history of previous frames. `autoregressive_step` is # the streaming primitive — it returns the *normalized motion-feature tensor* # for (history + new window), which can be fed straight back in as the next # window's history. We thread that tensor to (a) fill a requested clip length # and (b) CONTINUE a clip with a new prompt, exactly like the reference # interactive demo. Persisting the tensor in a gr.State lets a second "Continue" # click resume from where the first clip ended, on the same character. def _generate_sequence(rig_key, prompt, num_new_frames, steps, cfg_weight, init_tensor): """Run the AR loop for one prompt on the selected rig. `init_tensor`: normalized feature tensor [1, Th, D] on cuda (prior motion to continue), or None to start fresh. Returns the full normalized feature tensor.""" r = RIGS[rig_key] model = r["model"] gen_horizon = r["gen_horizon"] patch = r["patch"] hist_crop = r["hist_crop"] text_feat, text_pad_mask = model._encode_text([prompt]) motion_tensor = init_tensor target_new = max(1, int(np.ceil(num_new_frames / gen_horizon))) * gen_horizon produced = 0 while produced < target_new: if motion_tensor is None: history, hist_len = None, 0 else: hist_len = (min(motion_tensor.shape[1], hist_crop) // patch) * patch history = motion_tensor[:, motion_tensor.shape[1] - hist_len:] if hist_len else None hist_len = history.shape[1] if history is not None else 0 samples = model.autoregressive_step( num_frames=hist_len + gen_horizon, # exactly history + one window (no future) num_denoising_steps=steps, motion_mask=None, observed_motion=None, cfg_weight=float(cfg_weight), texts=None, text_feat=text_feat, text_pad_mask=text_pad_mask, init_history_sequence=history, init_global_translation=None, # first window -> defaults (origin / +Z heading) init_first_heading_angle=None, ) new_tail = samples[:, hist_len:] # the freshly generated window motion_tensor = new_tail if motion_tensor is None else torch.cat([motion_tensor, new_tail], dim=1) produced += new_tail.shape[1] return motion_tensor def _pack_payload(rig_key, motion_tensor, prompt, seed): """Decode a normalized feature tensor to the browser payload (per-frame joint affines + root ground-track), tagged with the rig so the viewer loads the correct skeleton/model.""" r = RIGS[rig_key] model = r["model"] with torch.no_grad(): out = to_numpy(model.motion_rep.inverse(motion_tensor, is_normalized=True)) posed = np.asarray(out["posed_joints"])[0] # [T, J, 3] global joint positions grm = np.asarray(out["global_rot_mats"])[0] # [T, J, 3, 3] global rotations if rig_key == "robot": affines = _joint_transforms_robot(grm, posed) else: affines = _joint_affines_human(grm, posed) return { "rig": rig_key, "fps": r["fps"], "num_frames": int(posed.shape[0]), "num_joints": int(posed.shape[1]), "affines": affines, # drives the browser rig "root": np.round(posed[:, r["root_idx"], :].astype(np.float32), 4).tolist(), "prompt": prompt, "seed": seed, } def _core_generate(rig, prompt, duration, diffusion_steps, cfg_weight, seed, randomize_seed, init_np): rig_key = _normalize_rig(rig) r = RIGS[rig_key] prompt = (prompt or "").strip() if not prompt: raise gr.Error("Please enter a text prompt describing the motion.") if randomize_seed: seed = random.randint(0, MAX_SEED) seed = int(seed) seed_everything(seed) steps = max(1, min(int(diffusion_steps), r["num_base_steps"])) num_new = max(r["patch"], int(round(float(duration) * r["fps"]))) init_tensor = None if init_np is None else torch.from_numpy(init_np).to("cuda") t0 = time.perf_counter() with torch.no_grad(): full = _generate_sequence(rig_key, prompt, num_new, steps, cfg_weight, init_tensor) payload = _pack_payload(rig_key, full, prompt, seed) tag = "continue" if init_np is not None else "generate" print(f"[{tag}:{rig_key}] '{prompt[:50]}' +{num_new}f -> {full.shape[1]}f total " f"steps={steps} seed={seed} {time.perf_counter() - t0:.1f}s", flush=True) return json.dumps(payload), seed, full.detach().cpu().numpy() @spaces.GPU def ui_generate(prompt, rig=DEFAULT_RIG, duration=5.0, diffusion_steps=NUM_BASE_STEPS, cfg_weight=2.0, seed=0, randomize_seed=True): """Start a fresh clip (resets the running sequence).""" return _core_generate(rig, prompt, duration, diffusion_steps, cfg_weight, seed, randomize_seed, None) @spaces.GPU def ui_continue(prompt, rig=DEFAULT_RIG, duration=5.0, diffusion_steps=NUM_BASE_STEPS, cfg_weight=2.0, seed=0, randomize_seed=True, state=None): """Append a new action, continuing from the previous clip's final pose.""" return _core_generate(rig, prompt, duration, diffusion_steps, cfg_weight, seed, randomize_seed, state) @spaces.GPU def generate_motion(prompt: str, rig: str = DEFAULT_RIG, duration: float = 5.0, diffusion_steps: int = NUM_BASE_STEPS, cfg_weight: float = 2.0, seed: int = 0, randomize_seed: bool = True) -> tuple[str, int]: """Generate a 3D motion clip from a text prompt with ARDY. Args: prompt: Natural-language description of the motion (e.g. "a person walks in a circle"). rig: Which character to animate — "human" (27-joint skeleton) or "robot" (Unitree G1). duration: Length of the generated motion in seconds. diffusion_steps: Number of denoising steps (1..num_base_steps). cfg_weight: Classifier-free-guidance weight for the text prompt. seed: Random seed for reproducibility. randomize_seed: If True, ignore `seed` and draw a fresh random one. Returns: A JSON string with the animated skeleton payload plus the seed used. """ payload_json, seed, _ = _core_generate( rig, prompt, duration, diffusion_steps, cfg_weight, seed, randomize_seed, None ) return payload_json, seed # ----------------------------------------------------------------------------- # Front-end: a self-contained Three.js playground, delivered as a Gradio-native # custom HTML component (Gradio 6 `gr.HTML` templates + js_on_load). # # The motion JSON is carried as the component's own `value` prop. `js_on_load` # imports three.js, builds the scene once, wires the controls, then registers a # `watch('value', ...)` callback that Gradio fires whenever the component is # updated as the output of a Python event (Generate button / Examples). # # Each payload is tagged with its `rig`. The viewer ships the static data for # BOTH rigs (human skin blob + G1 rigid-mesh blob) and switches at load time: # - "human": one skinned body mesh (linear-blend skinning in the browser). # - "robot": rigid per-joint G1 STL meshes posed by the joint transforms. # ----------------------------------------------------------------------------- PLAYER_TEMPLATE = """