Add rig-selection (human / Unitree G1 robot) to the playground
Browse files- .gitattributes +1 -0
- README.md +6 -4
- app.py +449 -136
- ardy/assets/skeletons/g1skel34/meshes/g1/head_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/left_ankle_pitch_link.STL +0 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/left_ankle_roll_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/left_elbow_link.STL +0 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/left_hip_pitch_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/left_hip_roll_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/left_hip_yaw_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/left_knee_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/left_rubber_hand.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/left_shoulder_pitch_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/left_shoulder_roll_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/left_shoulder_yaw_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/left_wrist_pitch_link.STL +0 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/left_wrist_roll_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/left_wrist_yaw_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/logo_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/pelvis.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/pelvis_contour_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/right_ankle_pitch_link.STL +0 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/right_ankle_roll_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/right_elbow_link.STL +0 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/right_hip_pitch_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/right_hip_roll_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/right_hip_yaw_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/right_knee_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/right_rubber_hand.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/right_shoulder_pitch_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/right_shoulder_roll_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/right_shoulder_yaw_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/right_wrist_pitch_link.STL +0 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/right_wrist_roll_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/right_wrist_yaw_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/torso_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/torso_link_rev_1_0.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/waist_roll_link.STL +0 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/waist_roll_link_rev_1_0.STL +0 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/waist_yaw_link.STL +3 -0
- ardy/assets/skeletons/g1skel34/meshes/g1/waist_yaw_link_rev_1_0.STL +3 -0
- requirements.txt +1 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
*.STL filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -17,11 +17,13 @@ pinned: false
|
|
| 17 |
Interactive **text-to-motion** demo for [ARDY](https://research.nvidia.com/labs/sil/projects/ardy/)
|
| 18 |
(*Autoregressive Diffusion with Hybrid Representation for Interactive Human Motion Generation*, NVIDIA, SIGGRAPH 2026).
|
| 19 |
|
| 20 |
-
|
| 21 |
-
play** it in the built-in Three.js playground (play/pause, timeline
|
| 22 |
-
loop, and a root-trajectory trail).
|
| 23 |
|
| 24 |
-
- **
|
|
|
|
|
|
|
| 25 |
- **Text encoder:** LLM2Vec-Llama-3-8B (gated base weights pulled via the Space `HF_TOKEN` secret).
|
| 26 |
- **Hardware:** ZeroGPU.
|
| 27 |
|
|
|
|
| 17 |
Interactive **text-to-motion** demo for [ARDY](https://research.nvidia.com/labs/sil/projects/ardy/)
|
| 18 |
(*Autoregressive Diffusion with Hybrid Representation for Interactive Human Motion Generation*, NVIDIA, SIGGRAPH 2026).
|
| 19 |
|
| 20 |
+
Pick a **rig** (human or robot), type a natural-language prompt, generate a 3D motion clip,
|
| 21 |
+
then **orbit, scrub, and play** it in the built-in Three.js playground (play/pause, timeline
|
| 22 |
+
scrubber, speed control, loop, and a root-trajectory trail).
|
| 23 |
|
| 24 |
+
- **Human rig:** `nvidia/ARDY-Core-RP-20FPS-Horizon40` — 27-joint "Core" skeleton at 20 fps.
|
| 25 |
+
- **Robot rig:** `nvidia/ARDY-G1-RP-25FPS-Horizon52` — 34-joint Unitree G1 humanoid robot at 25 fps
|
| 26 |
+
(same rig as the sibling Space `hugging-apps/ardy-g1-motion-generation`).
|
| 27 |
- **Text encoder:** LLM2Vec-Llama-3-8B (gated base weights pulled via the Space `HF_TOKEN` secret).
|
| 28 |
- **Hardware:** ZeroGPU.
|
| 29 |
|
app.py
CHANGED
|
@@ -10,11 +10,13 @@ import json
|
|
| 10 |
import random
|
| 11 |
import sys
|
| 12 |
import time
|
|
|
|
| 13 |
from pathlib import Path
|
| 14 |
|
| 15 |
import spaces # must precede torch / CUDA-touching imports
|
| 16 |
import torch
|
| 17 |
import numpy as np
|
|
|
|
| 18 |
import gradio as gr
|
| 19 |
|
| 20 |
# The vendored ARDY package lives next to this file.
|
|
@@ -25,7 +27,11 @@ from ardy.model.load_model import load_text_encoder # noqa: E402
|
|
| 25 |
from ardy.motion_rep.tools import length_to_mask # noqa: E402
|
| 26 |
from ardy.tools import seed_everything, to_numpy # noqa: E402
|
| 27 |
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
MAX_SEED = 2**31 - 1
|
| 30 |
|
| 31 |
# -----------------------------------------------------------------------------
|
|
@@ -43,64 +49,100 @@ MAX_SEED = 2**31 - 1
|
|
| 43 |
# with "No CUDA GPUs are available". Force every loader onto CPU by masking
|
| 44 |
# is_available() during construction, then restore it and .to("cuda") (which the
|
| 45 |
# ZeroGPU hijack packs to disk + streams into VRAM on the first request).
|
|
|
|
|
|
|
|
|
|
| 46 |
_real_cuda_available = torch.cuda.is_available
|
| 47 |
torch.cuda.is_available = lambda: False
|
| 48 |
try:
|
| 49 |
print("Loading ARDY text encoder (LLM2Vec-Llama-3-8B) on CPU…", flush=True)
|
| 50 |
_text_encoder = load_text_encoder(mode="local", device="cpu")
|
| 51 |
|
| 52 |
-
print("Loading ARDY motion model on CPU…", flush=True)
|
| 53 |
-
|
|
|
|
|
|
|
|
|
|
| 54 |
finally:
|
| 55 |
torch.cuda.is_available = _real_cuda_available
|
| 56 |
|
| 57 |
print("Moving models to CUDA (ZeroGPU-intercepted placement)…", flush=True)
|
| 58 |
_text_encoder = _text_encoder.to("cuda")
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
#
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
|
| 90 |
|
| 91 |
# -----------------------------------------------------------------------------
|
| 92 |
-
#
|
| 93 |
#
|
| 94 |
# The reference viz (ardy/viz/viser_utils.py) renders a smooth humanoid body by
|
| 95 |
# skinning a bind mesh with the per-frame *global* joint transforms:
|
| 96 |
# verts = CoreSkin.skin(global_rot_mats, posed_joints, rot_is_global=True)
|
| 97 |
-
#
|
| 98 |
-
#
|
| 99 |
-
#
|
| 100 |
# A[f,j] = fk[f,j] @ bind_rig_transform_inv[j] (fk = [R_global | pos])
|
| 101 |
-
# so the payload stays small (~0.1 MB/clip)
|
| 102 |
# -----------------------------------------------------------------------------
|
| 103 |
-
|
|
|
|
| 104 |
_skin = np.load(_SKIN_PATH)
|
| 105 |
BIND_RIG_INV = np.linalg.inv(
|
| 106 |
np.asarray(_skin["bind_rig_transform"], dtype=np.float64)
|
|
@@ -130,7 +172,7 @@ print(f"CoreSkin ready: {SKIN_META['V']} verts / {SKIN_META['F']} faces, "
|
|
| 130 |
f"blob {len(SKIN_B64) // 1024} KB", flush=True)
|
| 131 |
|
| 132 |
|
| 133 |
-
def
|
| 134 |
"""Per-frame joint affine matrices A = fk @ bind_rig_inv, base64 f32 [T,J,12].
|
| 135 |
|
| 136 |
global_rot_mats: [T, J, 3, 3]; posed_joints: [T, J, 3]."""
|
|
@@ -143,10 +185,179 @@ def _joint_affines(global_rot_mats: np.ndarray, posed_joints: np.ndarray) -> str
|
|
| 143 |
return base64.b64encode(A.tobytes()).decode("ascii")
|
| 144 |
|
| 145 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
# -----------------------------------------------------------------------------
|
| 147 |
# Inference
|
| 148 |
# -----------------------------------------------------------------------------
|
| 149 |
-
def _estimate_duration(prompt, duration=5.0, *args, **kwargs):
|
| 150 |
# Autoregressive: rough per-second-of-motion cost + text encoding overhead.
|
| 151 |
try:
|
| 152 |
d = float(duration)
|
|
@@ -162,26 +373,30 @@ def _estimate_duration(prompt, duration=5.0, *args, **kwargs):
|
|
| 162 |
# for (history + new window), which can be fed straight back in as the next
|
| 163 |
# window's history. We thread that tensor to (a) fill a requested clip length
|
| 164 |
# and (b) CONTINUE a clip with a new prompt, exactly like the reference
|
| 165 |
-
# interactive demo
|
| 166 |
-
#
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 173 |
motion_tensor = init_tensor
|
| 174 |
-
target_new = max(1, int(np.ceil(num_new_frames /
|
| 175 |
produced = 0
|
| 176 |
while produced < target_new:
|
| 177 |
if motion_tensor is None:
|
| 178 |
history, hist_len = None, 0
|
| 179 |
else:
|
| 180 |
-
hist_len = (min(motion_tensor.shape[1],
|
| 181 |
history = motion_tensor[:, motion_tensor.shape[1] - hist_len:] if hist_len else None
|
| 182 |
hist_len = history.shape[1] if history is not None else 0
|
| 183 |
-
samples =
|
| 184 |
-
num_frames=hist_len +
|
| 185 |
num_denoising_steps=steps,
|
| 186 |
motion_mask=None,
|
| 187 |
observed_motion=None,
|
|
@@ -199,25 +414,35 @@ def _generate_sequence(prompt, num_new_frames, steps, cfg_weight, init_tensor):
|
|
| 199 |
return motion_tensor
|
| 200 |
|
| 201 |
|
| 202 |
-
def _pack_payload(motion_tensor, prompt, seed):
|
| 203 |
"""Decode a normalized feature tensor to the browser payload (per-frame joint
|
| 204 |
-
affines
|
|
|
|
|
|
|
|
|
|
| 205 |
with torch.no_grad():
|
| 206 |
-
out = to_numpy(
|
| 207 |
posed = np.asarray(out["posed_joints"])[0] # [T, J, 3] global joint positions
|
| 208 |
grm = np.asarray(out["global_rot_mats"])[0] # [T, J, 3, 3] global rotations
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
return {
|
| 210 |
-
"
|
|
|
|
| 211 |
"num_frames": int(posed.shape[0]),
|
| 212 |
"num_joints": int(posed.shape[1]),
|
| 213 |
-
"affines":
|
| 214 |
-
"root": np.round(posed[:,
|
| 215 |
"prompt": prompt,
|
| 216 |
"seed": seed,
|
| 217 |
}
|
| 218 |
|
| 219 |
|
| 220 |
-
def _core_generate(prompt, duration, diffusion_steps, cfg_weight, seed, randomize_seed, init_np):
|
|
|
|
|
|
|
| 221 |
prompt = (prompt or "").strip()
|
| 222 |
if not prompt:
|
| 223 |
raise gr.Error("Please enter a text prompt describing the motion.")
|
|
@@ -225,41 +450,43 @@ def _core_generate(prompt, duration, diffusion_steps, cfg_weight, seed, randomiz
|
|
| 225 |
seed = random.randint(0, MAX_SEED)
|
| 226 |
seed = int(seed)
|
| 227 |
seed_everything(seed)
|
| 228 |
-
steps = max(1, min(int(diffusion_steps),
|
| 229 |
-
num_new = max(
|
| 230 |
init_tensor = None if init_np is None else torch.from_numpy(init_np).to("cuda")
|
| 231 |
|
| 232 |
t0 = time.perf_counter()
|
| 233 |
with torch.no_grad():
|
| 234 |
-
full = _generate_sequence(prompt, num_new, steps, cfg_weight, init_tensor)
|
| 235 |
-
payload = _pack_payload(full, prompt, seed)
|
| 236 |
tag = "continue" if init_np is not None else "generate"
|
| 237 |
-
print(f"[{tag}] '{prompt[:50]}' +{num_new}f -> {full.shape[1]}f total "
|
| 238 |
f"steps={steps} seed={seed} {time.perf_counter() - t0:.1f}s", flush=True)
|
| 239 |
return json.dumps(payload), seed, full.detach().cpu().numpy()
|
| 240 |
|
| 241 |
|
| 242 |
@spaces.GPU(duration=_estimate_duration)
|
| 243 |
-
def ui_generate(prompt, duration=5.0, diffusion_steps=NUM_BASE_STEPS,
|
| 244 |
cfg_weight=2.0, seed=0, randomize_seed=True):
|
| 245 |
"""Start a fresh clip (resets the running sequence)."""
|
| 246 |
-
return _core_generate(prompt, duration, diffusion_steps, cfg_weight, seed, randomize_seed, None)
|
| 247 |
|
| 248 |
|
| 249 |
@spaces.GPU(duration=_estimate_duration)
|
| 250 |
-
def ui_continue(prompt, duration=5.0, diffusion_steps=NUM_BASE_STEPS,
|
| 251 |
cfg_weight=2.0, seed=0, randomize_seed=True, state=None):
|
| 252 |
"""Append a new action, continuing from the previous clip's final pose."""
|
| 253 |
-
return _core_generate(prompt, duration, diffusion_steps, cfg_weight, seed, randomize_seed, state)
|
| 254 |
|
| 255 |
|
| 256 |
@spaces.GPU(duration=_estimate_duration)
|
| 257 |
-
def generate_motion(prompt: str,
|
| 258 |
-
|
| 259 |
-
|
|
|
|
| 260 |
|
| 261 |
Args:
|
| 262 |
prompt: Natural-language description of the motion (e.g. "a person walks in a circle").
|
|
|
|
| 263 |
duration: Length of the generated motion in seconds.
|
| 264 |
diffusion_steps: Number of denoising steps (1..num_base_steps).
|
| 265 |
cfg_weight: Classifier-free-guidance weight for the text prompt.
|
|
@@ -270,29 +497,26 @@ def generate_motion(prompt: str, duration: float = 5.0, diffusion_steps: int = N
|
|
| 270 |
A JSON string with the animated skeleton payload plus the seed used.
|
| 271 |
"""
|
| 272 |
payload_json, seed, _ = _core_generate(
|
| 273 |
-
prompt, duration, diffusion_steps, cfg_weight, seed, randomize_seed, None
|
| 274 |
)
|
| 275 |
return payload_json, seed
|
| 276 |
|
| 277 |
|
| 278 |
# -----------------------------------------------------------------------------
|
| 279 |
-
# Front-end: a self-contained Three.js
|
| 280 |
-
#
|
| 281 |
#
|
| 282 |
# The motion JSON is carried as the component's own `value` prop. `js_on_load`
|
| 283 |
# imports three.js, builds the scene once, wires the controls, then registers a
|
| 284 |
# `watch('value', ...)` callback that Gradio fires whenever the component is
|
| 285 |
-
# updated as the output of a Python event (Generate button / Examples).
|
| 286 |
-
#
|
| 287 |
-
#
|
| 288 |
-
#
|
| 289 |
-
#
|
| 290 |
-
#
|
| 291 |
# -----------------------------------------------------------------------------
|
| 292 |
|
| 293 |
-
# Static markup for the player. Lives *inside* the component so js_on_load can
|
| 294 |
-
# scope every query to `element` (its own DOM root) — no global element IDs, so
|
| 295 |
-
# nothing races or collides across rerenders.
|
| 296 |
PLAYER_TEMPLATE = """
|
| 297 |
<div class="ardy-playground">
|
| 298 |
<div class="ardy-canvas-wrap">
|
|
@@ -318,8 +542,6 @@ PLAYER_TEMPLATE = """
|
|
| 318 |
"""
|
| 319 |
|
| 320 |
# css_template rules are auto-scoped to this component by Gradio.
|
| 321 |
-
# Light "studio" look matching the ARDY project page: white stage, faint grid,
|
| 322 |
-
# soft contact shadow, light-blue body.
|
| 323 |
PLAYER_CSS_TEMPLATE = """
|
| 324 |
.ardy-playground { width: 100%; }
|
| 325 |
.ardy-canvas-wrap {
|
|
@@ -347,30 +569,34 @@ PLAYER_CSS_TEMPLATE = """
|
|
| 347 |
.ardy-caption { margin-top:6px; font-size:13px; color:#667085; }
|
| 348 |
"""
|
| 349 |
|
| 350 |
-
# Page-level CSS (container width + dark-mode text fix). The player's own
|
| 351 |
-
# styles live in PLAYER_CSS_TEMPLATE, auto-scoped to the component by Gradio.
|
| 352 |
APP_CSS = """
|
| 353 |
#col-container { max-width: 1200px; margin: 0 auto; }
|
| 354 |
.dark .gradio-container { color: var(--body-text-color); }
|
| 355 |
"""
|
| 356 |
|
| 357 |
# Runs once, when the component first renders. `element`, `props`, and `watch`
|
| 358 |
-
# are injected by Gradio. We import three.js, decode the static skin data
|
| 359 |
-
#
|
| 360 |
-
#
|
| 361 |
-
#
|
| 362 |
-
#
|
| 363 |
PLAYER_JS_ON_LOAD = r"""
|
| 364 |
const root = element;
|
| 365 |
const q = (sel) => root.querySelector(sel);
|
| 366 |
|
| 367 |
const state = {
|
| 368 |
-
ready:false,
|
| 369 |
THREE:null, OrbitControls:null,
|
| 370 |
renderer:null, scene:null, camera:null, controls:null,
|
| 371 |
-
|
| 372 |
data:null, verts:null, frame:0, playing:false, lastT:0,
|
| 373 |
speed:1, loop:true, trail:false, pending:null,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 374 |
};
|
| 375 |
|
| 376 |
// --- binary helpers ---------------------------------------------------------
|
|
@@ -385,7 +611,7 @@ async function gunzip(bytes){
|
|
| 385 |
return buf;
|
| 386 |
}
|
| 387 |
|
| 388 |
-
// Decode the one-time static skin blob into typed arrays.
|
| 389 |
async function decodeSkin(){
|
| 390 |
const buf = await gunzip(b64ToBytes(ARDY_SKIN_B64));
|
| 391 |
const V = ARDY_SKIN_META.V, F = ARDY_SKIN_META.F, W = ARDY_SKIN_META.W;
|
|
@@ -397,7 +623,18 @@ async function decodeSkin(){
|
|
| 397 |
return {V, F, W, bindV, faces, idx, wt};
|
| 398 |
}
|
| 399 |
|
| 400 |
-
//
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 401 |
// A[f,j] is a 3x4 affine (row-major, 12 floats); posed vertex =
|
| 402 |
// sum_k w_k * A[idx_k] @ [bind_x, bind_y, bind_z, 1].
|
| 403 |
function skinAllFrames(A, T, J){
|
|
@@ -423,7 +660,35 @@ function skinAllFrames(A, T, J){
|
|
| 423 |
return frames;
|
| 424 |
}
|
| 425 |
|
| 426 |
-
// -
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 427 |
function initScene(){
|
| 428 |
const THREE = state.THREE, OrbitControls = state.OrbitControls;
|
| 429 |
const wrap = q(".ardy-canvas-wrap");
|
|
@@ -448,7 +713,6 @@ function initScene(){
|
|
| 448 |
scene.add(key);
|
| 449 |
scene.add(new THREE.DirectionalLight(0xeef2ff, 0.35).translateX(-4).translateZ(-2));
|
| 450 |
|
| 451 |
-
// Soft contact shadow on an invisible ground plane.
|
| 452 |
const ground = new THREE.Mesh(
|
| 453 |
new THREE.PlaneGeometry(40, 40),
|
| 454 |
new THREE.ShadowMaterial({opacity:0.16})
|
|
@@ -469,16 +733,42 @@ function initScene(){
|
|
| 469 |
animate();
|
| 470 |
}
|
| 471 |
|
| 472 |
-
|
| 473 |
-
|
| 474 |
-
|
| 475 |
-
const
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 482 |
}
|
| 483 |
|
| 484 |
function setFrame(f){
|
|
@@ -486,11 +776,13 @@ function setFrame(f){
|
|
| 486 |
const T = state.data.num_frames;
|
| 487 |
f = Math.max(0, Math.min(T-1, f|0));
|
| 488 |
state.frame = f;
|
| 489 |
-
const
|
|
|
|
|
|
|
| 490 |
pos.array.set(state.verts[f]);
|
| 491 |
pos.needsUpdate = true;
|
| 492 |
-
|
| 493 |
-
|
| 494 |
updateTrail();
|
| 495 |
const scrub = q(".ardy-scrub"); if(scrub) scrub.value = f;
|
| 496 |
const lbl = q(".ardy-frame"); if(lbl) lbl.textContent = (f+1)+" / "+T;
|
|
@@ -543,19 +835,23 @@ function setPlaying(p){
|
|
| 543 |
|
| 544 |
function loadData(data){
|
| 545 |
initScene();
|
| 546 |
-
|
|
|
|
|
|
|
| 547 |
state.data = data;
|
| 548 |
-
// Decode per-frame affines and pre-
|
| 549 |
const T = data.num_frames, J = data.num_joints;
|
| 550 |
const Abytes = b64ToBytes(data.affines);
|
| 551 |
const A = new Float32Array(Abytes.buffer, Abytes.byteOffset, Abytes.byteLength/4);
|
| 552 |
-
state.verts = skinAllFrames(A, T, J);
|
| 553 |
if(state.trailLine){ state.scene.remove(state.trailLine); state.trailLine.geometry.dispose(); state.trailLine=null; }
|
| 554 |
const scrub = q(".ardy-scrub"); if(scrub){ scrub.max = T-1; scrub.value = 0; }
|
| 555 |
const hint = q(".ardy-hint"); if(hint) hint.style.display = "none";
|
| 556 |
const cap = q(".ardy-caption");
|
| 557 |
-
if(cap) cap.textContent = '"' + data.prompt + '" · ' +
|
|
|
|
| 558 |
root.dataset.ardyLoaded = "1";
|
|
|
|
| 559 |
root.dataset.ardyFrames = String(T);
|
| 560 |
setFrame(0);
|
| 561 |
setPlaying(true);
|
|
@@ -563,7 +859,7 @@ function loadData(data){
|
|
| 563 |
|
| 564 |
function applyPayload(payload){
|
| 565 |
if(!payload) return;
|
| 566 |
-
if(!state.ready){ state.pending = payload; return; } // three.js /
|
| 567 |
try { loadData(JSON.parse(payload)); }
|
| 568 |
catch(e){ console.error("ARDY playground load error", e); }
|
| 569 |
}
|
|
@@ -580,23 +876,28 @@ function wireControls(){
|
|
| 580 |
}
|
| 581 |
|
| 582 |
// Gradio-native hand-off: render whenever the component's value prop updates as
|
| 583 |
-
// the output of a Python event (Generate / Examples).
|
| 584 |
if (typeof watch === "function") {
|
| 585 |
watch("value", () => applyPayload(props.value));
|
| 586 |
}
|
| 587 |
|
| 588 |
// Import three.js (esm.sh, not jsDelivr: the OrbitControls addon has an internal
|
| 589 |
// bare `import ... from "three"` that a browser dynamic import() can't resolve
|
| 590 |
-
// without an import map; esm.sh rewrites it and dedupes three) and decode
|
| 591 |
-
//
|
| 592 |
Promise.all([
|
| 593 |
import("https://esm.sh/three@0.160.0"),
|
| 594 |
import("https://esm.sh/three@0.160.0/examples/jsm/controls/OrbitControls.js"),
|
| 595 |
decodeSkin(),
|
| 596 |
-
|
|
|
|
| 597 |
state.THREE = THREE;
|
| 598 |
state.OrbitControls = oc.OrbitControls;
|
| 599 |
state.skin = skin;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 600 |
state.ready = true;
|
| 601 |
wireControls();
|
| 602 |
initScene();
|
|
@@ -606,12 +907,12 @@ Promise.all([
|
|
| 606 |
"""
|
| 607 |
|
| 608 |
EXAMPLES = [
|
| 609 |
-
["A person walks forward confidently.", 5.0],
|
| 610 |
-
["A person walks in a circle.", 6.0],
|
| 611 |
-
["A person jumps up and down.", 4.0],
|
| 612 |
-
["
|
| 613 |
-
["
|
| 614 |
-
["
|
| 615 |
]
|
| 616 |
|
| 617 |
with gr.Blocks() as demo:
|
|
@@ -621,7 +922,8 @@ with gr.Blocks() as demo:
|
|
| 621 |
# 🕺 ARDY Motion Playground
|
| 622 |
Interactive text-to-motion with **[ARDY](https://research.nvidia.com/labs/sil/projects/ardy/)**
|
| 623 |
(Autoregressive Diffusion with Hybrid Representation) by NVIDIA.
|
| 624 |
-
|
|
|
|
| 625 |
Chain actions with **Continue ▸** — the same character keeps going from where it stopped.
|
| 626 |
"""
|
| 627 |
)
|
|
@@ -630,6 +932,14 @@ with gr.Blocks() as demo:
|
|
| 630 |
# resume the same character from the end of the previous clip.
|
| 631 |
seq_state = gr.State(None)
|
| 632 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 633 |
prompt = gr.Textbox(
|
| 634 |
label="Motion prompt",
|
| 635 |
placeholder="e.g. a person walks in a circle then waves",
|
|
@@ -641,18 +951,20 @@ with gr.Blocks() as demo:
|
|
| 641 |
|
| 642 |
# The interactive 3D playground — a Gradio-native custom HTML component.
|
| 643 |
# Its `value` (the motion JSON) is set directly by the Generate handler;
|
| 644 |
-
# `watch('value', ...)` in js_on_load renders it.
|
| 645 |
-
#
|
| 646 |
# prepended to js_on_load; the per-frame affines ride in each payload.
|
| 647 |
-
|
| 648 |
f'const ARDY_SKIN_B64="{SKIN_B64}";\n'
|
| 649 |
f"const ARDY_SKIN_META={json.dumps(SKIN_META)};\n"
|
|
|
|
|
|
|
| 650 |
)
|
| 651 |
player = gr.HTML(
|
| 652 |
value="",
|
| 653 |
html_template=PLAYER_TEMPLATE,
|
| 654 |
css_template=PLAYER_CSS_TEMPLATE,
|
| 655 |
-
js_on_load=
|
| 656 |
elem_id="ardy-player",
|
| 657 |
)
|
| 658 |
|
|
@@ -668,7 +980,7 @@ with gr.Blocks() as demo:
|
|
| 668 |
|
| 669 |
gr.Examples(
|
| 670 |
examples=EXAMPLES,
|
| 671 |
-
inputs=[prompt, duration],
|
| 672 |
outputs=[player, seed, seq_state],
|
| 673 |
fn=ui_generate,
|
| 674 |
cache_examples=False,
|
|
@@ -677,7 +989,8 @@ with gr.Blocks() as demo:
|
|
| 677 |
|
| 678 |
gr.Markdown(
|
| 679 |
"""
|
| 680 |
-
<small>
|
|
|
|
| 681 |
Text encoder: LLM2Vec-Llama-3-8B. Post-processing (foot-skate cleanup) is disabled in this demo.
|
| 682 |
Motion is generated autoregressively; longer clips take longer.
|
| 683 |
**Generate** starts a new clip; **Continue ▸** keeps the same character going,
|
|
@@ -685,10 +998,10 @@ with gr.Blocks() as demo:
|
|
| 685 |
"""
|
| 686 |
)
|
| 687 |
|
| 688 |
-
_gen_inputs = [prompt, duration, diffusion_steps, cfg_weight, seed, randomize_seed]
|
| 689 |
# Generate starts fresh; Continue resumes from the running latent state.
|
| 690 |
# The payload is written straight into the player's `value`; its js_on_load
|
| 691 |
-
# `watch('value', ...)` renders it
|
| 692 |
run.click(fn=ui_generate, inputs=_gen_inputs,
|
| 693 |
outputs=[player, seed, seq_state], api_name=False)
|
| 694 |
cont.click(fn=ui_continue, inputs=_gen_inputs + [seq_state],
|
|
|
|
| 10 |
import random
|
| 11 |
import sys
|
| 12 |
import time
|
| 13 |
+
import xml.etree.ElementTree as ET
|
| 14 |
from pathlib import Path
|
| 15 |
|
| 16 |
import spaces # must precede torch / CUDA-touching imports
|
| 17 |
import torch
|
| 18 |
import numpy as np
|
| 19 |
+
import trimesh
|
| 20 |
import gradio as gr
|
| 21 |
|
| 22 |
# The vendored ARDY package lives next to this file.
|
|
|
|
| 27 |
from ardy.motion_rep.tools import length_to_mask # noqa: E402
|
| 28 |
from ardy.tools import seed_everything, to_numpy # noqa: E402
|
| 29 |
|
| 30 |
+
# Two rigs are offered in the playground:
|
| 31 |
+
# "human" -> ARDY-Core-RP-20FPS-Horizon40, 27-joint skeleton @ 20 fps
|
| 32 |
+
# "robot" -> ARDY-G1-RP-25FPS-Horizon52, 34-joint Unitree G1 robot @ 25 fps
|
| 33 |
+
# The robot rig matches the sibling Space hugging-apps/ardy-g1-motion-generation.
|
| 34 |
+
DEFAULT_RIG = "human"
|
| 35 |
MAX_SEED = 2**31 - 1
|
| 36 |
|
| 37 |
# -----------------------------------------------------------------------------
|
|
|
|
| 49 |
# with "No CUDA GPUs are available". Force every loader onto CPU by masking
|
| 50 |
# is_available() during construction, then restore it and .to("cuda") (which the
|
| 51 |
# ZeroGPU hijack packs to disk + streams into VRAM on the first request).
|
| 52 |
+
#
|
| 53 |
+
# A single text encoder is built once and shared across both motion models
|
| 54 |
+
# (load_text_encoder is explicitly designed for this — see its docstring).
|
| 55 |
_real_cuda_available = torch.cuda.is_available
|
| 56 |
torch.cuda.is_available = lambda: False
|
| 57 |
try:
|
| 58 |
print("Loading ARDY text encoder (LLM2Vec-Llama-3-8B) on CPU…", flush=True)
|
| 59 |
_text_encoder = load_text_encoder(mode="local", device="cpu")
|
| 60 |
|
| 61 |
+
print("Loading ARDY human (core) motion model on CPU…", flush=True)
|
| 62 |
+
MODEL_HUMAN = load_model("core", device="cpu", text_encoder=_text_encoder)
|
| 63 |
+
|
| 64 |
+
print("Loading ARDY robot (G1) motion model on CPU…", flush=True)
|
| 65 |
+
MODEL_ROBOT = load_model("g1", device="cpu", text_encoder=_text_encoder)
|
| 66 |
finally:
|
| 67 |
torch.cuda.is_available = _real_cuda_available
|
| 68 |
|
| 69 |
print("Moving models to CUDA (ZeroGPU-intercepted placement)…", flush=True)
|
| 70 |
_text_encoder = _text_encoder.to("cuda")
|
| 71 |
+
for _m in (MODEL_HUMAN, MODEL_ROBOT):
|
| 72 |
+
_m.to("cuda")
|
| 73 |
+
# self.device was captured at construction (device="cpu"); generation creates
|
| 74 |
+
# many tensors on it, so retarget it to cuda to match the moved weights.
|
| 75 |
+
_m.device = "cuda"
|
| 76 |
+
_m.eval()
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def _rig_params(model):
|
| 80 |
+
fps = float(model.motion_rep.fps)
|
| 81 |
+
skeleton = model.skeleton
|
| 82 |
+
parents = skeleton.joint_parents.cpu().numpy().astype(int).tolist()
|
| 83 |
+
patch = model.num_frames_per_token
|
| 84 |
+
gen_horizon = model.gen_horizon_len
|
| 85 |
+
num_base_steps = int(model.diffusion.num_base_steps)
|
| 86 |
+
# History carried between autoregressive windows. The reference streaming demo
|
| 87 |
+
# keeps this SHORT so a new prompt takes effect within a window.
|
| 88 |
+
hist_crop = max(patch, (4 // patch) * patch)
|
| 89 |
+
root_idx = parents.index(-1) if -1 in parents else 0
|
| 90 |
+
return {
|
| 91 |
+
"model": model,
|
| 92 |
+
"fps": fps,
|
| 93 |
+
"skeleton": skeleton,
|
| 94 |
+
"parents": parents,
|
| 95 |
+
"patch": patch,
|
| 96 |
+
"gen_horizon": gen_horizon,
|
| 97 |
+
"num_base_steps": num_base_steps,
|
| 98 |
+
"hist_crop": hist_crop,
|
| 99 |
+
"root_idx": root_idx,
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
RIGS = {
|
| 104 |
+
"human": _rig_params(MODEL_HUMAN),
|
| 105 |
+
"robot": _rig_params(MODEL_ROBOT),
|
| 106 |
+
}
|
| 107 |
+
# The largest base-step count across rigs bounds the diffusion-steps slider.
|
| 108 |
+
NUM_BASE_STEPS = max(r["num_base_steps"] for r in RIGS.values())
|
| 109 |
+
for _name, _r in RIGS.items():
|
| 110 |
+
print(
|
| 111 |
+
f"[{_name}] rig ready: {_r['skeleton'].nbjoints} joints, {_r['fps']} fps, "
|
| 112 |
+
f"horizon {_r['gen_horizon']}, patch {_r['patch']}, "
|
| 113 |
+
f"hist_crop {_r['hist_crop']}, base_steps {_r['num_base_steps']}",
|
| 114 |
+
flush=True,
|
| 115 |
+
)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def _normalize_rig(rig) -> str:
|
| 119 |
+
"""Map any UI/API rig value onto a valid RIGS key ('human' | 'robot')."""
|
| 120 |
+
if rig is None:
|
| 121 |
+
return DEFAULT_RIG
|
| 122 |
+
key = str(rig).strip().lower()
|
| 123 |
+
if key in RIGS:
|
| 124 |
+
return key
|
| 125 |
+
if key.startswith("hum") or "core" in key or "person" in key:
|
| 126 |
+
return "human"
|
| 127 |
+
if key.startswith("rob") or "g1" in key or "unitree" in key:
|
| 128 |
+
return "robot"
|
| 129 |
+
return DEFAULT_RIG
|
| 130 |
|
| 131 |
|
| 132 |
# -----------------------------------------------------------------------------
|
| 133 |
+
# HUMAN rig: skinned body mesh (ARDY "CoreSkin" linear-blend skinning).
|
| 134 |
#
|
| 135 |
# The reference viz (ardy/viz/viser_utils.py) renders a smooth humanoid body by
|
| 136 |
# skinning a bind mesh with the per-frame *global* joint transforms:
|
| 137 |
# verts = CoreSkin.skin(global_rot_mats, posed_joints, rot_is_global=True)
|
| 138 |
+
# The browser holds the static skin data (bind vertices / faces / LBS
|
| 139 |
+
# indices+weights) and does the per-vertex blend, while the server sends only the
|
| 140 |
+
# tiny per-frame joint affine matrices
|
| 141 |
# A[f,j] = fk[f,j] @ bind_rig_transform_inv[j] (fk = [R_global | pos])
|
| 142 |
+
# so the payload stays small (~0.1 MB/clip).
|
| 143 |
# -----------------------------------------------------------------------------
|
| 144 |
+
_HUMAN_SKEL = RIGS["human"]["skeleton"]
|
| 145 |
+
_SKIN_PATH = Path(_HUMAN_SKEL.folder) / "skin_standard.npz"
|
| 146 |
_skin = np.load(_SKIN_PATH)
|
| 147 |
BIND_RIG_INV = np.linalg.inv(
|
| 148 |
np.asarray(_skin["bind_rig_transform"], dtype=np.float64)
|
|
|
|
| 172 |
f"blob {len(SKIN_B64) // 1024} KB", flush=True)
|
| 173 |
|
| 174 |
|
| 175 |
+
def _joint_affines_human(global_rot_mats: np.ndarray, posed_joints: np.ndarray) -> str:
|
| 176 |
"""Per-frame joint affine matrices A = fk @ bind_rig_inv, base64 f32 [T,J,12].
|
| 177 |
|
| 178 |
global_rot_mats: [T, J, 3, 3]; posed_joints: [T, J, 3]."""
|
|
|
|
| 185 |
return base64.b64encode(A.tobytes()).decode("ascii")
|
| 186 |
|
| 187 |
|
| 188 |
+
# -----------------------------------------------------------------------------
|
| 189 |
+
# ROBOT rig: G1 robot mesh rig (rigid per-joint STL meshes).
|
| 190 |
+
#
|
| 191 |
+
# Unlike the human skeleton (rendered with one skinned body mesh via LBS), the
|
| 192 |
+
# Unitree G1 robot is rendered by attaching a rigid STL mesh to each articulated
|
| 193 |
+
# joint. This mirrors ardy/viz/g1_rig.py (G1MeshRig): each mesh has a local
|
| 194 |
+
# transform (geom_pos, geom_rot) relative to its joint, read from the MuJoCo
|
| 195 |
+
# g1.xml, plus a coordinate change from MuJoCo to ARDY axes. We precompute — for
|
| 196 |
+
# each mesh — its geometry PRE-TRANSFORMED into the joint-local frame
|
| 197 |
+
# (v' = geom_rot @ v + geom_pos), so at render time the browser just applies the
|
| 198 |
+
# per-frame joint transform: world_v = joint_pos + joint_rot @ v'.
|
| 199 |
+
# -----------------------------------------------------------------------------
|
| 200 |
+
# G1 joint -> STL mesh mapping (mirrors ardy/viz/g1_rig.py G1_MESH_JOINT_MAP).
|
| 201 |
+
G1_MESH_JOINT_MAP = {
|
| 202 |
+
"pelvis_skel": ["pelvis.STL", "pelvis_contour_link.STL"],
|
| 203 |
+
"left_hip_pitch_skel": ["left_hip_pitch_link.STL"],
|
| 204 |
+
"left_hip_roll_skel": ["left_hip_roll_link.STL"],
|
| 205 |
+
"left_hip_yaw_skel": ["left_hip_yaw_link.STL"],
|
| 206 |
+
"left_knee_skel": ["left_knee_link.STL"],
|
| 207 |
+
"left_ankle_pitch_skel": ["left_ankle_pitch_link.STL"],
|
| 208 |
+
"left_ankle_roll_skel": ["left_ankle_roll_link.STL"],
|
| 209 |
+
"right_hip_pitch_skel": ["right_hip_pitch_link.STL"],
|
| 210 |
+
"right_hip_roll_skel": ["right_hip_roll_link.STL"],
|
| 211 |
+
"right_hip_yaw_skel": ["right_hip_yaw_link.STL"],
|
| 212 |
+
"right_knee_skel": ["right_knee_link.STL"],
|
| 213 |
+
"right_ankle_pitch_skel": ["right_ankle_pitch_link.STL"],
|
| 214 |
+
"right_ankle_roll_skel": ["right_ankle_roll_link.STL"],
|
| 215 |
+
"waist_yaw_skel": ["waist_yaw_link_rev_1_0.STL", "waist_yaw_link.STL"],
|
| 216 |
+
"waist_roll_skel": ["waist_roll_link_rev_1_0.STL", "waist_roll_link.STL"],
|
| 217 |
+
"waist_pitch_skel": [
|
| 218 |
+
"torso_link_rev_1_0.STL",
|
| 219 |
+
"torso_link.STL",
|
| 220 |
+
"logo_link.STL",
|
| 221 |
+
"head_link.STL",
|
| 222 |
+
],
|
| 223 |
+
"left_shoulder_pitch_skel": ["left_shoulder_pitch_link.STL"],
|
| 224 |
+
"left_shoulder_roll_skel": ["left_shoulder_roll_link.STL"],
|
| 225 |
+
"left_shoulder_yaw_skel": ["left_shoulder_yaw_link.STL"],
|
| 226 |
+
"left_elbow_skel": ["left_elbow_link.STL"],
|
| 227 |
+
"left_wrist_roll_skel": ["left_wrist_roll_link.STL"],
|
| 228 |
+
"left_wrist_pitch_skel": ["left_wrist_pitch_link.STL"],
|
| 229 |
+
"left_wrist_yaw_skel": ["left_wrist_yaw_link.STL", "left_rubber_hand.STL"],
|
| 230 |
+
"right_shoulder_pitch_skel": ["right_shoulder_pitch_link.STL"],
|
| 231 |
+
"right_shoulder_roll_skel": ["right_shoulder_roll_link.STL"],
|
| 232 |
+
"right_shoulder_yaw_skel": ["right_shoulder_yaw_link.STL"],
|
| 233 |
+
"right_elbow_skel": ["right_elbow_link.STL"],
|
| 234 |
+
"right_wrist_roll_skel": ["right_wrist_roll_link.STL"],
|
| 235 |
+
"right_wrist_pitch_skel": ["right_wrist_pitch_link.STL"],
|
| 236 |
+
"right_wrist_yaw_skel": ["right_wrist_yaw_link.STL", "right_rubber_hand.STL"],
|
| 237 |
+
}
|
| 238 |
+
|
| 239 |
+
_ROBOT_SKEL = RIGS["robot"]["skeleton"]
|
| 240 |
+
_MUJOCO_TO_ARDY = np.array(
|
| 241 |
+
[[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 0.0, 0.0]], dtype=np.float64
|
| 242 |
+
)
|
| 243 |
+
_G1_SKEL_DIR = Path(_ROBOT_SKEL.folder)
|
| 244 |
+
_G1_MESH_DIR = _G1_SKEL_DIR / "meshes" / "g1"
|
| 245 |
+
_G1_XML = _G1_SKEL_DIR / "xml" / "g1.xml"
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def _quat_wxyz_to_matrix(wxyz: np.ndarray) -> np.ndarray:
|
| 249 |
+
w, x, y, z = wxyz
|
| 250 |
+
n = np.sqrt(w * w + x * x + y * y + z * z)
|
| 251 |
+
if n < 1e-12:
|
| 252 |
+
return np.eye(3)
|
| 253 |
+
w, x, y, z = w / n, x / n, y / n, z / n
|
| 254 |
+
return np.array(
|
| 255 |
+
[
|
| 256 |
+
[1 - 2 * (y * y + z * z), 2 * (x * y - z * w), 2 * (x * z + y * w)],
|
| 257 |
+
[2 * (x * y + z * w), 1 - 2 * (x * x + z * z), 2 * (y * z - x * w)],
|
| 258 |
+
[2 * (x * z - y * w), 2 * (y * z + x * w), 1 - 2 * (x * x + y * y)],
|
| 259 |
+
],
|
| 260 |
+
dtype=np.float64,
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
def _mesh_local_transforms() -> dict:
|
| 265 |
+
"""mesh_file -> (geom_pos[3], geom_rot[3x3]) in ARDY axes, parsed from g1.xml."""
|
| 266 |
+
if not _G1_XML.exists():
|
| 267 |
+
return {}
|
| 268 |
+
root = ET.parse(_G1_XML).getroot()
|
| 269 |
+
file_to_name = {}
|
| 270 |
+
for mesh in root.findall(".//asset/mesh"):
|
| 271 |
+
name, file = mesh.get("name"), mesh.get("file")
|
| 272 |
+
if name and file:
|
| 273 |
+
file_to_name[file] = name
|
| 274 |
+
name_to_tf = {}
|
| 275 |
+
for geom in root.findall(".//geom"):
|
| 276 |
+
name = geom.get("mesh")
|
| 277 |
+
if name is None:
|
| 278 |
+
continue
|
| 279 |
+
pos = geom.get("pos")
|
| 280 |
+
quat = geom.get("quat")
|
| 281 |
+
gp = np.zeros(3) if pos is None else np.array([float(v) for v in pos.split()])
|
| 282 |
+
gr_ = np.eye(3) if quat is None else _quat_wxyz_to_matrix(
|
| 283 |
+
np.array([float(v) for v in quat.split()])
|
| 284 |
+
)
|
| 285 |
+
name_to_tf[name] = (gp, gr_)
|
| 286 |
+
out = {}
|
| 287 |
+
for file, name in file_to_name.items():
|
| 288 |
+
gp, gr_ = name_to_tf.get(name, (np.zeros(3), np.eye(3)))
|
| 289 |
+
gp = _MUJOCO_TO_ARDY @ gp
|
| 290 |
+
gr_ = _MUJOCO_TO_ARDY @ gr_ @ _MUJOCO_TO_ARDY.T
|
| 291 |
+
out[file] = (gp, gr_)
|
| 292 |
+
return out
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def _build_g1_mesh_blob():
|
| 296 |
+
"""Pack all rigid G1 meshes, pre-transformed into joint-local frame, into one
|
| 297 |
+
gzip+base64 blob. Returns (b64, meta). meta.parts lists per-mesh
|
| 298 |
+
{joint, v_off, v_cnt}. Layout: all verts f32[Vtot,3] then all faces
|
| 299 |
+
u32[Ftot,3] (face indices are GLOBAL into the concatenated vertex array)."""
|
| 300 |
+
skeleton = _ROBOT_SKEL
|
| 301 |
+
local_tf = _mesh_local_transforms()
|
| 302 |
+
all_v = []
|
| 303 |
+
all_f = []
|
| 304 |
+
parts = []
|
| 305 |
+
v_cursor = 0
|
| 306 |
+
for joint_name, mesh_files in G1_MESH_JOINT_MAP.items():
|
| 307 |
+
if joint_name not in skeleton.bone_index:
|
| 308 |
+
continue
|
| 309 |
+
joint_idx = int(skeleton.bone_index[joint_name])
|
| 310 |
+
for mesh_file in mesh_files:
|
| 311 |
+
mp = _G1_MESH_DIR / mesh_file
|
| 312 |
+
if not mp.exists():
|
| 313 |
+
continue
|
| 314 |
+
mesh = trimesh.load_mesh(str(mp), process=True)
|
| 315 |
+
if isinstance(mesh, trimesh.Scene):
|
| 316 |
+
mesh = trimesh.util.concatenate(mesh.dump())
|
| 317 |
+
verts = np.asarray(mesh.vertices, dtype=np.float64) @ _MUJOCO_TO_ARDY.T
|
| 318 |
+
faces = np.asarray(mesh.faces, dtype=np.int64)
|
| 319 |
+
gp, gr_ = local_tf.get(mesh_file, (np.zeros(3), np.eye(3)))
|
| 320 |
+
# Pre-apply the mesh's joint-local transform: v' = geom_rot @ v + geom_pos.
|
| 321 |
+
verts = (verts @ gr_.T) + gp
|
| 322 |
+
vcnt = verts.shape[0]
|
| 323 |
+
parts.append({"joint": joint_idx, "v_off": v_cursor, "v_cnt": vcnt})
|
| 324 |
+
all_v.append(verts.astype(np.float32))
|
| 325 |
+
all_f.append((faces + v_cursor).astype(np.uint32)) # global vertex indices
|
| 326 |
+
v_cursor += vcnt
|
| 327 |
+
V = np.concatenate(all_v, axis=0) if all_v else np.zeros((0, 3), np.float32)
|
| 328 |
+
F = np.concatenate(all_f, axis=0) if all_f else np.zeros((0, 3), np.uint32)
|
| 329 |
+
raw = np.ascontiguousarray(V).tobytes() + np.ascontiguousarray(F).tobytes()
|
| 330 |
+
meta = {"Vtot": int(V.shape[0]), "Ftot": int(F.shape[0]), "parts": parts}
|
| 331 |
+
b64 = base64.b64encode(gzip.compress(raw, 6)).decode("ascii")
|
| 332 |
+
return b64, meta
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
G1_MESH_B64, G1_MESH_META = _build_g1_mesh_blob()
|
| 336 |
+
print(
|
| 337 |
+
f"G1 rig ready: {len(G1_MESH_META['parts'])} meshes, "
|
| 338 |
+
f"{G1_MESH_META['Vtot']} verts / {G1_MESH_META['Ftot']} faces, "
|
| 339 |
+
f"blob {len(G1_MESH_B64) // 1024} KB",
|
| 340 |
+
flush=True,
|
| 341 |
+
)
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
def _joint_transforms_robot(global_rot_mats: np.ndarray, posed_joints: np.ndarray) -> str:
|
| 345 |
+
"""Per-frame joint affine matrices [R_global | pos], base64 f32 [T,J,12].
|
| 346 |
+
|
| 347 |
+
The browser applies world_v = pos + R_global @ v' per mesh (v' already in
|
| 348 |
+
joint-local frame)."""
|
| 349 |
+
T, J = posed_joints.shape[:2]
|
| 350 |
+
A = np.zeros((T, J, 3, 4), dtype=np.float32)
|
| 351 |
+
A[..., :3, :3] = global_rot_mats.astype(np.float32)
|
| 352 |
+
A[..., :3, 3] = posed_joints.astype(np.float32)
|
| 353 |
+
A = np.ascontiguousarray(A.reshape(T, J, 12).astype(np.float32))
|
| 354 |
+
return base64.b64encode(A.tobytes()).decode("ascii")
|
| 355 |
+
|
| 356 |
+
|
| 357 |
# -----------------------------------------------------------------------------
|
| 358 |
# Inference
|
| 359 |
# -----------------------------------------------------------------------------
|
| 360 |
+
def _estimate_duration(prompt, rig=DEFAULT_RIG, duration=5.0, *args, **kwargs):
|
| 361 |
# Autoregressive: rough per-second-of-motion cost + text encoding overhead.
|
| 362 |
try:
|
| 363 |
d = float(duration)
|
|
|
|
| 373 |
# for (history + new window), which can be fed straight back in as the next
|
| 374 |
# window's history. We thread that tensor to (a) fill a requested clip length
|
| 375 |
# and (b) CONTINUE a clip with a new prompt, exactly like the reference
|
| 376 |
+
# interactive demo. Persisting the tensor in a gr.State lets a second "Continue"
|
| 377 |
+
# click resume from where the first clip ended, on the same character.
|
| 378 |
+
def _generate_sequence(rig_key, prompt, num_new_frames, steps, cfg_weight, init_tensor):
|
| 379 |
+
"""Run the AR loop for one prompt on the selected rig. `init_tensor`:
|
| 380 |
+
normalized feature tensor [1, Th, D] on cuda (prior motion to continue), or
|
| 381 |
+
None to start fresh. Returns the full normalized feature tensor."""
|
| 382 |
+
r = RIGS[rig_key]
|
| 383 |
+
model = r["model"]
|
| 384 |
+
gen_horizon = r["gen_horizon"]
|
| 385 |
+
patch = r["patch"]
|
| 386 |
+
hist_crop = r["hist_crop"]
|
| 387 |
+
text_feat, text_pad_mask = model._encode_text([prompt])
|
| 388 |
motion_tensor = init_tensor
|
| 389 |
+
target_new = max(1, int(np.ceil(num_new_frames / gen_horizon))) * gen_horizon
|
| 390 |
produced = 0
|
| 391 |
while produced < target_new:
|
| 392 |
if motion_tensor is None:
|
| 393 |
history, hist_len = None, 0
|
| 394 |
else:
|
| 395 |
+
hist_len = (min(motion_tensor.shape[1], hist_crop) // patch) * patch
|
| 396 |
history = motion_tensor[:, motion_tensor.shape[1] - hist_len:] if hist_len else None
|
| 397 |
hist_len = history.shape[1] if history is not None else 0
|
| 398 |
+
samples = model.autoregressive_step(
|
| 399 |
+
num_frames=hist_len + gen_horizon, # exactly history + one window (no future)
|
| 400 |
num_denoising_steps=steps,
|
| 401 |
motion_mask=None,
|
| 402 |
observed_motion=None,
|
|
|
|
| 414 |
return motion_tensor
|
| 415 |
|
| 416 |
|
| 417 |
+
def _pack_payload(rig_key, motion_tensor, prompt, seed):
|
| 418 |
"""Decode a normalized feature tensor to the browser payload (per-frame joint
|
| 419 |
+
affines + root ground-track), tagged with the rig so the viewer loads the
|
| 420 |
+
correct skeleton/model."""
|
| 421 |
+
r = RIGS[rig_key]
|
| 422 |
+
model = r["model"]
|
| 423 |
with torch.no_grad():
|
| 424 |
+
out = to_numpy(model.motion_rep.inverse(motion_tensor, is_normalized=True))
|
| 425 |
posed = np.asarray(out["posed_joints"])[0] # [T, J, 3] global joint positions
|
| 426 |
grm = np.asarray(out["global_rot_mats"])[0] # [T, J, 3, 3] global rotations
|
| 427 |
+
if rig_key == "robot":
|
| 428 |
+
affines = _joint_transforms_robot(grm, posed)
|
| 429 |
+
else:
|
| 430 |
+
affines = _joint_affines_human(grm, posed)
|
| 431 |
return {
|
| 432 |
+
"rig": rig_key,
|
| 433 |
+
"fps": r["fps"],
|
| 434 |
"num_frames": int(posed.shape[0]),
|
| 435 |
"num_joints": int(posed.shape[1]),
|
| 436 |
+
"affines": affines, # drives the browser rig
|
| 437 |
+
"root": np.round(posed[:, r["root_idx"], :].astype(np.float32), 4).tolist(),
|
| 438 |
"prompt": prompt,
|
| 439 |
"seed": seed,
|
| 440 |
}
|
| 441 |
|
| 442 |
|
| 443 |
+
def _core_generate(rig, prompt, duration, diffusion_steps, cfg_weight, seed, randomize_seed, init_np):
|
| 444 |
+
rig_key = _normalize_rig(rig)
|
| 445 |
+
r = RIGS[rig_key]
|
| 446 |
prompt = (prompt or "").strip()
|
| 447 |
if not prompt:
|
| 448 |
raise gr.Error("Please enter a text prompt describing the motion.")
|
|
|
|
| 450 |
seed = random.randint(0, MAX_SEED)
|
| 451 |
seed = int(seed)
|
| 452 |
seed_everything(seed)
|
| 453 |
+
steps = max(1, min(int(diffusion_steps), r["num_base_steps"]))
|
| 454 |
+
num_new = max(r["patch"], int(round(float(duration) * r["fps"])))
|
| 455 |
init_tensor = None if init_np is None else torch.from_numpy(init_np).to("cuda")
|
| 456 |
|
| 457 |
t0 = time.perf_counter()
|
| 458 |
with torch.no_grad():
|
| 459 |
+
full = _generate_sequence(rig_key, prompt, num_new, steps, cfg_weight, init_tensor)
|
| 460 |
+
payload = _pack_payload(rig_key, full, prompt, seed)
|
| 461 |
tag = "continue" if init_np is not None else "generate"
|
| 462 |
+
print(f"[{tag}:{rig_key}] '{prompt[:50]}' +{num_new}f -> {full.shape[1]}f total "
|
| 463 |
f"steps={steps} seed={seed} {time.perf_counter() - t0:.1f}s", flush=True)
|
| 464 |
return json.dumps(payload), seed, full.detach().cpu().numpy()
|
| 465 |
|
| 466 |
|
| 467 |
@spaces.GPU(duration=_estimate_duration)
|
| 468 |
+
def ui_generate(prompt, rig=DEFAULT_RIG, duration=5.0, diffusion_steps=NUM_BASE_STEPS,
|
| 469 |
cfg_weight=2.0, seed=0, randomize_seed=True):
|
| 470 |
"""Start a fresh clip (resets the running sequence)."""
|
| 471 |
+
return _core_generate(rig, prompt, duration, diffusion_steps, cfg_weight, seed, randomize_seed, None)
|
| 472 |
|
| 473 |
|
| 474 |
@spaces.GPU(duration=_estimate_duration)
|
| 475 |
+
def ui_continue(prompt, rig=DEFAULT_RIG, duration=5.0, diffusion_steps=NUM_BASE_STEPS,
|
| 476 |
cfg_weight=2.0, seed=0, randomize_seed=True, state=None):
|
| 477 |
"""Append a new action, continuing from the previous clip's final pose."""
|
| 478 |
+
return _core_generate(rig, prompt, duration, diffusion_steps, cfg_weight, seed, randomize_seed, state)
|
| 479 |
|
| 480 |
|
| 481 |
@spaces.GPU(duration=_estimate_duration)
|
| 482 |
+
def generate_motion(prompt: str, rig: str = DEFAULT_RIG, duration: float = 5.0,
|
| 483 |
+
diffusion_steps: int = NUM_BASE_STEPS, cfg_weight: float = 2.0,
|
| 484 |
+
seed: int = 0, randomize_seed: bool = True) -> tuple[str, int]:
|
| 485 |
+
"""Generate a 3D motion clip from a text prompt with ARDY.
|
| 486 |
|
| 487 |
Args:
|
| 488 |
prompt: Natural-language description of the motion (e.g. "a person walks in a circle").
|
| 489 |
+
rig: Which character to animate — "human" (27-joint skeleton) or "robot" (Unitree G1).
|
| 490 |
duration: Length of the generated motion in seconds.
|
| 491 |
diffusion_steps: Number of denoising steps (1..num_base_steps).
|
| 492 |
cfg_weight: Classifier-free-guidance weight for the text prompt.
|
|
|
|
| 497 |
A JSON string with the animated skeleton payload plus the seed used.
|
| 498 |
"""
|
| 499 |
payload_json, seed, _ = _core_generate(
|
| 500 |
+
rig, prompt, duration, diffusion_steps, cfg_weight, seed, randomize_seed, None
|
| 501 |
)
|
| 502 |
return payload_json, seed
|
| 503 |
|
| 504 |
|
| 505 |
# -----------------------------------------------------------------------------
|
| 506 |
+
# Front-end: a self-contained Three.js playground, delivered as a Gradio-native
|
| 507 |
+
# custom HTML component (Gradio 6 `gr.HTML` templates + js_on_load).
|
| 508 |
#
|
| 509 |
# The motion JSON is carried as the component's own `value` prop. `js_on_load`
|
| 510 |
# imports three.js, builds the scene once, wires the controls, then registers a
|
| 511 |
# `watch('value', ...)` callback that Gradio fires whenever the component is
|
| 512 |
+
# updated as the output of a Python event (Generate button / Examples).
|
| 513 |
+
#
|
| 514 |
+
# Each payload is tagged with its `rig`. The viewer ships the static data for
|
| 515 |
+
# BOTH rigs (human skin blob + G1 rigid-mesh blob) and switches at load time:
|
| 516 |
+
# - "human": one skinned body mesh (linear-blend skinning in the browser).
|
| 517 |
+
# - "robot": rigid per-joint G1 STL meshes posed by the joint transforms.
|
| 518 |
# -----------------------------------------------------------------------------
|
| 519 |
|
|
|
|
|
|
|
|
|
|
| 520 |
PLAYER_TEMPLATE = """
|
| 521 |
<div class="ardy-playground">
|
| 522 |
<div class="ardy-canvas-wrap">
|
|
|
|
| 542 |
"""
|
| 543 |
|
| 544 |
# css_template rules are auto-scoped to this component by Gradio.
|
|
|
|
|
|
|
| 545 |
PLAYER_CSS_TEMPLATE = """
|
| 546 |
.ardy-playground { width: 100%; }
|
| 547 |
.ardy-canvas-wrap {
|
|
|
|
| 569 |
.ardy-caption { margin-top:6px; font-size:13px; color:#667085; }
|
| 570 |
"""
|
| 571 |
|
|
|
|
|
|
|
| 572 |
APP_CSS = """
|
| 573 |
#col-container { max-width: 1200px; margin: 0 auto; }
|
| 574 |
.dark .gradio-container { color: var(--body-text-color); }
|
| 575 |
"""
|
| 576 |
|
| 577 |
# Runs once, when the component first renders. `element`, `props`, and `watch`
|
| 578 |
+
# are injected by Gradio. We import three.js, decode the static skin data (human
|
| 579 |
+
# rig) and the static rigid-mesh data (robot rig), build the scene, and subscribe
|
| 580 |
+
# to value changes with `watch('value', ...)`. Each generated payload carries the
|
| 581 |
+
# per-frame joint affine matrices plus a `rig` tag; the viewer renders whichever
|
| 582 |
+
# rig the payload requests, swapping the on-screen mesh as needed.
|
| 583 |
PLAYER_JS_ON_LOAD = r"""
|
| 584 |
const root = element;
|
| 585 |
const q = (sel) => root.querySelector(sel);
|
| 586 |
|
| 587 |
const state = {
|
| 588 |
+
ready:false,
|
| 589 |
THREE:null, OrbitControls:null,
|
| 590 |
renderer:null, scene:null, camera:null, controls:null,
|
| 591 |
+
trailLine:null,
|
| 592 |
data:null, verts:null, frame:0, playing:false, lastT:0,
|
| 593 |
speed:1, loop:true, trail:false, pending:null,
|
| 594 |
+
rig:null, // which rig mesh is currently mounted in the scene
|
| 595 |
+
// human rig
|
| 596 |
+
skin:null, humanMesh:null, humanGeom:null,
|
| 597 |
+
// robot rig
|
| 598 |
+
robotMesh:null, robotGeom:null,
|
| 599 |
+
robotBaseVerts:null, robotFaces:null, robotVtot:0, robotParts:null,
|
| 600 |
};
|
| 601 |
|
| 602 |
// --- binary helpers ---------------------------------------------------------
|
|
|
|
| 611 |
return buf;
|
| 612 |
}
|
| 613 |
|
| 614 |
+
// Decode the one-time static human skin blob into typed arrays.
|
| 615 |
async function decodeSkin(){
|
| 616 |
const buf = await gunzip(b64ToBytes(ARDY_SKIN_B64));
|
| 617 |
const V = ARDY_SKIN_META.V, F = ARDY_SKIN_META.F, W = ARDY_SKIN_META.W;
|
|
|
|
| 623 |
return {V, F, W, bindV, faces, idx, wt};
|
| 624 |
}
|
| 625 |
|
| 626 |
+
// Decode the one-time static robot rig blob: pre-transformed mesh vertices
|
| 627 |
+
// (joint-local frame) + global-indexed faces + per-mesh part table.
|
| 628 |
+
async function decodeRig(){
|
| 629 |
+
const buf = await gunzip(b64ToBytes(G1_MESH_B64));
|
| 630 |
+
const V = G1_MESH_META.Vtot, F = G1_MESH_META.Ftot;
|
| 631 |
+
let o = 0;
|
| 632 |
+
const baseVerts = new Float32Array(buf.slice(o, o+V*3*4)); o += V*3*4;
|
| 633 |
+
const faces = new Uint32Array(buf.slice(o, o+F*3*4)); o += F*3*4;
|
| 634 |
+
return {V, F, baseVerts, faces, parts: G1_MESH_META.parts};
|
| 635 |
+
}
|
| 636 |
+
|
| 637 |
+
// HUMAN: per-vertex linear-blend skinning for every frame (once per clip).
|
| 638 |
// A[f,j] is a 3x4 affine (row-major, 12 floats); posed vertex =
|
| 639 |
// sum_k w_k * A[idx_k] @ [bind_x, bind_y, bind_z, 1].
|
| 640 |
function skinAllFrames(A, T, J){
|
|
|
|
| 660 |
return frames;
|
| 661 |
}
|
| 662 |
|
| 663 |
+
// ROBOT: per-frame rigid transform: for every mesh part, world_v = pos + R @ v'
|
| 664 |
+
// where (R,pos) is the driving joint's global transform this frame and v' is
|
| 665 |
+
// the vertex already baked into that joint's local frame.
|
| 666 |
+
function poseAllFrames(A, T, J){
|
| 667 |
+
const V = state.robotVtot, base = state.robotBaseVerts, parts = state.robotParts;
|
| 668 |
+
const frames = new Array(T);
|
| 669 |
+
for(let f=0; f<T; f++){
|
| 670 |
+
const out = new Float32Array(V*3);
|
| 671 |
+
const Ab = f*J*12;
|
| 672 |
+
for(let p=0; p<parts.length; p++){
|
| 673 |
+
const jp = parts[p];
|
| 674 |
+
const a = Ab + jp.joint*12;
|
| 675 |
+
const r0=A[a], r1=A[a+1], r2=A[a+2], px=A[a+3];
|
| 676 |
+
const r3=A[a+4], r4=A[a+5], r5=A[a+6], py=A[a+7];
|
| 677 |
+
const r6=A[a+8], r7=A[a+9], r8=A[a+10], pz=A[a+11];
|
| 678 |
+
const vs = jp.v_off, ve = jp.v_off + jp.v_cnt;
|
| 679 |
+
for(let v=vs; v<ve; v++){
|
| 680 |
+
const bx=base[v*3], by=base[v*3+1], bz=base[v*3+2];
|
| 681 |
+
out[v*3] = px + r0*bx + r1*by + r2*bz;
|
| 682 |
+
out[v*3+1] = py + r3*bx + r4*by + r5*bz;
|
| 683 |
+
out[v*3+2] = pz + r6*bx + r7*by + r8*bz;
|
| 684 |
+
}
|
| 685 |
+
}
|
| 686 |
+
frames[f] = out;
|
| 687 |
+
}
|
| 688 |
+
return frames;
|
| 689 |
+
}
|
| 690 |
+
|
| 691 |
+
// --- three.js scene ---------------------------------------------------------
|
| 692 |
function initScene(){
|
| 693 |
const THREE = state.THREE, OrbitControls = state.OrbitControls;
|
| 694 |
const wrap = q(".ardy-canvas-wrap");
|
|
|
|
| 713 |
scene.add(key);
|
| 714 |
scene.add(new THREE.DirectionalLight(0xeef2ff, 0.35).translateX(-4).translateZ(-2));
|
| 715 |
|
|
|
|
| 716 |
const ground = new THREE.Mesh(
|
| 717 |
new THREE.PlaneGeometry(40, 40),
|
| 718 |
new THREE.ShadowMaterial({opacity:0.16})
|
|
|
|
| 733 |
animate();
|
| 734 |
}
|
| 735 |
|
| 736 |
+
// Mount the mesh for the requested rig (lazily built, then shown/hidden). Only
|
| 737 |
+
// one rig mesh is visible at a time; both share the scene once created.
|
| 738 |
+
function mountRig(rig){
|
| 739 |
+
const THREE = state.THREE;
|
| 740 |
+
if(rig === "robot"){
|
| 741 |
+
if(!state.robotMesh){
|
| 742 |
+
const geom = new THREE.BufferGeometry();
|
| 743 |
+
geom.setIndex(new THREE.BufferAttribute(state.robotFaces, 1));
|
| 744 |
+
geom.setAttribute("position", new THREE.BufferAttribute(new Float32Array(state.robotVtot*3), 3));
|
| 745 |
+
const mat = new THREE.MeshStandardMaterial({color:0xd7dde6, roughness:0.5, metalness:0.55});
|
| 746 |
+
const mesh = new THREE.Mesh(geom, mat);
|
| 747 |
+
mesh.castShadow = true; mesh.frustumCulled = false;
|
| 748 |
+
state.scene.add(mesh); state.robotMesh = mesh; state.robotGeom = geom;
|
| 749 |
+
}
|
| 750 |
+
if(state.humanMesh) state.humanMesh.visible = false;
|
| 751 |
+
state.robotMesh.visible = true;
|
| 752 |
+
return state.robotGeom;
|
| 753 |
+
} else {
|
| 754 |
+
if(!state.humanMesh){
|
| 755 |
+
const s = state.skin;
|
| 756 |
+
const geom = new THREE.BufferGeometry();
|
| 757 |
+
geom.setIndex(new THREE.BufferAttribute(s.faces, 1));
|
| 758 |
+
geom.setAttribute("position", new THREE.BufferAttribute(new Float32Array(s.V*3), 3));
|
| 759 |
+
const mat = new THREE.MeshStandardMaterial({color:0x98bdff, roughness:0.85, metalness:0.0});
|
| 760 |
+
const mesh = new THREE.Mesh(geom, mat);
|
| 761 |
+
mesh.castShadow = true; mesh.frustumCulled = false;
|
| 762 |
+
state.scene.add(mesh); state.humanMesh = mesh; state.humanGeom = geom;
|
| 763 |
+
}
|
| 764 |
+
if(state.robotMesh) state.robotMesh.visible = false;
|
| 765 |
+
state.humanMesh.visible = true;
|
| 766 |
+
return state.humanGeom;
|
| 767 |
+
}
|
| 768 |
+
}
|
| 769 |
+
|
| 770 |
+
function activeGeom(){
|
| 771 |
+
return (state.rig === "robot") ? state.robotGeom : state.humanGeom;
|
| 772 |
}
|
| 773 |
|
| 774 |
function setFrame(f){
|
|
|
|
| 776 |
const T = state.data.num_frames;
|
| 777 |
f = Math.max(0, Math.min(T-1, f|0));
|
| 778 |
state.frame = f;
|
| 779 |
+
const geom = activeGeom();
|
| 780 |
+
if(!geom) return;
|
| 781 |
+
const pos = geom.getAttribute("position");
|
| 782 |
pos.array.set(state.verts[f]);
|
| 783 |
pos.needsUpdate = true;
|
| 784 |
+
geom.computeVertexNormals();
|
| 785 |
+
geom.computeBoundingSphere();
|
| 786 |
updateTrail();
|
| 787 |
const scrub = q(".ardy-scrub"); if(scrub) scrub.value = f;
|
| 788 |
const lbl = q(".ardy-frame"); if(lbl) lbl.textContent = (f+1)+" / "+T;
|
|
|
|
| 835 |
|
| 836 |
function loadData(data){
|
| 837 |
initScene();
|
| 838 |
+
const rig = (data.rig === "robot") ? "robot" : "human";
|
| 839 |
+
state.rig = rig;
|
| 840 |
+
mountRig(rig);
|
| 841 |
state.data = data;
|
| 842 |
+
// Decode per-frame affines and pre-pose every frame (one-time cost per clip).
|
| 843 |
const T = data.num_frames, J = data.num_joints;
|
| 844 |
const Abytes = b64ToBytes(data.affines);
|
| 845 |
const A = new Float32Array(Abytes.buffer, Abytes.byteOffset, Abytes.byteLength/4);
|
| 846 |
+
state.verts = (rig === "robot") ? poseAllFrames(A, T, J) : skinAllFrames(A, T, J);
|
| 847 |
if(state.trailLine){ state.scene.remove(state.trailLine); state.trailLine.geometry.dispose(); state.trailLine=null; }
|
| 848 |
const scrub = q(".ardy-scrub"); if(scrub){ scrub.max = T-1; scrub.value = 0; }
|
| 849 |
const hint = q(".ardy-hint"); if(hint) hint.style.display = "none";
|
| 850 |
const cap = q(".ardy-caption");
|
| 851 |
+
if(cap) cap.textContent = '"' + data.prompt + '" · ' + (rig==="robot"?"robot":"human") +
|
| 852 |
+
' · ' + T + ' frames @ ' + data.fps + ' fps · seed ' + data.seed;
|
| 853 |
root.dataset.ardyLoaded = "1";
|
| 854 |
+
root.dataset.ardyRig = rig;
|
| 855 |
root.dataset.ardyFrames = String(T);
|
| 856 |
setFrame(0);
|
| 857 |
setPlaying(true);
|
|
|
|
| 859 |
|
| 860 |
function applyPayload(payload){
|
| 861 |
if(!payload) return;
|
| 862 |
+
if(!state.ready){ state.pending = payload; return; } // three.js / rigs still loading
|
| 863 |
try { loadData(JSON.parse(payload)); }
|
| 864 |
catch(e){ console.error("ARDY playground load error", e); }
|
| 865 |
}
|
|
|
|
| 876 |
}
|
| 877 |
|
| 878 |
// Gradio-native hand-off: render whenever the component's value prop updates as
|
| 879 |
+
// the output of a Python event (Generate / Continue / Examples).
|
| 880 |
if (typeof watch === "function") {
|
| 881 |
watch("value", () => applyPayload(props.value));
|
| 882 |
}
|
| 883 |
|
| 884 |
// Import three.js (esm.sh, not jsDelivr: the OrbitControls addon has an internal
|
| 885 |
// bare `import ... from "three"` that a browser dynamic import() can't resolve
|
| 886 |
+
// without an import map; esm.sh rewrites it and dedupes three) and decode both
|
| 887 |
+
// rigs, then flush any value that already arrived.
|
| 888 |
Promise.all([
|
| 889 |
import("https://esm.sh/three@0.160.0"),
|
| 890 |
import("https://esm.sh/three@0.160.0/examples/jsm/controls/OrbitControls.js"),
|
| 891 |
decodeSkin(),
|
| 892 |
+
decodeRig(),
|
| 893 |
+
]).then(([THREE, oc, skin, rig]) => {
|
| 894 |
state.THREE = THREE;
|
| 895 |
state.OrbitControls = oc.OrbitControls;
|
| 896 |
state.skin = skin;
|
| 897 |
+
state.robotBaseVerts = rig.baseVerts;
|
| 898 |
+
state.robotFaces = rig.faces;
|
| 899 |
+
state.robotVtot = rig.V;
|
| 900 |
+
state.robotParts = rig.parts;
|
| 901 |
state.ready = true;
|
| 902 |
wireControls();
|
| 903 |
initScene();
|
|
|
|
| 907 |
"""
|
| 908 |
|
| 909 |
EXAMPLES = [
|
| 910 |
+
["A person walks forward confidently.", "human", 5.0],
|
| 911 |
+
["A person walks in a circle.", "human", 6.0],
|
| 912 |
+
["A person jumps up and down.", "human", 4.0],
|
| 913 |
+
["The robot walks forward confidently.", "robot", 5.0],
|
| 914 |
+
["The robot waves with the right hand.", "robot", 4.0],
|
| 915 |
+
["The robot crouches down and then stands back up.", "robot", 5.0],
|
| 916 |
]
|
| 917 |
|
| 918 |
with gr.Blocks() as demo:
|
|
|
|
| 922 |
# 🕺 ARDY Motion Playground
|
| 923 |
Interactive text-to-motion with **[ARDY](https://research.nvidia.com/labs/sil/projects/ardy/)**
|
| 924 |
(Autoregressive Diffusion with Hybrid Representation) by NVIDIA.
|
| 925 |
+
Pick a **rig** (human or robot), type a prompt and **Generate** a 3D motion clip,
|
| 926 |
+
then **orbit, scrub, and play** it below.
|
| 927 |
Chain actions with **Continue ▸** — the same character keeps going from where it stopped.
|
| 928 |
"""
|
| 929 |
)
|
|
|
|
| 932 |
# resume the same character from the end of the previous clip.
|
| 933 |
seq_state = gr.State(None)
|
| 934 |
|
| 935 |
+
rig = gr.Radio(
|
| 936 |
+
choices=[("🕺 Human", "human"), ("🤖 Robot (Unitree G1)", "robot")],
|
| 937 |
+
value=DEFAULT_RIG,
|
| 938 |
+
label="Rig",
|
| 939 |
+
info="Choose which character to animate. Human uses the 27-joint skeleton; "
|
| 940 |
+
"Robot uses the Unitree G1 humanoid.",
|
| 941 |
+
)
|
| 942 |
+
|
| 943 |
prompt = gr.Textbox(
|
| 944 |
label="Motion prompt",
|
| 945 |
placeholder="e.g. a person walks in a circle then waves",
|
|
|
|
| 951 |
|
| 952 |
# The interactive 3D playground — a Gradio-native custom HTML component.
|
| 953 |
# Its `value` (the motion JSON) is set directly by the Generate handler;
|
| 954 |
+
# `watch('value', ...)` in js_on_load renders it. Ship the static data for
|
| 955 |
+
# both rigs (human skin blob + G1 rigid-mesh blob) once, as a header
|
| 956 |
# prepended to js_on_load; the per-frame affines ride in each payload.
|
| 957 |
+
_rig_header = (
|
| 958 |
f'const ARDY_SKIN_B64="{SKIN_B64}";\n'
|
| 959 |
f"const ARDY_SKIN_META={json.dumps(SKIN_META)};\n"
|
| 960 |
+
f'const G1_MESH_B64="{G1_MESH_B64}";\n'
|
| 961 |
+
f"const G1_MESH_META={json.dumps(G1_MESH_META)};\n"
|
| 962 |
)
|
| 963 |
player = gr.HTML(
|
| 964 |
value="",
|
| 965 |
html_template=PLAYER_TEMPLATE,
|
| 966 |
css_template=PLAYER_CSS_TEMPLATE,
|
| 967 |
+
js_on_load=_rig_header + PLAYER_JS_ON_LOAD,
|
| 968 |
elem_id="ardy-player",
|
| 969 |
)
|
| 970 |
|
|
|
|
| 980 |
|
| 981 |
gr.Examples(
|
| 982 |
examples=EXAMPLES,
|
| 983 |
+
inputs=[prompt, rig, duration],
|
| 984 |
outputs=[player, seed, seq_state],
|
| 985 |
fn=ui_generate,
|
| 986 |
cache_examples=False,
|
|
|
|
| 989 |
|
| 990 |
gr.Markdown(
|
| 991 |
"""
|
| 992 |
+
<small>Rigs: **ARDY-Core-RP-20FPS-Horizon40** (human, 27-joint skeleton @ 20 fps)
|
| 993 |
+
and **ARDY-G1-RP-25FPS-Horizon52** (Unitree G1 robot, 34-joint skeleton @ 25 fps).
|
| 994 |
Text encoder: LLM2Vec-Llama-3-8B. Post-processing (foot-skate cleanup) is disabled in this demo.
|
| 995 |
Motion is generated autoregressively; longer clips take longer.
|
| 996 |
**Generate** starts a new clip; **Continue ▸** keeps the same character going,
|
|
|
|
| 998 |
"""
|
| 999 |
)
|
| 1000 |
|
| 1001 |
+
_gen_inputs = [prompt, rig, duration, diffusion_steps, cfg_weight, seed, randomize_seed]
|
| 1002 |
# Generate starts fresh; Continue resumes from the running latent state.
|
| 1003 |
# The payload is written straight into the player's `value`; its js_on_load
|
| 1004 |
+
# `watch('value', ...)` renders it (loading the correct rig from the payload).
|
| 1005 |
run.click(fn=ui_generate, inputs=_gen_inputs,
|
| 1006 |
outputs=[player, seed, seq_state], api_name=False)
|
| 1007 |
cont.click(fn=ui_continue, inputs=_gen_inputs + [seq_state],
|
ardy/assets/skeletons/g1skel34/meshes/g1/head_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:005fb67fbd3eff94aa8bf4a6e83238174e9f91b6721f7111594322f223724411
|
| 3 |
+
size 932784
|
ardy/assets/skeletons/g1skel34/meshes/g1/left_ankle_pitch_link.STL
ADDED
|
Binary file (71.2 kB). View file
|
|
|
ardy/assets/skeletons/g1skel34/meshes/g1/left_ankle_roll_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c4092af943141d4d9f74232f3cfa345afc6565f46a077793b8ae0e68b39dc33f
|
| 3 |
+
size 653384
|
ardy/assets/skeletons/g1skel34/meshes/g1/left_elbow_link.STL
ADDED
|
Binary file (88.8 kB). View file
|
|
|
ardy/assets/skeletons/g1skel34/meshes/g1/left_hip_pitch_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4725168105ee768ee31638ef22b53f6be2d7641bfd7cfefe803488d884776fa4
|
| 3 |
+
size 181684
|
ardy/assets/skeletons/g1skel34/meshes/g1/left_hip_roll_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:91f25922ee8a7c3152790051bebad17b4d9cd243569c38fe340285ff93a97acf
|
| 3 |
+
size 192184
|
ardy/assets/skeletons/g1skel34/meshes/g1/left_hip_yaw_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a16d88aa6ddac8083aa7ad55ed317bea44b1fa003d314fba88965b7ed0f3b55b
|
| 3 |
+
size 296284
|
ardy/assets/skeletons/g1skel34/meshes/g1/left_knee_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8d92b9e3d3a636761150bb8025e32514c4602b91c7028d523ee42b3e632de477
|
| 3 |
+
size 854884
|
ardy/assets/skeletons/g1skel34/meshes/g1/left_rubber_hand.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cff2221a690fa69303f61fce68f2d155c1517b52efb6ca9262dd56e0bc6e70fe
|
| 3 |
+
size 2287484
|
ardy/assets/skeletons/g1skel34/meshes/g1/left_shoulder_pitch_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f0d1cfd02fcf0d42f95e678eeca33da3afbcc366ffba5c052847773ec4f31d52
|
| 3 |
+
size 176784
|
ardy/assets/skeletons/g1skel34/meshes/g1/left_shoulder_roll_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fb9df21687773522598dc384f1a2945c7519f11cbc8bd372a49170316d6eee88
|
| 3 |
+
size 400284
|
ardy/assets/skeletons/g1skel34/meshes/g1/left_shoulder_yaw_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1aa97e9748e924336567992181f78c7cd0652fd52a4afcca3df6b2ef6f9e712e
|
| 3 |
+
size 249184
|
ardy/assets/skeletons/g1skel34/meshes/g1/left_wrist_pitch_link.STL
ADDED
|
Binary file (86 kB). View file
|
|
|
ardy/assets/skeletons/g1skel34/meshes/g1/left_wrist_roll_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:edc387c9a0ba8c2237e9b296d32531426fabeb6f53e58df45c76106bca74148c
|
| 3 |
+
size 356184
|
ardy/assets/skeletons/g1skel34/meshes/g1/left_wrist_yaw_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:83f8fb3a726bf9613d65dd14f0f447cb918c3c95b3938042a0c9c09749267d3b
|
| 3 |
+
size 318684
|
ardy/assets/skeletons/g1skel34/meshes/g1/logo_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8571a0a19bc4916fa55f91449f51d5fdefd751000054865a842449429d5f155b
|
| 3 |
+
size 243384
|
ardy/assets/skeletons/g1skel34/meshes/g1/pelvis.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5ba6bbc888e630550140d3c26763f10206da8c8bd30ed886b8ede41c61f57a31
|
| 3 |
+
size 1060884
|
ardy/assets/skeletons/g1skel34/meshes/g1/pelvis_contour_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5cc5c2c7a312329e3feeb2b03d3fc09fc29705bd01864f6767e51be959662420
|
| 3 |
+
size 1805184
|
ardy/assets/skeletons/g1skel34/meshes/g1/right_ankle_pitch_link.STL
ADDED
|
Binary file (71.2 kB). View file
|
|
|
ardy/assets/skeletons/g1skel34/meshes/g1/right_ankle_roll_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4b66222ea56653e627711b56d0a8949b4920da5df091da0ceb343f54e884e3a5
|
| 3 |
+
size 653784
|
ardy/assets/skeletons/g1skel34/meshes/g1/right_elbow_link.STL
ADDED
|
Binary file (88.8 kB). View file
|
|
|
ardy/assets/skeletons/g1skel34/meshes/g1/right_hip_pitch_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e4f3c99d7f4a7d34eadbef9461fc66e3486cb5442db1ec50c86317d459f1a9c6
|
| 3 |
+
size 181284
|
ardy/assets/skeletons/g1skel34/meshes/g1/right_hip_roll_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4c254ef66a356f492947f360dd931965477b631e3fcc841f91ccc46d945d54f6
|
| 3 |
+
size 192684
|
ardy/assets/skeletons/g1skel34/meshes/g1/right_hip_yaw_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e479c2936ca2057e9eb2f7dff6c189b7419d7b8484dea0b298cbb36a2a6aa668
|
| 3 |
+
size 296284
|
ardy/assets/skeletons/g1skel34/meshes/g1/right_knee_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:63c4008449c9bbe701a6e2b557b7a252e90cf3a5abcf54cee46862b9a69f8ec8
|
| 3 |
+
size 852284
|
ardy/assets/skeletons/g1skel34/meshes/g1/right_rubber_hand.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:99533b778bca6246144fa511bb9d4e555e075c641f2a0251e04372869cd99d67
|
| 3 |
+
size 2192684
|
ardy/assets/skeletons/g1skel34/meshes/g1/right_shoulder_pitch_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:24cdb387e0128dfe602770a81c56cdce3a0181d34d039a11d1aaf8819b7b8c02
|
| 3 |
+
size 176784
|
ardy/assets/skeletons/g1skel34/meshes/g1/right_shoulder_roll_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:962b97c48f9ce9e8399f45dd9522e0865d19aa9fd299406b2d475a8fc4a53e81
|
| 3 |
+
size 401884
|
ardy/assets/skeletons/g1skel34/meshes/g1/right_shoulder_yaw_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a0b76489271da0c72461a344c9ffb0f0c6e64f019ea5014c1624886c442a2fe5
|
| 3 |
+
size 249984
|
ardy/assets/skeletons/g1skel34/meshes/g1/right_wrist_pitch_link.STL
ADDED
|
Binary file (79.6 kB). View file
|
|
|
ardy/assets/skeletons/g1skel34/meshes/g1/right_wrist_roll_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a7ee9212ff5b94d6cb7f52bb1bbf3f352194d5b598acff74f4c77d340c5b344f
|
| 3 |
+
size 356084
|
ardy/assets/skeletons/g1skel34/meshes/g1/right_wrist_yaw_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bc9dece2d12509707e0057ba2e48df8f3d56db0c79410212963a25e8a50f61a6
|
| 3 |
+
size 341484
|
ardy/assets/skeletons/g1skel34/meshes/g1/torso_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e96d023f0368a4e3450b86ca5d4f10227d8141156a373e7da8cb3c93266523e0
|
| 3 |
+
size 2232984
|
ardy/assets/skeletons/g1skel34/meshes/g1/torso_link_rev_1_0.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:11ddb46f2098efbbd8816b1d65893632d8e78be936376c7cdcd6771899ccc723
|
| 3 |
+
size 2570584
|
ardy/assets/skeletons/g1skel34/meshes/g1/waist_roll_link.STL
ADDED
|
Binary file (24.2 kB). View file
|
|
|
ardy/assets/skeletons/g1skel34/meshes/g1/waist_roll_link_rev_1_0.STL
ADDED
|
Binary file (85.9 kB). View file
|
|
|
ardy/assets/skeletons/g1skel34/meshes/g1/waist_yaw_link.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2883f20e03f09b669b5b4ce10677ee6b5191c0934b584d7cbaef2d0662856ffb
|
| 3 |
+
size 336284
|
ardy/assets/skeletons/g1skel34/meshes/g1/waist_yaw_link_rev_1_0.STL
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ec6db442b11f25eed898b5add07940c85d804f300de24dcbd264ccd8be7d554c
|
| 3 |
+
size 619984
|
requirements.txt
CHANGED
|
@@ -8,5 +8,6 @@ peft>=0.19
|
|
| 8 |
einops>=0.7
|
| 9 |
vector-quantize-pytorch>=1.25,<1.26
|
| 10 |
safetensors>=0.4
|
|
|
|
| 11 |
tqdm>=4.0
|
| 12 |
packaging>=21.0
|
|
|
|
| 8 |
einops>=0.7
|
| 9 |
vector-quantize-pytorch>=1.25,<1.26
|
| 10 |
safetensors>=0.4
|
| 11 |
+
trimesh>=4.0
|
| 12 |
tqdm>=4.0
|
| 13 |
packaging>=21.0
|