File size: 8,880 Bytes
df8f96e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 | {
"title": "Xperience-10M Foundation Model Plan",
"status": "planning_artifact",
"current_boundary": "No held-out multi-episode foundation-model result has been completed in this repo. The current foundation-model artifacts are setup-stage until enough valid episodes are staged and evaluated.",
"decision": {
"immediate_trainable_backbone": "Qwen3-Omni",
"first_world_model_branch": "Cosmos 3",
"first_policy_branch_candidates": [
"OpenVLA / OpenVLA-OFT",
"openpi pi0/pi0.5",
"NVIDIA GR00T"
],
"external_reasoning_reference": "Gemini Robotics"
},
"model_families": [
{
"priority": 1,
"family": "Qwen3-Omni",
"category": "omni_instruction_model",
"openness": "open_weights_available_from_official_hf_repo",
"best_role": "First selected-episode multimodal LoRA pilot and structured task predictor.",
"xperience10m_fit": [
"RGB/fisheye video, embedded audio, and language prompts can enter directly.",
"Depth, pose/SLAM, mocap, contacts, and IMU enter through the existing sensor bridge.",
"Matches current task outputs: labels, structured JSON, captions, and short decisions."
],
"current_decision": "keep_as_first_pilot",
"entry_condition": "Selected episodes staged with held-out episode split.",
"public_source": "https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct"
},
{
"priority": 2,
"family": "Cosmos 3",
"category": "world_foundation_model",
"openness": "track_official_nvidia_release_and_available_weights",
"best_role": "Embodied world modeling, action generation, future-window prediction, and synthetic-data expansion.",
"xperience10m_fit": [
"Uses video streams as visual state.",
"Uses pose/SLAM, depth, mocap, IMU, and language as physical-world conditioning signals.",
"Better aligned with prediction/generation objectives than simple label classification."
],
"current_decision": "add_as_first_world_model_branch_after_data_gate",
"entry_condition": "Multi-episode data plus enough storage/compute for generated or latent video-state outputs.",
"public_source": "https://www.nvidia.com/en-us/ai/cosmos/"
},
{
"priority": 3,
"family": "NVIDIA GR00T",
"category": "humanoid_policy_foundation_model",
"openness": "track_official_nvidia_release_and_tooling",
"best_role": "Humanoid action understanding, retargeting, contact/action prediction, and embodied skill transfer.",
"xperience10m_fit": [
"Hand/body mocap and contact cues can be retargeted into humanoid state/action targets.",
"Egocentric video plus human motion can support affordance and interaction tasks."
],
"current_decision": "track_as_humanoid_policy_branch",
"entry_condition": "Retargeting artifact and action-space definition exist.",
"public_source": "https://developer.nvidia.com/isaac/gr00t"
},
{
"priority": 4,
"family": "OpenVLA / OpenVLA-OFT",
"category": "vision_language_action_policy",
"openness": "open_project_and_weights",
"best_role": "Open robot-policy baseline after observations and action labels are converted into a VLA format.",
"xperience10m_fit": [
"Good candidate when each window is expressed as visual observation, instruction/context, and action token.",
"Requires an explicit action target; current human egocentric labels are not robot controls by default."
],
"current_decision": "candidate_after_action_space_design",
"entry_condition": "Window-to-action-token conversion is implemented and audited.",
"public_source": "https://openvla.github.io/"
},
{
"priority": 5,
"family": "openpi pi0/pi0.5",
"category": "robot_policy_model",
"openness": "open_source_policy_training_stack",
"best_role": "Action-chunking, policy fine-tuning, and embodiment-transfer experiments.",
"xperience10m_fit": [
"Useful once hand trajectories, contacts, or retargeted body motion are converted into policy targets.",
"Better for policy branch than for current structured task JSON outputs."
],
"current_decision": "candidate_policy_branch",
"entry_condition": "Action target and train/eval protocol exist for at least 64 episodes.",
"public_source": "https://github.com/Physical-Intelligence/openpi"
},
{
"priority": 6,
"family": "Gemini Robotics",
"category": "closed_embodied_reasoning_reference",
"openness": "closed_or_limited_access",
"best_role": "Qualitative reasoning reference, annotation helper, and external comparison when API access exists.",
"xperience10m_fit": [
"Can help reason over egocentric scenes and task descriptions.",
"Not a local fine-tune target for this repo."
],
"current_decision": "external_reference_only",
"entry_condition": "API/access exists and outputs are logged separately from trainable model metrics.",
"public_source": "https://deepmind.google/discover/blog/gemini-robotics-brings-ai-into-the-physical-world/"
},
{
"priority": 7,
"family": "Octo / SmolVLA-style lightweight policies",
"category": "lightweight_robot_policy_baselines",
"openness": "open_projects",
"best_role": "Cheaper policy baselines for observation-to-action experiments.",
"xperience10m_fit": [
"Useful after action target design.",
"Less directly omni-modal than Qwen3-Omni or Cosmos 3."
],
"current_decision": "optional_baseline_after_data_staging",
"entry_condition": "Action labels and baseline protocol exist.",
"public_source": "https://github.com/huggingface/lerobot"
}
],
"execution_order": [
{
"step": 1,
"name": "Data gate",
"action": "Stage at least 32 valid Xperience-10M episodes with held-out episode split."
},
{
"step": 2,
"name": "First held-out baseline",
"action": "Run Qwen3-Omni LoRA to establish the full train/eval loop."
},
{
"step": 3,
"name": "Model-selection dry run",
"action": "Run 3-8 episode dry runs for Qwen3-Omni prompt/LoRA, Cosmos 3 preprocessing, and one policy candidate."
},
{
"step": 4,
"name": "World-model branch",
"action": "Promote Cosmos 3 if future-window/action-conditioned preprocessing fits storage and compute."
},
{
"step": 5,
"name": "Policy branch",
"action": "Promote OpenVLA/openpi/GR00T after action target conversion and retargeting artifacts are traceable."
},
{
"step": 6,
"name": "Publication rule",
"action": "Publish branch results only with real manifests, predictions, metrics, and qualitative examples."
}
],
"evaluation_additions": [
{
"target": "structured_task_prediction",
"metrics": [
"JSON validity",
"macro-F1",
"accuracy",
"micro-F1"
],
"model_families": [
"Qwen3-Omni",
"Gemini Robotics reference"
]
},
{
"target": "future_state_prediction",
"metrics": [
"retrieval rank",
"temporal consistency",
"feature reconstruction",
"qualitative visual inspection"
],
"model_families": [
"Cosmos 3"
]
},
{
"target": "action_conditioned_dynamics",
"metrics": [
"transition accuracy",
"contact accuracy",
"next-action accuracy"
],
"model_families": [
"Cosmos 3",
"OpenVLA",
"openpi",
"GR00T"
]
},
{
"target": "cross_episode_generalization",
"metrics": [
"held-out episode metrics",
"held-out session metrics",
"leakage audit"
],
"model_families": [
"all trainable branches"
]
}
],
"source_links": [
{
"label": "Qwen3-Omni official HF model",
"url": "https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct"
},
{
"label": "NVIDIA Cosmos",
"url": "https://www.nvidia.com/en-us/ai/cosmos/"
},
{
"label": "NVIDIA Isaac GR00T",
"url": "https://developer.nvidia.com/isaac/gr00t"
},
{
"label": "OpenVLA",
"url": "https://openvla.github.io/"
},
{
"label": "openpi",
"url": "https://github.com/Physical-Intelligence/openpi"
},
{
"label": "Gemini Robotics",
"url": "https://deepmind.google/discover/blog/gemini-robotics-brings-ai-into-the-physical-world/"
},
{
"label": "Octo",
"url": "https://octo-models.github.io/"
},
{
"label": "LeRobot / SmolVLA",
"url": "https://github.com/huggingface/lerobot"
}
]
}
|