File size: 10,312 Bytes
fe4bbfa 596ac86 16a39bb 05637a9 01f57c3 16a39bb d9e465e 596ac86 16a39bb 01f57c3 16a39bb 05637a9 fe4bbfa 596ac86 05637a9 16a39bb 596ac86 fe4bbfa 596ac86 05637a9 16a39bb 596ac86 fe4bbfa 596ac86 05637a9 16a39bb 596ac86 fe4bbfa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 | {
"title": "Three Foundation Pipeline Tracks",
"status": "pipeline_plan",
"source_document": "THREE_FOUNDATION_PIPELINES.md",
"claim_boundary": "These are supported pipeline directions, not three completed model-quality claims.",
"diagram_assets": {
"status": "published_high_resolution_slide_diagrams",
"asset_root": "docs/assets/foundation-pipelines",
"source": "Clean direction-slide PNGs supplied for the three public direction figures, with original presentation photos retained as provenance",
"source_slide_root": "docs/assets/foundation-pipelines/source-slides",
"source_photo_root": "docs/assets/foundation-pipelines/source-photos",
"provenance_file": "docs/assets/foundation-pipelines/prompts.md",
"renderer_script": "scripts/render_foundation_pipeline_diagrams.py",
"diagram_type": "direction_slide_diagram",
"source_update": "2026-06-19: clean Spatial intelligence, Human-video world model, and Vision-language-action PNGs are committed as source-slide assets and published as 2560-pixel public images.",
"note": "Images are slide-diagram communication assets for pipeline tracks. Technical claims remain governed by the Markdown/JSON contracts and verified metrics."
},
"shared_principles": [
"Use episode-level train/validation/test separation.",
"Build manifest-first exporters before training.",
"Keep target-side future labels and captions out of inputs unless the task explicitly queries them.",
"Report task-specific metrics and saved predictions before updating public cards.",
"Exclude raw private data and heavyweight base model weights from public packages."
],
"tracks": [
{
"id": "spatial_intelligence",
"title": "Spatial intelligence models",
"question": "Can the model recover and reason over space from video?",
"core_inputs": [
"multiview RGB",
"egocentric video",
"depth",
"camera pose",
"calibration",
"object cues",
"language questions"
],
"intermediate_artifacts": [
"synchronized camera window manifest",
"pose and depth availability report",
"scene and object memory records",
"object permanence targets",
"spatial relation targets",
"spatial QA prompts"
],
"outputs": [
"object count",
"object persistence",
"relative location",
"3D geometry consistency",
"multiview retrieval",
"camera-motion-aware scene memory",
"language answers grounded in the scene"
],
"first_pipeline": "Build a spatial-memory exporter, start with metric depth and pose consistency tasks, then evaluate spatial QA, object permanence, counting, retrieval, and pose-aware consistency.",
"current_maturity": "Ready as a pipeline and evaluation contract.",
"next_gate": "Raw depth and pose artifacts plus held-out multi-episode spatial metrics.",
"diagram_image": "docs/assets/foundation-pipelines/spatial-intelligence-pipeline.png",
"website_image": "assets/foundation-pipelines/spatial-intelligence-pipeline.png",
"image_alt": "High-resolution slide diagram showing the Spatial intelligence models direction for Xperience-10M.",
"diagram_flow": [
{
"stage": "inputs",
"items": [
"multiview RGB plus egocentric video",
"metric depth and confidence",
"camera pose, calibration, SLAM",
"object, contact, and language cues"
]
},
{
"stage": "tasks_targets",
"items": [
"spatial QA and object count",
"object permanence across windows",
"relative location and retrieval",
"pose-aware 3D consistency"
]
},
{
"stage": "train_models",
"items": [
"export scene/object memory records",
"train spatial-memory encoder",
"add geometry-aware QA and retrieval heads",
"keep episode-level split discipline"
]
},
{
"stage": "evaluate_gates",
"items": [
"held-out episode spatial metrics",
"count and relation accuracy",
"retrieval rank and consistency",
"saved predictions before public claim"
]
}
],
"avoid_claiming_now": [
"full neural rendering",
"full 3D reconstruction",
"general spatial intelligence without artifact-level evidence"
]
},
{
"id": "human_video_world_models",
"title": "Human-video world models",
"question": "Can the model predict what happens next?",
"core_inputs": [
"observed video windows",
"audio",
"sensor windows",
"hand and body motion",
"object and contact state",
"action and subtask labels",
"future windows"
],
"intermediate_artifacts": [
"observed and future window pairs",
"future label targets",
"action-conditioned target records",
"visual or latent reconstruction targets",
"temporal consistency metadata"
],
"outputs": [
"next action",
"next subtask",
"future object set",
"future state embedding",
"camera-motion delta",
"contact transition",
"future-window quality metrics"
],
"first_pipeline": "Keep Qwen-style structured future probes for task interpretability, keep Cosmos-style dynamics branches separate, and add latent or feature-reconstruction metrics before claiming world-model quality.",
"current_maturity": "Partially evidenced by current future-task probes and Cosmos-style branch artifacts.",
"next_gate": "Stronger future-state metrics, qualitative future examples, and held-out episode breakdowns.",
"diagram_image": "docs/assets/foundation-pipelines/human-video-world-model-pipeline.png",
"website_image": "assets/foundation-pipelines/human-video-world-model-pipeline.png",
"image_alt": "High-resolution slide diagram showing the Human-video world models direction for Xperience-10M.",
"diagram_flow": [
{
"stage": "inputs",
"items": [
"observed video/audio/sensor window",
"hand/body motion and camera pose",
"object/contact state",
"action and subtask labels"
]
},
{
"stage": "tasks_targets",
"items": [
"next action and next subtask",
"future object set",
"contact transition",
"camera-motion delta or latent future"
]
},
{
"stage": "train_models",
"items": [
"Qwen structured future probes",
"Cosmos/dynamics branch separately",
"latent rollout or reconstruction loss",
"no target-side future leakage"
]
},
{
"stage": "evaluate_gates",
"items": [
"held-out future-task metrics",
"contact and object-set F1",
"rollout or latent consistency",
"per-episode breakdown and examples"
]
}
],
"avoid_claiming_now": [
"strong world model from structured future-task scores alone",
"visual future quality without visual or latent future metrics"
]
},
{
"id": "vision_language_action",
"title": "Vision-language-action models",
"question": "Can the model turn what it sees and reads into action?",
"core_inputs": [
"egocentric video",
"language captions",
"hand and body motion",
"contacts",
"objects",
"procedure and subtask labels"
],
"intermediate_artifacts": [
"action-token vocabulary",
"action-chunk windows",
"normalization stats",
"retargeting report",
"leakage audit",
"action-space model card"
],
"outputs": [
"next action",
"action chunk",
"object-conditioned action",
"contact state",
"subtask transition",
"policy or VLA held-out metrics"
],
"first_pipeline": "Define the action space, use existing 20-task next-action/contact/object-conditioned tasks first, then add hand-trajectory or policy-compatible action chunks after conversion is traceable.",
"current_maturity": "Feasible but gated by action-target conversion.",
"next_gate": "Traceable action tokens, normalization, retargeting metadata, and held-out policy metrics.",
"diagram_image": "docs/assets/foundation-pipelines/vision-language-action-pipeline.png",
"website_image": "assets/foundation-pipelines/vision-language-action-pipeline.png",
"image_alt": "High-resolution slide diagram showing the Vision-language-action models direction for Xperience-10M.",
"diagram_flow": [
{
"stage": "inputs",
"items": [
"egocentric video and captions",
"objects, contacts, and procedures",
"hand/body motion windows",
"subtask labels and language context"
]
},
{
"stage": "tasks_targets",
"items": [
"action-token vocabulary",
"next action and action chunks",
"object-conditioned actions",
"contact state and subtask transition"
]
},
{
"stage": "train_models",
"items": [
"build action-space converter",
"normalize and audit action chunks",
"train VLA/policy-compatible head",
"track leakage and retargeting reports"
]
},
{
"stage": "evaluate_gates",
"items": [
"held-out action metrics",
"chunk and next-action accuracy",
"object/contact-conditioned scores",
"policy card before robot-policy claim"
]
}
],
"avoid_claiming_now": [
"robot policy quality",
"policy generalization before action-space evidence exists"
]
}
]
}
|