| { |
| "title": "Three Foundation Pipeline Tracks", |
| "status": "pipeline_plan", |
| "source_document": "THREE_FOUNDATION_PIPELINES.md", |
| "reader_note": "These are supported pipeline directions with concrete data exports, training recipes, and evaluation gates.", |
| "diagram_assets": { |
| "status": "published_high_resolution_slide_diagrams", |
| "asset_root": "docs/assets/foundation-pipelines", |
| "source": "Clean direction-slide PNGs supplied for the three public direction figures, with original presentation photos retained as provenance", |
| "source_slide_root": "docs/assets/foundation-pipelines/source-slides", |
| "source_photo_root": "docs/assets/foundation-pipelines/source-photos", |
| "provenance_file": "docs/assets/foundation-pipelines/prompts.md", |
| "renderer_script": "scripts/render_foundation_pipeline_diagrams.py", |
| "diagram_type": "direction_slide_diagram", |
| "source_update": "2026-06-19: clean Spatial intelligence, Human-video world model, and Vision-language-action PNGs are committed as source-slide assets and published as 2560-pixel public images.", |
| "note": "Images are slide-diagram communication assets for pipeline tracks. Technical readouts remain governed by the Markdown/JSON contracts and verified metrics." |
| }, |
| "shared_principles": [ |
| "Use episode-level train/validation/test separation.", |
| "Build manifest-first exporters before training.", |
| "Keep target-side future labels and captions out of inputs unless the task explicitly queries them.", |
| "Report task-specific metrics and saved predictions before updating public cards.", |
| "Exclude raw private data and heavyweight base model weights from public packages." |
| ], |
| "tracks": [ |
| { |
| "id": "spatial_intelligence", |
| "title": "Spatial intelligence models", |
| "question": "Can the model recover and reason over space from video?", |
| "core_inputs": [ |
| "multiview RGB", |
| "egocentric video", |
| "depth", |
| "camera pose", |
| "calibration", |
| "object cues", |
| "language questions" |
| ], |
| "intermediate_artifacts": [ |
| "synchronized camera window manifest", |
| "pose and depth availability report", |
| "scene and object memory records", |
| "object permanence targets", |
| "spatial relation targets", |
| "spatial QA prompts" |
| ], |
| "outputs": [ |
| "object count", |
| "object persistence", |
| "relative location", |
| "3D geometry consistency", |
| "multiview retrieval", |
| "camera-motion-aware scene memory", |
| "language answers grounded in the scene" |
| ], |
| "first_pipeline": "Build a spatial-memory exporter, start with metric depth and pose consistency tasks, then evaluate spatial QA, object permanence, counting, retrieval, and pose-aware consistency.", |
| "current_maturity": "Ready as a pipeline and evaluation contract.", |
| "next_gate": "Raw depth and pose artifacts plus held-out multi-episode spatial metrics.", |
| "diagram_image": "docs/assets/foundation-pipelines/spatial-intelligence-pipeline.png", |
| "website_image": "assets/foundation-pipelines/spatial-intelligence-pipeline.png", |
| "image_alt": "High-resolution slide diagram showing the Spatial intelligence models direction for Xperience-10M.", |
| "one_sample_training_io": { |
| "sample_basis": "Single public sample episode: 5,821 frames, 1,161 overlapping 20-frame windows, 5-frame stride, about 20 FPS.", |
| "source_artifacts": [ |
| "results/episode_task_suite/windows.csv", |
| "results/episode_task_suite/shared_windows.npz", |
| "results/episode_task_suite/feature_manifest.json", |
| "official sample annotation.hdf5", |
| "official sample six MP4 camera streams" |
| ], |
| "input_builder": "Slice each 20-frame window, then join multiview RGB summaries with depth, camera pose, SLAM/calibration, object cues, contact cues, and optional language questions from the public annotation timeline.", |
| "target_builder": "Create spatial targets such as camera-view match, object relevance, object-set memory, depth/pose reconstruction proxy, caption-grounded retrieval, and spatial QA answers.", |
| "existing_task_hooks": [ |
| "object_relevance", |
| "modality_reconstruction", |
| "caption_grounding", |
| "object_set_forecast", |
| "camera_view_sync_retrieval" |
| ], |
| "boundary": "This yields a one-episode spatial training-pair recipe and proxy tasks; the next spatial-intelligence readout is held-out depth, pose, and scene-memory metrics." |
| }, |
| "diagram_flow": [ |
| { |
| "stage": "inputs", |
| "items": [ |
| "multiview RGB plus egocentric video", |
| "metric depth and confidence", |
| "camera pose, calibration, SLAM", |
| "object, contact, and language cues" |
| ] |
| }, |
| { |
| "stage": "tasks_targets", |
| "items": [ |
| "spatial QA and object count", |
| "object permanence across windows", |
| "relative location and retrieval", |
| "pose-aware 3D consistency" |
| ] |
| }, |
| { |
| "stage": "train_models", |
| "items": [ |
| "export scene/object memory records", |
| "train spatial-memory encoder", |
| "add geometry-aware QA and retrieval heads", |
| "keep episode-level split discipline" |
| ] |
| }, |
| { |
| "stage": "evaluate_gates", |
| "items": [ |
| "held-out episode spatial metrics", |
| "count and relation accuracy", |
| "retrieval rank and consistency", |
| "saved predictions before public package update" |
| ] |
| } |
| ], |
| "next_readout_before_stronger_positioning": [ |
| "held-out spatial QA", |
| "pose consistency", |
| "object-counting and scene-memory metrics" |
| ] |
| }, |
| { |
| "id": "human_video_world_models", |
| "title": "Human-video world models", |
| "question": "Can the model predict what happens next?", |
| "core_inputs": [ |
| "observed video windows", |
| "audio", |
| "sensor windows", |
| "hand and body motion", |
| "object and contact state", |
| "action and subtask labels", |
| "future windows" |
| ], |
| "intermediate_artifacts": [ |
| "observed and future window pairs", |
| "future label targets", |
| "action-conditioned target records", |
| "visual or latent reconstruction targets", |
| "temporal consistency metadata" |
| ], |
| "outputs": [ |
| "next action", |
| "next subtask", |
| "future object set", |
| "future state embedding", |
| "camera-motion delta", |
| "contact transition", |
| "future-window quality metrics" |
| ], |
| "first_pipeline": "Keep Qwen-style structured future probes for task interpretability, keep Cosmos-style dynamics branches separate, and add latent or feature-reconstruction metrics before presenting world-model quality.", |
| "current_maturity": "Partially evidenced by current future-task probes and Cosmos-style branch artifacts.", |
| "next_gate": "Stronger future-state metrics, qualitative future examples, and held-out episode breakdowns.", |
| "diagram_image": "docs/assets/foundation-pipelines/human-video-world-model-pipeline.png", |
| "website_image": "assets/foundation-pipelines/human-video-world-model-pipeline.png", |
| "image_alt": "High-resolution slide diagram showing the Human-video world models direction for Xperience-10M.", |
| "one_sample_training_io": { |
| "sample_basis": "Single public sample episode: current observed windows are paired with shifted future labels or future-window features from the same timeline.", |
| "source_artifacts": [ |
| "results/episode_task_suite/windows.csv", |
| "results/episode_task_suite/shared_windows.npz", |
| "results/episode_task_suite/tier2_task_suite/tier2_task_suite_results.json", |
| "results/episode_task_suite/research_direction_extensions/research_direction_extension_results.json" |
| ], |
| "input_builder": "Use the current 20-frame observed window at time t: RGB/audio/sensor summaries, hand/body motion, camera pose, current object/contact state, and current action/subtask context only.", |
| "target_builder": "Shift the episode timeline forward to produce next-action, next-subtask, future object-set, contact-transition, time-to-transition, camera-motion delta, or latent/future-feature targets.", |
| "existing_task_hooks": [ |
| "next_action", |
| "long_horizon_next_action", |
| "next_subtask_forecast", |
| "object_set_forecast", |
| "time_to_transition", |
| "ego_motion_forecast" |
| ], |
| "boundary": "Future labels and future windows must stay out of the input. Structured future probes show the pipeline, while visual world-model readouts need latent or visual future metrics." |
| }, |
| "diagram_flow": [ |
| { |
| "stage": "inputs", |
| "items": [ |
| "observed video/audio/sensor window", |
| "hand/body motion and camera pose", |
| "object/contact state", |
| "action and subtask labels" |
| ] |
| }, |
| { |
| "stage": "tasks_targets", |
| "items": [ |
| "next action and next subtask", |
| "future object set", |
| "contact transition", |
| "camera-motion delta or latent future" |
| ] |
| }, |
| { |
| "stage": "train_models", |
| "items": [ |
| "Qwen structured future probes", |
| "Cosmos/dynamics branch separately", |
| "latent rollout or reconstruction loss", |
| "no target-side future leakage" |
| ] |
| }, |
| { |
| "stage": "evaluate_gates", |
| "items": [ |
| "held-out future-task metrics", |
| "contact and object-set F1", |
| "rollout or latent consistency", |
| "per-episode breakdown and examples" |
| ] |
| } |
| ], |
| "next_readout_before_stronger_positioning": [ |
| "latent or visual future metrics", |
| "per-episode future-task breakdowns", |
| "qualitative examples backed by saved targets" |
| ] |
| }, |
| { |
| "id": "vision_language_action", |
| "title": "Vision-language-action models", |
| "question": "Can the model turn what it sees and reads into action?", |
| "core_inputs": [ |
| "egocentric video", |
| "language captions", |
| "hand and body motion", |
| "contacts", |
| "objects", |
| "procedure and subtask labels" |
| ], |
| "intermediate_artifacts": [ |
| "action-token vocabulary", |
| "action-chunk windows", |
| "normalization stats", |
| "retargeting report", |
| "leakage audit", |
| "action-space model card" |
| ], |
| "outputs": [ |
| "next action", |
| "action chunk", |
| "object-conditioned action", |
| "contact state", |
| "subtask transition", |
| "policy or VLA held-out metrics" |
| ], |
| "first_pipeline": "Define the action space, use existing 20-task next-action/contact/object-conditioned tasks first, then add hand-trajectory or policy-compatible action chunks after conversion is traceable.", |
| "current_maturity": "Feasible but gated by action-target conversion.", |
| "next_gate": "Traceable action tokens, normalization, retargeting metadata, and held-out policy metrics.", |
| "diagram_image": "docs/assets/foundation-pipelines/vision-language-action-pipeline.png", |
| "website_image": "assets/foundation-pipelines/vision-language-action-pipeline.png", |
| "image_alt": "High-resolution slide diagram showing the Vision-language-action models direction for Xperience-10M.", |
| "one_sample_training_io": { |
| "sample_basis": "Single public sample episode: observation-language windows are paired with action-token proxies because robot retargeted action chunks are not part of the public sample yet.", |
| "source_artifacts": [ |
| "results/episode_task_suite/windows.csv", |
| "results/episode_task_suite/shared_windows.npz", |
| "results/episode_task_suite/task_walkthroughs/task_walkthroughs.json", |
| "official sample annotation.hdf5" |
| ], |
| "input_builder": "Use egocentric/fisheye video windows, caption and object context, hand/body mocap, contact state, and current subtask text as the observation-language side of each training pair.", |
| "target_builder": "Create action-token proxy targets: current or next action, object-conditioned action relation, contact state, interaction-text class, subtask transition, or hand-trajectory/action-chunk proxy.", |
| "existing_task_hooks": [ |
| "timeline_action", |
| "next_action", |
| "hand_trajectory_forecast", |
| "contact_prediction", |
| "interaction_text_prediction", |
| "action_object_relation" |
| ], |
| "boundary": "This is a VLA/policy data-conversion recipe for the one-sample suite. Robot policy readouts require a later action-space converter, normalization, retargeting report, and held-out policy metrics." |
| }, |
| "diagram_flow": [ |
| { |
| "stage": "inputs", |
| "items": [ |
| "egocentric video and captions", |
| "objects, contacts, and procedures", |
| "hand/body motion windows", |
| "subtask labels and language context" |
| ] |
| }, |
| { |
| "stage": "tasks_targets", |
| "items": [ |
| "action-token vocabulary", |
| "next action and action chunks", |
| "object-conditioned actions", |
| "contact state and subtask transition" |
| ] |
| }, |
| { |
| "stage": "train_models", |
| "items": [ |
| "build action-space converter", |
| "normalize and audit action chunks", |
| "train VLA/policy-compatible head", |
| "track leakage and retargeting reports" |
| ] |
| }, |
| { |
| "stage": "evaluate_gates", |
| "items": [ |
| "held-out action metrics", |
| "chunk and next-action accuracy", |
| "object/contact-conditioned scores", |
| "policy card before robot-policy quality readout" |
| ] |
| } |
| ], |
| "next_readout_before_stronger_positioning": [ |
| "action-space conversion", |
| "normalized action chunks", |
| "held-out policy metrics" |
| ] |
| } |
| ] |
| } |
|
|
