metrics/three_foundation_pipelines.json

{
  "title": "Three Foundation Pipeline Tracks",
  "status": "pipeline_plan",
  "source_document": "THREE_FOUNDATION_PIPELINES.md",
  "reader_note": "These are supported pipeline directions with concrete data exports, training recipes, and evaluation gates.",
  "diagram_assets": {
    "status": "published_high_resolution_slide_diagrams",
    "asset_root": "docs/assets/foundation-pipelines",
    "source": "Clean direction-slide PNGs supplied for the three public direction figures, with original presentation photos retained as provenance",
    "source_slide_root": "docs/assets/foundation-pipelines/source-slides",
    "source_photo_root": "docs/assets/foundation-pipelines/source-photos",
    "provenance_file": "docs/assets/foundation-pipelines/prompts.md",
    "renderer_script": "scripts/render_foundation_pipeline_diagrams.py",
    "diagram_type": "direction_slide_diagram",
    "source_update": "2026-06-19: clean Spatial intelligence, Human-video world model, and Vision-language-action PNGs are committed as source-slide assets and published as 2560-pixel public images.",
    "note": "Images are slide-diagram communication assets for pipeline tracks. Technical readouts remain governed by the Markdown/JSON contracts and verified metrics."
  },
  "shared_principles": [
    "Use episode-level train/validation/test separation.",
    "Build manifest-first exporters before training.",
    "Keep target-side future labels and captions out of inputs unless the task explicitly queries them.",
    "Report task-specific metrics and saved predictions before updating public cards.",
    "Exclude raw private data and heavyweight base model weights from public packages."
  ],
  "tracks": [
    {
      "id": "spatial_intelligence",
      "title": "Spatial intelligence models",
      "question": "Can the model recover and reason over space from video?",
      "core_inputs": [
        "multiview RGB",
        "egocentric video",
        "depth",
        "camera pose",
        "calibration",
        "object cues",
        "language questions"
      ],
      "intermediate_artifacts": [
        "synchronized camera window manifest",
        "pose and depth availability report",
        "scene and object memory records",
        "object permanence targets",
        "spatial relation targets",
        "spatial QA prompts"
      ],
      "outputs": [
        "object count",
        "object persistence",
        "relative location",
        "3D geometry consistency",
        "multiview retrieval",
        "camera-motion-aware scene memory",
        "language answers grounded in the scene"
      ],
      "first_pipeline": "Build a spatial-memory exporter, start with metric depth and pose consistency tasks, then evaluate spatial QA, object permanence, counting, retrieval, and pose-aware consistency.",
      "current_maturity": "Ready as a pipeline and evaluation contract.",
      "next_gate": "Raw depth and pose artifacts plus held-out multi-episode spatial metrics.",
      "diagram_image": "docs/assets/foundation-pipelines/spatial-intelligence-pipeline.png",
      "website_image": "assets/foundation-pipelines/spatial-intelligence-pipeline.png",
      "image_alt": "High-resolution slide diagram showing the Spatial intelligence models direction for Xperience-10M.",
      "one_sample_training_io": {
        "sample_basis": "Single public sample episode: 5,821 frames, 1,161 overlapping 20-frame windows, 5-frame stride, about 20 FPS.",
        "source_artifacts": [
          "results/episode_task_suite/windows.csv",
          "results/episode_task_suite/shared_windows.npz",
          "results/episode_task_suite/feature_manifest.json",
          "official sample annotation.hdf5",
          "official sample six MP4 camera streams"
        ],
        "input_builder": "Slice each 20-frame window, then join multiview RGB summaries with depth, camera pose, SLAM/calibration, object cues, contact cues, and optional language questions from the public annotation timeline.",
        "target_builder": "Create spatial targets such as camera-view match, object relevance, object-set memory, depth/pose reconstruction proxy, caption-grounded retrieval, and spatial QA answers.",
        "existing_task_hooks": [
          "object_relevance",
          "modality_reconstruction",
          "caption_grounding",
          "object_set_forecast",
          "camera_view_sync_retrieval"
        ],
        "boundary": "This yields a one-episode spatial training-pair recipe and proxy tasks; the next spatial-intelligence readout is held-out depth, pose, and scene-memory metrics."
      },
      "diagram_flow": [
        {
          "stage": "inputs",
          "items": [
            "multiview RGB plus egocentric video",
            "metric depth and confidence",
            "camera pose, calibration, SLAM",
            "object, contact, and language cues"
          ]
        },
        {
          "stage": "tasks_targets",
          "items": [
            "spatial QA and object count",
            "object permanence across windows",
            "relative location and retrieval",
            "pose-aware 3D consistency"
          ]
        },
        {
          "stage": "train_models",
          "items": [
            "export scene/object memory records",
            "train spatial-memory encoder",
            "add geometry-aware QA and retrieval heads",
            "keep episode-level split discipline"
          ]
        },
        {
          "stage": "evaluate_gates",
          "items": [
            "held-out episode spatial metrics",
            "count and relation accuracy",
            "retrieval rank and consistency",
            "saved predictions before public package update"
          ]
        }
      ],
      "next_readout_before_stronger_positioning": [
        "held-out spatial QA",
        "pose consistency",
        "object-counting and scene-memory metrics"
      ]
    },
    {
      "id": "human_video_world_models",
      "title": "Human-video world models",
      "question": "Can the model predict what happens next?",
      "core_inputs": [
        "observed video windows",
        "audio",
        "sensor windows",
        "hand and body motion",
        "object and contact state",
        "action and subtask labels",
        "future windows"
      ],
      "intermediate_artifacts": [
        "observed and future window pairs",
        "future label targets",
        "action-conditioned target records",
        "visual or latent reconstruction targets",
        "temporal consistency metadata"
      ],
      "outputs": [
        "next action",
        "next subtask",
        "future object set",
        "future state embedding",
        "camera-motion delta",
        "contact transition",
        "future-window quality metrics"
      ],
      "first_pipeline": "Keep Qwen-style structured future probes for task interpretability, keep Cosmos-style dynamics branches separate, and add latent or feature-reconstruction metrics before presenting world-model quality.",
      "current_maturity": "Partially evidenced by current future-task probes and Cosmos-style branch artifacts.",
      "next_gate": "Stronger future-state metrics, qualitative future examples, and held-out episode breakdowns.",
      "diagram_image": "docs/assets/foundation-pipelines/human-video-world-model-pipeline.png",
      "website_image": "assets/foundation-pipelines/human-video-world-model-pipeline.png",
      "image_alt": "High-resolution slide diagram showing the Human-video world models direction for Xperience-10M.",
      "one_sample_training_io": {
        "sample_basis": "Single public sample episode: current observed windows are paired with shifted future labels or future-window features from the same timeline.",
        "source_artifacts": [
          "results/episode_task_suite/windows.csv",
          "results/episode_task_suite/shared_windows.npz",
          "results/episode_task_suite/tier2_task_suite/tier2_task_suite_results.json",
          "results/episode_task_suite/research_direction_extensions/research_direction_extension_results.json"
        ],
        "input_builder": "Use the current 20-frame observed window at time t: RGB/audio/sensor summaries, hand/body motion, camera pose, current object/contact state, and current action/subtask context only.",
        "target_builder": "Shift the episode timeline forward to produce next-action, next-subtask, future object-set, contact-transition, time-to-transition, camera-motion delta, or latent/future-feature targets.",
        "existing_task_hooks": [
          "next_action",
          "long_horizon_next_action",
          "next_subtask_forecast",
          "object_set_forecast",
          "time_to_transition",
          "ego_motion_forecast"
        ],
        "boundary": "Future labels and future windows must stay out of the input. Structured future probes show the pipeline, while visual world-model readouts need latent or visual future metrics."
      },
      "diagram_flow": [
        {
          "stage": "inputs",
          "items": [
            "observed video/audio/sensor window",
            "hand/body motion and camera pose",
            "object/contact state",
            "action and subtask labels"
          ]
        },
        {
          "stage": "tasks_targets",
          "items": [
            "next action and next subtask",
            "future object set",
            "contact transition",
            "camera-motion delta or latent future"
          ]
        },
        {
          "stage": "train_models",
          "items": [
            "Qwen structured future probes",
            "Cosmos/dynamics branch separately",
            "latent rollout or reconstruction loss",
            "no target-side future leakage"
          ]
        },
        {
          "stage": "evaluate_gates",
          "items": [
            "held-out future-task metrics",
            "contact and object-set F1",
            "rollout or latent consistency",
            "per-episode breakdown and examples"
          ]
        }
      ],
      "next_readout_before_stronger_positioning": [
        "latent or visual future metrics",
        "per-episode future-task breakdowns",
        "qualitative examples backed by saved targets"
      ]
    },
    {
      "id": "vision_language_action",
      "title": "Vision-language-action models",
      "question": "Can the model turn what it sees and reads into action?",
      "core_inputs": [
        "egocentric video",
        "language captions",
        "hand and body motion",
        "contacts",
        "objects",
        "procedure and subtask labels"
      ],
      "intermediate_artifacts": [
        "action-token vocabulary",
        "action-chunk windows",
        "normalization stats",
        "retargeting report",
        "leakage audit",
        "action-space model card"
      ],
      "outputs": [
        "next action",
        "action chunk",
        "object-conditioned action",
        "contact state",
        "subtask transition",
        "policy or VLA held-out metrics"
      ],
      "first_pipeline": "Define the action space, use existing 20-task next-action/contact/object-conditioned tasks first, then add hand-trajectory or policy-compatible action chunks after conversion is traceable.",
      "current_maturity": "Feasible but gated by action-target conversion.",
      "next_gate": "Traceable action tokens, normalization, retargeting metadata, and held-out policy metrics.",
      "diagram_image": "docs/assets/foundation-pipelines/vision-language-action-pipeline.png",
      "website_image": "assets/foundation-pipelines/vision-language-action-pipeline.png",
      "image_alt": "High-resolution slide diagram showing the Vision-language-action models direction for Xperience-10M.",
      "one_sample_training_io": {
        "sample_basis": "Single public sample episode: observation-language windows are paired with action-token proxies because robot retargeted action chunks are not part of the public sample yet.",
        "source_artifacts": [
          "results/episode_task_suite/windows.csv",
          "results/episode_task_suite/shared_windows.npz",
          "results/episode_task_suite/task_walkthroughs/task_walkthroughs.json",
          "official sample annotation.hdf5"
        ],
        "input_builder": "Use egocentric/fisheye video windows, caption and object context, hand/body mocap, contact state, and current subtask text as the observation-language side of each training pair.",
        "target_builder": "Create action-token proxy targets: current or next action, object-conditioned action relation, contact state, interaction-text class, subtask transition, or hand-trajectory/action-chunk proxy.",
        "existing_task_hooks": [
          "timeline_action",
          "next_action",
          "hand_trajectory_forecast",
          "contact_prediction",
          "interaction_text_prediction",
          "action_object_relation"
        ],
        "boundary": "This is a VLA/policy data-conversion recipe for the one-sample suite. Robot policy readouts require a later action-space converter, normalization, retargeting report, and held-out policy metrics."
      },
      "diagram_flow": [
        {
          "stage": "inputs",
          "items": [
            "egocentric video and captions",
            "objects, contacts, and procedures",
            "hand/body motion windows",
            "subtask labels and language context"
          ]
        },
        {
          "stage": "tasks_targets",
          "items": [
            "action-token vocabulary",
            "next action and action chunks",
            "object-conditioned actions",
            "contact state and subtask transition"
          ]
        },
        {
          "stage": "train_models",
          "items": [
            "build action-space converter",
            "normalize and audit action chunks",
            "train VLA/policy-compatible head",
            "track leakage and retargeting reports"
          ]
        },
        {
          "stage": "evaluate_gates",
          "items": [
            "held-out action metrics",
            "chunk and next-action accuracy",
            "object/contact-conditioned scores",
            "policy card before robot-policy quality readout"
          ]
        }
      ],
      "next_readout_before_stronger_positioning": [
        "action-space conversion",
        "normalized action chunks",
        "held-out policy metrics"
      ]
    }
  ]
}