metrics/research_roadmap.json

{
  "title": "Ropedia Xperience-10M Research Roadmap",
  "summary": "Staged path from the public-sample task lab to a verified validation-aware Qwen3-Omni diagnostic pilot, structured-output improvement pass, foundation-model selection, world/policy branches, and a future Xperience-native embodied foundation model.",
  "current_decision_point": "Keep the public-sample task suite as the development harness, use the verified selected-episode Qwen3-Omni validation-aware diagnostic pilot as the first cross-episode baseline, improve structured-output reliability and task-quality error analysis, then branch into Cosmos 3 world modeling and policy-model experiments after their targets are implemented. The Xperience Embodied Foundation Model is a later full-corpus pretraining goal, not a current result.",
  "additional_development_directions": {
    "source_document": "ADDITIONAL_DEVELOPMENT_DIRECTIONS.md",
    "source_json": "docs/data/additional_development_directions.json",
    "summary": "Additional concrete tracks include episode taxonomy and data selection, benchmark protocol, multimodal representation learning, skill graphs, affordance modeling, 3D/4D scene memory, data-quality diagnostics, and policy/simulation transfer."
  },
  "phases": [
    {
      "id": "public_sample_task_lab",
      "name": "Public-Sample Task Lab",
      "status": "implemented",
      "entry_condition": "One public Xperience-10M sample episode is available.",
      "deliverables": [
        "1161 aligned windows",
        "12 task contracts",
        "minimal baseline heads",
        "neural MLP heads",
        "modality atlas",
        "task walkthroughs",
        "derived figures"
      ],
      "completion_evidence": [
        "PROJECT_STATUS.md",
        "EVALUATION_PROTOCOL.md",
        "RESEARCH_TAKEAWAYS.md",
        "docs/data/summary_metrics.json",
        "results/episode_task_suite/summary_report.json"
      ],
      "reader_takeaway": "The public sample supports task design, feature contracts, walkthroughs, and baseline comparisons."
    },
    {
      "id": "multi_episode_data_staging",
      "name": "Multi-Episode Data Preparation",
      "status": "implemented_for_first_pilot",
      "entry_condition": "Gated dataset availability and enough storage for selected episodes.",
      "deliverables": [
        "128 selected episodes",
        "episode manifest",
        "missing-view manifest",
        "held-out episode split",
        "source-discovery report"
      ],
      "completion_evidence": [
        "results/omni_finetune/DATA_ACCESS_STATUS.md",
        "results/omni_finetune/MULTI_EPISODE_ACCESS_STATUS.md",
        "results/omni_finetune/source_discovery.json"
      ],
      "reader_takeaway": "The first selected split is available for Qwen3-Omni diagnostics, with train/test separation at the episode level."
    },
    {
      "id": "qwen3_omni_lora_diagnostic_pilot",
      "name": "Qwen3-Omni LoRA Validation-Aware Diagnostic Pilot",
      "status": "verified_baseline",
      "entry_condition": "Selected episodes are prepared locally with no train/test episode leakage.",
      "deliverables": [
        "dataset JSONL/media manifests",
        "LoRA adapter checkpoint",
        "progress logs",
        "validation monitoring",
        "held-out predictions",
        "metrics",
        "confusion matrices",
        "run report"
      ],
      "completion_evidence": [
        "docs/data/omni_finetune_verified_result.json",
        "results/omni_finetune/verified_public/",
        "dataset_manifest.json",
        "training_metadata.json",
        "progress.jsonl",
        "metrics.json",
        "predictions.jsonl",
        "RUN_REPORT.md"
      ],
      "reader_takeaway": "The first omni-model pilot establishes the full held-out training/validation/evaluation loop, but the weak metrics make it a diagnostic baseline."
    },
    {
      "id": "qwen3_omni_structured_output_error_analysis",
      "name": "Structured-Output And Error-Analysis Pass",
      "status": "active_next_step",
      "entry_condition": "The validation-aware diagnostic package exists and shows weak held-out quality.",
      "deliverables": [
        "same 96/16/16 episode split",
        "stricter JSON decoding or target formatting",
        "episode/action/object error analysis",
        "held-out test evaluation",
        "comparison to the verified validation-aware baseline"
      ],
      "completion_evidence": [
        "quality-target report",
        "error-analysis tables",
        "held-out metrics",
        "verified public-safe package"
      ],
      "reader_takeaway": "The next pass should improve output reliability and task metrics before larger model-quality claims."
    },
    {
      "id": "foundation_model_selection_matrix",
      "name": "Foundation-Model Selection Matrix",
      "status": "next",
      "entry_condition": "The selected episodes are prepared or a 3-8 episode dry run is available for preprocessing checks.",
      "deliverables": [
        "backbone registry",
        "Cosmos 3 world-model branch plan",
        "Qwen3-Omni LoRA baseline plan",
        "OpenVLA/openpi/GR00T policy-branch candidates",
        "model-specific evaluation additions"
      ],
      "completion_evidence": [
        "FOUNDATION_MODEL_PLAN.md",
        "docs/data/foundation_model_plan.json",
        "research_roadmap_interactive.json"
      ],
      "reader_takeaway": "Qwen3-Omni remains the first trainable held-out pilot; Cosmos 3 is the first world-model branch; VLA/policy models wait for explicit action targets."
    },
    {
      "id": "robustness_run_64_128_episode",
      "name": "64-128 Episode Robustness Run",
      "status": "planned",
      "entry_condition": "The selected-episode pilot trains and evaluates cleanly.",
      "deliverables": [
        "split-by-session metrics",
        "modality ablations",
        "calibration/object/language error analysis",
        "missing-view sensitivity analysis"
      ],
      "completion_evidence": [
        "held-out metrics by session",
        "held-out metrics by task",
        "held-out metrics by modality",
        "ablation tables",
        "qualitative error analysis"
      ],
      "reader_takeaway": "The robustness run tests whether the pilot conclusions survive broader sessions and missing modalities."
    },
    {
      "id": "foundation_world_model_extensions",
      "name": "Cosmos 3 and Policy-Model Extensions",
      "status": "planned",
      "entry_condition": "Enough multi-episode data, compute budget, and model-specific action/world-state targets.",
      "deliverables": [
        "Cosmos 3 future-window or action-conditioned world-model probe",
        "OpenVLA/openpi/GR00T action-policy baseline",
        "audio/video/depth/pose/mocap conditioning checks",
        "affordance and object-interaction tasks",
        "synthetic-data usefulness test"
      ],
      "completion_evidence": [
        "task-specific held-out evaluations",
        "qualitative inspection",
        "updated model cards"
      ],
      "reader_takeaway": "The long-term direction is richer multimodal representation learning for embodied-AI reasoning, with model branches chosen by task fit rather than by a single default backbone."
    },
    {
      "id": "xperience_embodied_foundation_pretraining",
      "name": "Xperience Embodied Foundation Model Pretraining",
      "status": "future",
      "entry_condition": "Full-corpus access, PB-scale storage path, high-throughput data loading, multi-node compute, and positive scaling evidence from smaller multi-episode runs.",
      "deliverables": [
        "full-corpus episode and split manifests",
        "pretraining shard and provenance manifests",
        "0.3B-1B and 1B-3B scaling pilots",
        "3B-7B Xperience-native domain model target",
        "held-out episode/session/activity/object evaluations",
        "missing-modality robustness report",
        "model card and data-boundary report"
      ],
      "completion_evidence": [
        "pretraining metadata",
        "checkpoint inventory",
        "scaling curves",
        "held-out evaluation reports",
        "qualitative retrieval or future-state examples",
        "safety and data-boundary report"
      ],
      "reader_takeaway": "The final research direction is a domain-specific embodied foundation model trained directly on Xperience-10M, after smaller pilots justify the cost and infrastructure."
    }
  ],
  "public_surfaces_to_update": [
    "README.md",
    "PROJECT_STATUS.md",
    "RESEARCH_TAKEAWAYS.md",
    "EVALUATION_PROTOCOL.md",
    "ARTIFACT_GUIDE.md",
    "ADDITIONAL_DEVELOPMENT_DIRECTIONS.md",
    "XPERIENCE_EMBODIED_FOUNDATION_MODEL_PRETRAINING.md",
    "docs/index.html",
    "docs/data/additional_development_directions.json",
    "docs/data/research_roadmap.json",
    "Hugging Face Space card",
    "Hugging Face artifact dataset card",
    "Hugging Face model card"
  ]
}