data/research_roadmap.json

{
    "title": "Ropedia Xperience-10M Research Roadmap",
    "summary": "Staged path from the public-sample task lab to verified Qwen3-Omni, Cosmos3-Nano, and Cosmos3-Super diagnostics, same-split 128-episode baseline alignment, a no-new-episode 128-suite enhancement pack, action/subtask error analysis, world/policy branches, and a future Xperience-native embodied foundation model.",
    "current_decision_point": "Push the current selected 128 episodes harder before requesting more storage: keep the public-sample task suite as the development harness, use the latest verified selected-episode Qwen3-Omni v6 diagnostic branch plus the pinned v5 row as structured-task references, read Cosmos3-Nano and Cosmos3-Super Forward-Dynamics LoRA as separate world-model results, continue with hierarchical action/subtask targets and label-normalized scoring, and defer policy-model experiments until robot-compatible targets are implemented. The three headline directions should be organized as spatial-intelligence, human-video world-model, and vision-language-action pipeline tracks with separate artifact gates. The Xperience Embodied Foundation Model is a later full-corpus pretraining goal, not a current result.",
    "three_foundation_pipelines": {
        "source_document": "THREE_FOUNDATION_PIPELINES.md",
        "source_json": "docs/data/three_foundation_pipelines.json",
        "summary": "Three pipeline tracks organize the foundation-model story: spatial intelligence needs depth/pose-backed scene-memory targets and spatial metrics, human-video world modeling needs future-state or visual/latent future metrics, and vision-language-action needs action-token conversion plus policy-style held-out metrics."
    },
    "additional_development_directions": {
        "source_document": "ADDITIONAL_DEVELOPMENT_DIRECTIONS.md",
        "source_json": "docs/data/additional_development_directions.json",
        "summary": "Additional concrete tracks include episode taxonomy and data selection, benchmark protocol, multimodal representation learning, skill graphs, affordance modeling, 3D/4D scene memory, data-quality diagnostics, and policy/simulation transfer."
    },
    "phases": [
        {
            "id": "public_sample_task_lab",
            "name": "Public-Sample Task Lab",
            "status": "implemented",
            "entry_condition": "One public Xperience-10M sample episode is available.",
            "deliverables": [
                "1161 aligned windows",
                "12 task contracts",
                "minimal baseline heads",
                "neural MLP heads",
                "modality atlas",
                "task walkthroughs",
                "derived figures"
            ],
            "completion_evidence": [
                "PROJECT_STATUS.md",
                "EVALUATION_PROTOCOL.md",
                "RESEARCH_TAKEAWAYS.md",
                "docs/data/summary_metrics.json",
                "results/episode_task_suite/summary_report.json"
            ],
            "reader_takeaway": "The public sample supports task design, feature contracts, walkthroughs, and baseline comparisons."
        },
        {
            "id": "multi_episode_data_staging",
            "name": "Multi-Episode Data Preparation",
            "status": "implemented_for_first_pilot",
            "entry_condition": "Gated dataset availability and enough storage for selected episodes.",
            "deliverables": [
                "128 selected episodes",
                "episode manifest",
                "missing-view manifest",
                "held-out episode split",
                "source-discovery report"
            ],
            "completion_evidence": [
                "results/omni_finetune/DATA_ACCESS_STATUS.md",
                "results/omni_finetune/MULTI_EPISODE_ACCESS_STATUS.md",
                "results/omni_finetune/source_discovery.json"
            ],
            "reader_takeaway": "The first selected split is available for Qwen3-Omni diagnostics, with train/test separation at the episode level."
        },
        {
            "id": "qwen3_omni_lora_diagnostic_pilot",
            "name": "Qwen3-Omni LoRA Latest Diagnostic Branch",
            "status": "verified_latest_branch",
            "entry_condition": "Selected episodes are prepared locally with no train/test episode leakage.",
            "deliverables": [
                "dataset JSONL/media manifests",
                "LoRA adapter checkpoint",
                "progress logs",
                "validation monitoring",
                "held-out predictions",
                "metrics",
                "confusion matrices",
                "run report",
                "v5/v6 comparison",
                "public LoRA adapter repo"
            ],
            "completion_evidence": [
                "docs/data/omni_finetune_verified_result.json",
                "docs/data/qwen3_v5_v6_comparison.json",
                "results/omni_finetune/QWEN3_V5_V6_COMPARISON_20260614.md",
                "results/omni_finetune/verified_public/",
                "dataset_manifest.json",
                "training_metadata.json",
                "progress.jsonl",
                "metrics.json",
                "predictions.jsonl",
                "RUN_REPORT.md"
            ],
            "reader_takeaway": "The final omni-model diagnostic result establishes the full held-out training/validation/evaluation loop and meets the strict-JSON target, but weak action/subtask metrics make it a diagnostic baseline."
        },
        {
            "id": "multi_episode_128_same_split_baselines",
            "name": "128-Episode Same-Split Simple/NN Baselines",
            "status": "verified_companion_result",
            "entry_condition": "Derived Qwen JSONL export for the selected 96/16/16 split.",
            "deliverables": [
                "same 12 task ids",
                "simple metadata/text baselines",
                "neural MLP baselines for JSON-supported labels",
                "explicit unsupported markers for raw-feature-only tasks"
            ],
            "completion_evidence": [
                "results/omni_finetune/multi_episode_128_task_baselines/BASELINE_ALIGNMENT_REPORT.md",
                "results/omni_finetune/multi_episode_128_task_baselines/summary_report.json",
                "scripts/omni/run_128_task_baselines.py"
            ],
            "reader_takeaway": "The simple and neural baseline framing is now aligned to the selected 128-episode setup; trajectory, retrieval, reconstruction, and misalignment variants still need raw 128 feature blocks for exact feature-level reproduction."
        },
        {
            "id": "task_suite_enhancement_128",
            "name": "128-Episode Task Suite Enhancement Pack",
            "status": "current",
            "entry_condition": "Same selected 96/16/16 split and current public 3,808-window export.",
            "deliverables": [
                "dense-window and multiscale export estimates",
                "hierarchical action/subtask target contract",
                "raw-feature shard priorities for unsupported tasks",
                "Qwen v5 and Cosmos continuation run cards",
                "publication-ready enhancement artifacts"
            ],
            "completion_evidence": [
                "TASK_SUITE_ENHANCEMENT_128.md",
                "docs/data/task_suite_enhancement_128.json",
                "results/omni_finetune/task_suite_enhancement_128_v1_20260608/enhancement_plan.json",
                "scripts/omni/build_task_suite_enhancement_128.py"
            ],
            "reader_takeaway": "The current 128-episode setup still has headroom: use multiscale_20s10_40s20_80s40, hierarchical labels, label-normalized scoring, and raw-feature shards before adding more episodes."
        },
        {
            "id": "qwen3_omni_structured_output_error_analysis",
            "name": "Action/Subtask Error-Analysis Pass",
            "status": "active_next_step",
            "entry_condition": "The final diagnostic package meets strict JSON validity but has weak action/subtask held-out quality.",
            "deliverables": [
                "same 96/16/16 episode split",
                "action/subtask confusion analysis",
                "unseen-label analysis",
                "object/action family breakdowns",
                "held-out test evaluation",
                "comparison to the final verified Qwen baseline"
            ],
            "completion_evidence": [
                "error-analysis tables",
                "held-out metrics by failure type",
                "verified public-safe package"
            ],
            "reader_takeaway": "The next pass should improve action/subtask quality before presenting stronger model-quality numbers."
        },
        {
            "id": "foundation_model_selection_matrix",
            "name": "Foundation-Model Selection Matrix",
            "status": "current",
            "entry_condition": "The selected episodes are prepared or a 3-8 episode dry run is available for preprocessing checks.",
            "deliverables": [
                "backbone registry",
                "Cosmos 3 world-model track plan",
                "Cosmos3-Super Forward-Dynamics LoRA verified package",
                "Qwen3-Omni LoRA baseline plan",
                "OpenVLA/openpi/GR00T policy-branch candidates",
                "model-specific evaluation additions"
            ],
            "completion_evidence": [
                "FOUNDATION_MODEL_PLAN.md",
                "docs/data/foundation_model_plan.json",
                "research_roadmap_interactive.json"
            ],
            "reader_takeaway": "Qwen3-Omni remains the structured JSON held-out pilot; Cosmos 3 is the first world-model track. Cosmos3-Super now has a verified forward-dynamics LoRA over camera-pose proxy targets, while VLA/policy models wait for robot-compatible action targets."
        },
        {
            "id": "robustness_run_64_128_episode",
            "name": "64-128 Episode Robustness Run",
            "status": "partially_implemented",
            "entry_condition": "The selected-episode pilot trains and evaluates cleanly.",
            "deliverables": [
                "split-by-session metrics",
                "modality ablations",
                "calibration/object/language error analysis",
                "missing-view sensitivity analysis"
            ],
            "completion_evidence": [
                "held-out metrics by session",
                "held-out metrics by task",
                "held-out metrics by modality",
                "ablation tables",
                "qualitative error analysis"
            ],
            "reader_takeaway": "The robustness run tests whether the pilot conclusions survive broader sessions and missing modalities."
        },
        {
            "id": "foundation_world_model_extensions",
            "name": "Cosmos 3 and Policy-Model Extensions",
            "status": "planned",
            "entry_condition": "Enough multi-episode data, compute budget, and model-specific action/world-state targets.",
            "deliverables": [
                "Cosmos 3 future-window and action-conditioned world-model probes",
                "OpenVLA/openpi/GR00T action-policy baseline",
                "audio/video/depth/pose/mocap conditioning checks",
                "affordance and object-interaction tasks",
                "synthetic-data usefulness test"
            ],
            "completion_evidence": [
                "task-specific held-out evaluations",
                "verified Cosmos3-Super forward-dynamics LoRA package",
                "qualitative inspection",
                "updated model cards"
            ],
            "reader_takeaway": "The Cosmos3 track now includes Nano future-window compatibility and Super forward-dynamics LoRA; the long-term direction remains richer multimodal representation learning with model tracks chosen by task fit rather than by a single default backbone."
        },
        {
            "id": "xperience_embodied_foundation_pretraining",
            "name": "Xperience Embodied Foundation Model Pretraining",
            "status": "future",
            "entry_condition": "Full-corpus access, PB-scale storage path, high-throughput data loading, multi-node compute, and positive scaling evidence from smaller multi-episode runs.",
            "deliverables": [
                "full-corpus episode and split manifests",
                "pretraining shard and provenance manifests",
                "0.3B-1B and 1B-3B scaling pilots",
                "3B-7B Xperience-native domain model target",
                "held-out episode/session/activity/object evaluations",
                "missing-modality robustness report",
                "model card and data-boundary report"
            ],
            "completion_evidence": [
                "pretraining metadata",
                "checkpoint inventory",
                "scaling curves",
                "held-out evaluation reports",
                "qualitative retrieval or future-state examples",
                "safety and data-boundary report"
            ],
            "reader_takeaway": "The final research direction is a domain-specific embodied foundation model trained directly on Xperience-10M, after smaller pilots justify the cost and infrastructure."
        }
    ],
    "public_surfaces_to_update": [
        "README.md",
        "docs/data/task_suite_enhancement_128.json",
        "TASK_SUITE_ENHANCEMENT_128.md",
        "PROJECT_STATUS.md",
        "RESEARCH_TAKEAWAYS.md",
        "EVALUATION_PROTOCOL.md",
        "ARTIFACT_GUIDE.md",
        "ADDITIONAL_DEVELOPMENT_DIRECTIONS.md",
        "XPERIENCE_EMBODIED_FOUNDATION_MODEL_PRETRAINING.md",
        "docs/index.html",
        "docs/data/additional_development_directions.json",
        "docs/data/research_roadmap.json",
        "Hugging Face Space card",
        "Hugging Face artifact dataset card",
        "Hugging Face model card"
    ]
}