| { |
| "source": "docs/data/task_suite_20.json plus results/episode_task_suite/summary_report.json", |
| "dataset_scope": { |
| "sample_episode_count": 1, |
| "num_frames": 5821, |
| "num_windows": 1161, |
| "feature_dim": 8546, |
| "warning": "Single public sample episode; this supports pipeline/task evidence, while cross-episode generalization requires held-out episodes." |
| }, |
| "baselines": { |
| "minimal": "Interpretable softmax, logistic, ridge, and retrieval heads over the 8,546-d window feature vector.", |
| "neural_mlp": "Small PyTorch MLP classifiers/regressors using the same features, splits, and task contracts." |
| }, |
| "task_count": 20, |
| "directions": { |
| "A": { |
| "id": "human_motion", |
| "name": "Human Modeling & Motion Understanding", |
| "focus": "Human/hand/body motion, deformation priors, human-object interaction, affordance modeling.", |
| "preferred_background": "Human pose/shape estimation, SMPL-style models, motion capture, or motion generation.", |
| "current_status": "partially implemented", |
| "current_readout": "The sample supports hand trajectory forecasting and contact/object probes, but it does not yet include a full body/shape model or multi-person priors.", |
| "next_steps": [ |
| "Add SMPL/SMPL-X or MANO-style body/hand parameter targets where available.", |
| "Train sequence models over multi-episode motion trajectories instead of isolated windows.", |
| "Evaluate affordance prediction on held-out objects and held-out episodes." |
| ], |
| "tasks": [ |
| "timeline_action", |
| "hand_trajectory_forecast", |
| "contact_prediction", |
| "object_relevance", |
| "interaction_text_prediction", |
| "imu_to_hand_pose" |
| ], |
| "task_display_names": [ |
| "Action Recognition", |
| "Hand Trajectory Forecasting", |
| "Contact State Prediction", |
| "Object Relevance Prediction", |
| "Interaction Text Prediction", |
| "IMU-to-Hand Pose Reconstruction" |
| ], |
| "counts": { |
| "direct": 3, |
| "proxy": 3, |
| "diagnostic": 0, |
| "total_links": 6 |
| } |
| }, |
| "B": { |
| "id": "reconstruction_rendering", |
| "name": "3D/4D Reconstruction & Neural Rendering", |
| "focus": "Multi-view dynamic scene reconstruction, NeRF/Gaussian Splatting, novel-view synthesis.", |
| "preferred_background": "3D reconstruction, neural rendering, camera calibration, and bundle adjustment.", |
| "current_status": "proxy tasks only", |
| "current_readout": "The current suite checks cross-modal alignment and depth/video reconstruction proxies; it does not yet train a renderer or reconstruct geometry.", |
| "next_steps": [ |
| "Use calibrated multi-view video plus SLAM pose to build per-episode camera trajectories.", |
| "Add depth-supervised point clouds, TSDF, Gaussian Splatting, or NeRF baselines.", |
| "Evaluate novel-view synthesis and temporal consistency across held-out views/time." |
| ], |
| "tasks": [ |
| "cross_modal_retrieval", |
| "modality_reconstruction", |
| "misalignment_detection", |
| "imu_to_hand_pose", |
| "camera_view_sync_retrieval" |
| ], |
| "task_display_names": [ |
| "Cross-Modal Retrieval", |
| "Cross-Modal Reconstruction", |
| "Multimodal Synchronization Detection", |
| "IMU-to-Hand Pose Reconstruction", |
| "Camera-View Synchronization Retrieval" |
| ], |
| "counts": { |
| "direct": 1, |
| "proxy": 3, |
| "diagnostic": 1, |
| "total_links": 5 |
| } |
| }, |
| "C": { |
| "id": "egocentric_interaction", |
| "name": "Egocentric Vision & Interaction", |
| "focus": "Egocentric action and intention understanding, hand-object interaction, gaze/attention modeling, task structure modeling.", |
| "preferred_background": "Video understanding, action recognition, or egocentric vision.", |
| "current_status": "strongest implemented track", |
| "current_readout": "The unified 20-task suite directly targets egocentric action, task state, interaction, grounding, forecasting, and alignment.", |
| "next_steps": [ |
| "Move from single-episode chronological splits to held-out-episode splits.", |
| "Use audio together with stronger multimodal backbones for action, intent, and grounding.", |
| "Evaluate long-horizon task success prediction and action-conditioned generation." |
| ], |
| "tasks": [ |
| "timeline_action", |
| "timeline_subtask", |
| "transition_detection", |
| "next_action", |
| "hand_trajectory_forecast", |
| "contact_prediction", |
| "object_relevance", |
| "caption_grounding", |
| "cross_modal_retrieval", |
| "temporal_order", |
| "misalignment_detection", |
| "long_horizon_next_action", |
| "next_subtask_forecast", |
| "interaction_text_prediction", |
| "action_object_relation", |
| "object_set_forecast", |
| "time_to_transition" |
| ], |
| "task_display_names": [ |
| "Action Recognition", |
| "Procedure Step Recognition", |
| "Action Boundary Detection", |
| "Next-Action Prediction", |
| "Hand Trajectory Forecasting", |
| "Contact State Prediction", |
| "Object Relevance Prediction", |
| "Language Grounding", |
| "Cross-Modal Retrieval", |
| "Temporal Order Verification", |
| "Multimodal Synchronization Detection", |
| "Long-Horizon Next-Action Forecasting", |
| "Long-Horizon Next-Subtask Forecasting", |
| "Interaction Text Prediction", |
| "Action-Object Relation Prediction", |
| "Future Object-Set Forecasting", |
| "Time-to-Next-Transition Regression" |
| ], |
| "counts": { |
| "direct": 10, |
| "proxy": 3, |
| "diagnostic": 4, |
| "total_links": 17 |
| } |
| }, |
| "D": { |
| "id": "world_modeling", |
| "name": "Scene Reconstruction & World Modeling", |
| "focus": "Long-term consistent 3D/4D scene mapping, scene graphs, object- and space-centric representations, spatial reasoning.", |
| "preferred_background": "Large-scale mapping, semantic reconstruction, or agent world models.", |
| "current_status": "early proxy tasks", |
| "current_readout": "The current tasks probe temporal structure, object relevance, cross-modal retrieval, and modality prediction, but they do not yet build persistent maps or scene graphs.", |
| "next_steps": [ |
| "Convert windows into persistent object/scene-state nodes with timestamps and camera poses.", |
| "Add map consistency, object permanence, and spatial relation prediction tasks.", |
| "Train held-out-episode world models that predict future observations and task state." |
| ], |
| "tasks": [ |
| "timeline_subtask", |
| "transition_detection", |
| "next_action", |
| "object_relevance", |
| "caption_grounding", |
| "cross_modal_retrieval", |
| "modality_reconstruction", |
| "temporal_order", |
| "misalignment_detection", |
| "long_horizon_next_action", |
| "next_subtask_forecast", |
| "action_object_relation", |
| "object_set_forecast", |
| "camera_view_sync_retrieval", |
| "time_to_transition" |
| ], |
| "task_display_names": [ |
| "Procedure Step Recognition", |
| "Action Boundary Detection", |
| "Next-Action Prediction", |
| "Object Relevance Prediction", |
| "Language Grounding", |
| "Cross-Modal Retrieval", |
| "Cross-Modal Reconstruction", |
| "Temporal Order Verification", |
| "Multimodal Synchronization Detection", |
| "Long-Horizon Next-Action Forecasting", |
| "Long-Horizon Next-Subtask Forecasting", |
| "Action-Object Relation Prediction", |
| "Future Object-Set Forecasting", |
| "Camera-View Synchronization Retrieval", |
| "Time-to-Next-Transition Regression" |
| ], |
| "counts": { |
| "direct": 1, |
| "proxy": 10, |
| "diagnostic": 4, |
| "total_links": 15 |
| } |
| } |
| }, |
| "tasks": { |
| "timeline_action": { |
| "name": "Timeline action recognition", |
| "family": "supervised", |
| "input": "all featurized modalities", |
| "output": "current action label", |
| "primary_direction": "C", |
| "direction_roles": { |
| "C": "direct", |
| "A": "proxy" |
| }, |
| "why": "Reads egocentric sensor state as the current human action; also provides a weak human-motion readout.", |
| "current_limit": "Chronological single-episode split creates unseen future action classes.", |
| "display_name": "Action Recognition", |
| "artifact_id": "timeline_action", |
| "metric": { |
| "key": "macro_f1", |
| "name": "macro-F1", |
| "direction": "higher", |
| "minimal": 0.05, |
| "neural_mlp": 0.014814814814814814, |
| "better_baseline": "minimal" |
| } |
| }, |
| "timeline_subtask": { |
| "name": "Timeline subtask recognition", |
| "family": "supervised", |
| "input": "all featurized modalities", |
| "output": "current subtask label", |
| "primary_direction": "C", |
| "direction_roles": { |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "why": "Segments egocentric task state and provides a first proxy for symbolic world/task state.", |
| "current_limit": "Single-episode ordering makes future subtasks hard to generalize.", |
| "display_name": "Procedure Step Recognition", |
| "artifact_id": "timeline_subtask", |
| "metric": { |
| "key": "macro_f1", |
| "name": "macro-F1", |
| "direction": "higher", |
| "minimal": 0.05056355513846935, |
| "neural_mlp": 0.02810810810810811, |
| "better_baseline": "minimal" |
| } |
| }, |
| "transition_detection": { |
| "name": "Action transition detection", |
| "family": "diagnostic", |
| "input": "all featurized modalities", |
| "output": "boundary vs steady state", |
| "primary_direction": "C", |
| "direction_roles": { |
| "C": "direct", |
| "D": "diagnostic" |
| }, |
| "why": "Localizes egocentric task boundaries and diagnoses temporal state changes.", |
| "current_limit": "Boundary class is sparse, so accuracy alone is misleading.", |
| "display_name": "Action Boundary Detection", |
| "artifact_id": "transition_detection", |
| "metric": { |
| "key": "macro_f1", |
| "name": "macro-F1", |
| "direction": "higher", |
| "minimal": 0.6118237590630229, |
| "neural_mlp": 0.5862068965517241, |
| "better_baseline": "minimal" |
| } |
| }, |
| "next_action": { |
| "name": "Short-horizon next action", |
| "family": "supervised", |
| "input": "current multimodal window", |
| "output": "action 20 frames later", |
| "primary_direction": "C", |
| "direction_roles": { |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "why": "Tests action intention/task-flow prediction from egocentric context.", |
| "current_limit": "Unseen future labels dominate the single-episode chronological test.", |
| "display_name": "Next-Action Prediction", |
| "artifact_id": "next_action", |
| "metric": { |
| "key": "macro_f1", |
| "name": "macro-F1", |
| "direction": "higher", |
| "minimal": 0.05925925925925927, |
| "neural_mlp": 0.04186046511627907, |
| "better_baseline": "minimal" |
| } |
| }, |
| "hand_trajectory_forecast": { |
| "name": "Hand trajectory forecasting", |
| "family": "forecast", |
| "input": "current multimodal window", |
| "output": "future left/right hand 3D joints", |
| "primary_direction": "A", |
| "direction_roles": { |
| "A": "direct", |
| "C": "proxy" |
| }, |
| "why": "Directly predicts human hand motion and supports hand-object interaction modeling.", |
| "current_limit": "Forecasting is window-level and not yet a full sequence or policy model.", |
| "display_name": "Hand Trajectory Forecasting", |
| "artifact_id": "hand_trajectory_forecast", |
| "metric": { |
| "key": "mpjpe", |
| "name": "MPJPE", |
| "direction": "lower", |
| "minimal": 0.8646570444107056, |
| "neural_mlp": 0.10785018652677536, |
| "better_baseline": "neural_mlp" |
| } |
| }, |
| "contact_prediction": { |
| "name": "Body/object contact prediction", |
| "family": "supervised", |
| "input": "non-contact/non-caption features", |
| "output": "binary contact label", |
| "primary_direction": "A", |
| "direction_roles": { |
| "A": "direct", |
| "C": "proxy" |
| }, |
| "why": "Targets physical interaction state, a core affordance and manipulation signal.", |
| "current_limit": "The public sample is degenerate for this target because one class dominates.", |
| "display_name": "Contact State Prediction", |
| "artifact_id": "contact_prediction", |
| "metric": { |
| "key": "macro_f1", |
| "name": "macro-F1", |
| "direction": "higher", |
| "minimal": 1.0, |
| "neural_mlp": 1.0, |
| "better_baseline": "tie" |
| } |
| }, |
| "object_relevance": { |
| "name": "Relevant object set prediction", |
| "family": "supervised", |
| "input": "non-caption feature blocks", |
| "output": "multi-label object set", |
| "primary_direction": "C", |
| "direction_roles": { |
| "C": "direct", |
| "A": "proxy", |
| "D": "proxy" |
| }, |
| "why": "Connects egocentric activity to manipulated objects and early object-centric state.", |
| "current_limit": "Object labels are language-derived and sparse in one episode.", |
| "display_name": "Object Relevance Prediction", |
| "artifact_id": "object_relevance", |
| "metric": { |
| "key": "micro_f1", |
| "name": "micro-F1", |
| "direction": "higher", |
| "minimal": 0.18034382095361662, |
| "neural_mlp": 0.1679279279279279, |
| "better_baseline": "minimal" |
| } |
| }, |
| "caption_grounding": { |
| "name": "Caption-to-window grounding", |
| "family": "retrieval", |
| "input": "caption objects/interaction query and candidate sensor windows", |
| "output": "matching time window", |
| "primary_direction": "C", |
| "direction_roles": { |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "why": "Grounds language annotation into egocentric sensor time and task state.", |
| "current_limit": "Bag-of-objects language features are too weak for rich grounding.", |
| "display_name": "Language Grounding", |
| "artifact_id": "caption_grounding", |
| "metric": { |
| "key": "mrr", |
| "name": "MRR", |
| "direction": "higher", |
| "minimal": 0.016023479050338015, |
| "neural_mlp": 0.01684125567132316, |
| "better_baseline": "neural_mlp" |
| } |
| }, |
| "cross_modal_retrieval": { |
| "name": "Cross-modal retrieval", |
| "family": "retrieval", |
| "input": "motion/IMU/camera query", |
| "output": "matching depth/video window", |
| "primary_direction": "C", |
| "direction_roles": { |
| "C": "diagnostic", |
| "B": "proxy", |
| "D": "proxy" |
| }, |
| "why": "Tests whether synchronized modalities identify the same 4D moment, a prerequisite for reconstruction and world modeling.", |
| "current_limit": "Retrieval shows an alignment signal, not geometric reconstruction.", |
| "display_name": "Cross-Modal Retrieval", |
| "artifact_id": "cross_modal_retrieval", |
| "metric": { |
| "key": "mrr", |
| "name": "MRR", |
| "direction": "higher", |
| "minimal": 0.26925966892956127, |
| "neural_mlp": 0.1299971898648288, |
| "better_baseline": "minimal" |
| } |
| }, |
| "modality_reconstruction": { |
| "name": "Modality reconstruction", |
| "family": "forecast", |
| "input": "motion/IMU/camera", |
| "output": "depth/video feature vector", |
| "primary_direction": "B", |
| "direction_roles": { |
| "B": "proxy", |
| "D": "proxy" |
| }, |
| "why": "Predicts visual/depth state from non-target sensors as a weak reconstruction/world-model objective.", |
| "current_limit": "Feature-vector reconstruction is not pixel, depth-map, mesh, NeRF, or Gaussian reconstruction.", |
| "display_name": "Cross-Modal Reconstruction", |
| "artifact_id": "modality_reconstruction", |
| "metric": { |
| "key": "r2", |
| "name": "R2", |
| "direction": "higher", |
| "minimal": -0.015271898913936655, |
| "neural_mlp": -0.010171410134180991, |
| "better_baseline": "neural_mlp" |
| } |
| }, |
| "temporal_order": { |
| "name": "Temporal order verification", |
| "family": "diagnostic", |
| "input": "two adjacent windows", |
| "output": "correct vs reversed order", |
| "primary_direction": "C", |
| "direction_roles": { |
| "C": "diagnostic", |
| "D": "diagnostic" |
| }, |
| "why": "Checks whether features encode local time direction and task progression.", |
| "current_limit": "Only local adjacent ordering, not long-horizon causal modeling.", |
| "display_name": "Temporal Order Verification", |
| "artifact_id": "temporal_order", |
| "metric": { |
| "key": "f1", |
| "name": "F1", |
| "direction": "higher", |
| "minimal": 0.5399515738498789, |
| "neural_mlp": 0.8520179372197308, |
| "better_baseline": "neural_mlp" |
| } |
| }, |
| "misalignment_detection": { |
| "name": "Cross-modal misalignment detection", |
| "family": "diagnostic", |
| "input": "motion plus visual/depth pair", |
| "output": "aligned vs shifted", |
| "primary_direction": "C", |
| "direction_roles": { |
| "C": "diagnostic", |
| "B": "diagnostic", |
| "D": "diagnostic" |
| }, |
| "why": "Detects temporal desynchronization, a key data-quality gate for multimodal reconstruction and world models.", |
| "current_limit": "Synthetic shifts diagnose alignment but do not solve calibration or mapping.", |
| "display_name": "Multimodal Synchronization Detection", |
| "artifact_id": "misalignment_detection", |
| "metric": { |
| "key": "f1", |
| "name": "F1", |
| "direction": "higher", |
| "minimal": 0.5051698670605613, |
| "neural_mlp": 0.7152682255845944, |
| "better_baseline": "neural_mlp" |
| } |
| }, |
| "long_horizon_next_action": { |
| "name": "Long-horizon next-action forecasting", |
| "family": "classification", |
| "input": "current and historical windows", |
| "output": "future action label", |
| "primary_direction": "C", |
| "direction_roles": { |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "why": "Extends short-horizon intention prediction into longer activity futures, a key egocentric and world-model signal.", |
| "current_limit": "Evaluated from sample-supported future labels, not full open-world action generation.", |
| "display_name": "Long-Horizon Next-Action Forecasting", |
| "artifact_id": "long_horizon_next_action", |
| "metric": { |
| "key": "macro_f1", |
| "name": "macro-F1", |
| "direction": "higher", |
| "minimal": 0.07499999999999998, |
| "neural_mlp": 0.06545454545454546, |
| "better_baseline": "minimal" |
| } |
| }, |
| "next_subtask_forecast": { |
| "name": "Long-horizon next-subtask forecasting", |
| "family": "classification", |
| "input": "current and historical windows", |
| "output": "future procedure-step label", |
| "primary_direction": "C", |
| "direction_roles": { |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "why": "Measures whether the model can anticipate the next procedural phase rather than only the current frame state.", |
| "current_limit": "Subtask labels are constrained to the available annotation vocabulary.", |
| "display_name": "Long-Horizon Next-Subtask Forecasting", |
| "artifact_id": "next_subtask_forecast", |
| "metric": { |
| "key": "macro_f1", |
| "name": "macro-F1", |
| "direction": "higher", |
| "minimal": 0.04545454545454545, |
| "neural_mlp": 0.050724637681159424, |
| "better_baseline": "neural_mlp" |
| } |
| }, |
| "interaction_text_prediction": { |
| "name": "Interaction text prediction", |
| "family": "classification", |
| "input": "window features without target text leakage", |
| "output": "natural-language interaction class", |
| "primary_direction": "C", |
| "direction_roles": { |
| "C": "direct", |
| "A": "proxy" |
| }, |
| "why": "Connects egocentric observations to the natural-language interaction semantics carried by the annotation.", |
| "current_limit": "Public derived features retain hashed text targets; raw full text requires the official annotation source.", |
| "display_name": "Interaction Text Prediction", |
| "artifact_id": "interaction_text_prediction", |
| "metric": { |
| "key": "macro_f1", |
| "name": "macro-F1", |
| "direction": "higher", |
| "minimal": 0.04444444444444444, |
| "neural_mlp": 0.0380952380952381, |
| "better_baseline": "minimal" |
| } |
| }, |
| "action_object_relation": { |
| "name": "Action-object relation prediction", |
| "family": "classification", |
| "input": "window features with target-side relation leakage excluded", |
| "output": "action-object relation class", |
| "primary_direction": "C", |
| "direction_roles": { |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "why": "Tests whether action recognition and object state are connected as a relational interaction representation.", |
| "current_limit": "Relation labels are derived from the public-sample annotation scope.", |
| "display_name": "Action-Object Relation Prediction", |
| "artifact_id": "action_object_relation", |
| "metric": { |
| "key": "macro_f1", |
| "name": "macro-F1", |
| "direction": "higher", |
| "minimal": 0.0, |
| "neural_mlp": 0.0, |
| "better_baseline": "tie" |
| } |
| }, |
| "object_set_forecast": { |
| "name": "Future object-set forecasting", |
| "family": "multi-label", |
| "input": "current and historical windows", |
| "output": "future object set", |
| "primary_direction": "D", |
| "direction_roles": { |
| "D": "direct", |
| "C": "proxy" |
| }, |
| "why": "Asks whether the current scene state supports predicting which objects will matter later.", |
| "current_limit": "This is a set-level proxy, not a persistent 3D scene graph.", |
| "display_name": "Future Object-Set Forecasting", |
| "artifact_id": "object_set_forecast", |
| "metric": { |
| "key": "micro_f1", |
| "name": "micro-F1", |
| "direction": "higher", |
| "minimal": 0.16939890710382516, |
| "neural_mlp": 0.19718309859154928, |
| "better_baseline": "neural_mlp" |
| } |
| }, |
| "imu_to_hand_pose": { |
| "name": "IMU-to-hand pose reconstruction", |
| "family": "regression", |
| "input": "IMU and motion context", |
| "output": "hand pose target", |
| "primary_direction": "A", |
| "direction_roles": { |
| "A": "direct", |
| "B": "proxy" |
| }, |
| "why": "Measures human-motion reconstruction from wearable and motion cues.", |
| "current_limit": "Pose reconstruction is window-level and does not yet fit a full parametric hand/body model.", |
| "display_name": "IMU-to-Hand Pose Reconstruction", |
| "artifact_id": "imu_to_hand_pose", |
| "metric": { |
| "key": "mae", |
| "name": "MAE", |
| "direction": "lower", |
| "minimal": 0.042049407958984375, |
| "neural_mlp": 0.042562149465084076, |
| "better_baseline": "minimal" |
| } |
| }, |
| "camera_view_sync_retrieval": { |
| "name": "Camera-view synchronization retrieval", |
| "family": "retrieval", |
| "input": "one camera-view/window query", |
| "output": "matching synchronized view", |
| "primary_direction": "B", |
| "direction_roles": { |
| "B": "direct", |
| "D": "proxy" |
| }, |
| "why": "Tests whether synchronized multi-view structure is recoverable across camera streams.", |
| "current_limit": "Retrieval checks view consistency but does not reconstruct geometry by itself.", |
| "display_name": "Camera-View Synchronization Retrieval", |
| "artifact_id": "camera_view_sync_retrieval", |
| "metric": { |
| "key": "mrr", |
| "name": "MRR", |
| "direction": "higher", |
| "minimal": 0.4943004846572876, |
| "neural_mlp": 0.24086658656597137, |
| "better_baseline": "minimal" |
| } |
| }, |
| "time_to_transition": { |
| "name": "Time-to-next-transition regression", |
| "family": "regression", |
| "input": "current temporal window state", |
| "output": "frames/time until the next transition", |
| "primary_direction": "C", |
| "direction_roles": { |
| "C": "diagnostic", |
| "D": "diagnostic" |
| }, |
| "why": "Measures temporal boundary awareness as a continuous timing target.", |
| "current_limit": "Regression is local to the annotated public sample timeline.", |
| "display_name": "Time-to-Next-Transition Regression", |
| "artifact_id": "time_to_transition", |
| "metric": { |
| "key": "mae", |
| "name": "MAE frames", |
| "direction": "lower", |
| "minimal": 10.53735637664795, |
| "neural_mlp": 10.55449390411377, |
| "better_baseline": "minimal" |
| } |
| } |
| } |
| } |
|
|
