eval/metaworld_mt50.json

{
  "schema_version": "0.1",
  "source": {
    "paper": "https://arxiv.org/abs/1910.10897",
    "arxiv": "https://arxiv.org/abs/1910.10897",
    "model_variant": "smolvla",
    "evaluated_by": "OpenRAL:openral benchmark run",
    "reproduced_locally": true,
    "reproduction_planned": null,
    "reproduction_cli": "openral benchmark run --suite metaworld_mt50 --rskill rskills/smolvla-metaworld",
    "table": null,
    "status": "reproduced"
  },
  "benchmark": {
    "name": "MetaWorld MT50",
    "dataset": null,
    "protocol": "1 episodes per task, success_key=success, max_steps=200",
    "robot": "sawyer",
    "simulator": "MetaWorld v3 (MuJoCo via lerobot)"
  },
  "eval_config": {
    "n_episodes": 1,
    "seeds": [
      0
    ],
    "success_key": "success",
    "max_steps": 200,
    "vla_id": "smolvla",
    "weights_uri": "rskills/smolvla-metaworld"
  },
  "results": {
    "metaworld/assembly-v3_success_rate": 1.0,
    "metaworld/basketball-v3_success_rate": 0.0,
    "metaworld/bin-picking-v3_success_rate": 0.0,
    "metaworld/box-close-v3_success_rate": 0.0,
    "metaworld/button-press-topdown-v3_success_rate": 1.0,
    "metaworld/button-press-topdown-wall-v3_success_rate": 0.0,
    "metaworld/button-press-v3_success_rate": 1.0,
    "metaworld/button-press-wall-v3_success_rate": 0.0,
    "metaworld/coffee-button-v3_success_rate": 1.0,
    "metaworld/coffee-pull-v3_success_rate": 0.0,
    "metaworld/coffee-push-v3_success_rate": 0.0,
    "metaworld/dial-turn-v3_success_rate": 0.0,
    "metaworld/disassemble-v3_success_rate": 0.0,
    "metaworld/door-close-v3_success_rate": 1.0,
    "metaworld/door-lock-v3_success_rate": 1.0,
    "metaworld/door-open-v3_success_rate": 0.0,
    "metaworld/door-unlock-v3_success_rate": 0.0,
    "metaworld/drawer-close-v3_success_rate": 1.0,
    "metaworld/drawer-open-v3_success_rate": 0.0,
    "metaworld/faucet-close-v3_success_rate": 1.0,
    "metaworld/faucet-open-v3_success_rate": 0.0,
    "metaworld/hammer-v3_success_rate": 0.0,
    "metaworld/hand-insert-v3_success_rate": 0.0,
    "metaworld/handle-press-side-v3_success_rate": 1.0,
    "metaworld/handle-press-v3_success_rate": 1.0,
    "metaworld/handle-pull-side-v3_success_rate": 0.0,
    "metaworld/handle-pull-v3_success_rate": 0.0,
    "metaworld/lever-pull-v3_success_rate": 0.0,
    "metaworld/peg-insert-side-v3_success_rate": 0.0,
    "metaworld/peg-unplug-side-v3_success_rate": 0.0,
    "metaworld/pick-out-of-hole-v3_success_rate": 0.0,
    "metaworld/pick-place-v3_success_rate": 1.0,
    "metaworld/pick-place-wall-v3_success_rate": 1.0,
    "metaworld/plate-slide-back-side-v3_success_rate": 1.0,
    "metaworld/plate-slide-back-v3_success_rate": 0.0,
    "metaworld/plate-slide-side-v3_success_rate": 1.0,
    "metaworld/plate-slide-v3_success_rate": 0.0,
    "metaworld/push-back-v3_success_rate": 0.0,
    "metaworld/push-v3_success_rate": 1.0,
    "metaworld/push-wall-v3_success_rate": 0.0,
    "metaworld/reach-v3_success_rate": 0.0,
    "metaworld/reach-wall-v3_success_rate": 0.0,
    "metaworld/shelf-place-v3_success_rate": 0.0,
    "metaworld/soccer-v3_success_rate": 0.0,
    "metaworld/stick-pull-v3_success_rate": 0.0,
    "metaworld/stick-push-v3_success_rate": 0.0,
    "metaworld/sweep-into-v3_success_rate": 0.0,
    "metaworld/sweep-v3_success_rate": 0.0,
    "metaworld/window-close-v3_success_rate": 0.0,
    "metaworld/window-open-v3_success_rate": 0.0,
    "avg_success_rate": 0.3,
    "n_tasks": 50,
    "n_episodes_per_task": 1,
    "n_episodes_total": 50,
    "mean_step_latency_ms_avg": 9.514092124575981
  },
  "baselines": {},
  "trace_id": null
}