Initial commit: SmolVLA 50ep bread_skillet (60% sim eval)

Browse files

Files changed (8) hide show

README.md +69 -0
config.json +101 -0
model.safetensors +3 -0
policy_postprocessor.json +32 -0
policy_postprocessor_step_0_unnormalizer_processor.safetensors +3 -0
policy_preprocessor.json +91 -0
policy_preprocessor_step_5_normalizer_processor.safetensors +3 -0
train_config.json +294 -0

README.md ADDED Viewed

	@@ -0,0 +1,69 @@

+---
+license: apache-2.0
+base_model: lerobot/smolvla_robotwin
+tags:
+- robotics
+- vla
+- smolvla
+- lerobot
+- robotwin
+- imitation-learning
+pipeline_tag: robotics
+library_name: lerobot
+---
+# SmolVLA RoboTwin `place_bread_skillet` (50 ep, single instruction)
+SmolVLA policy fine-tuned on 50 demonstration episodes of the **`place_bread_skillet`** task from **RoboTwin 2.0** (`demo_clean` config), starting from the [`lerobot/smolvla_robotwin`](https://huggingface.co/lerobot/smolvla_robotwin) base checkpoint.
+## Task
+Dual-arm pick-and-place: pick up the bread and place it inside the skillet.
+- **Robot**: Agilex dual-arm, end-effector control (16D state, 16D action)
+- **Cameras**: 3 RGB streams — `dual_cam_global`, `cam_wrist_65`, `cam_wrist_75`
+- **Control rate**: 10 Hz
+- **Single fixed instruction**: `"place the bread in the skillet"` (Strategy A, not random per-episode)
+## Training
+| Config | Value |
+|---|---|
+| Base checkpoint | `lerobot/smolvla_robotwin` |
+| Training data | 50 RoboTwin demonstrations (subset of `place_bread_skillet_300ep`), strategy A single instruction |
+| Frames | 8,298 (~165 frames/ep) |
+| Batch size | 32 |
+| Steps | 6000 (~23 epochs) |
+| Optimizer | AdamW, lr=1e-4 |
+| Scheduler | Cosine, warmup=300, decay=6000 |
+| Chunk size | 50 |
+| Final train loss | 0.008 |
+| Walltime | ~2h 15min (A100) |
+## Evaluation
+Evaluated in RoboTwin 2.0 simulator (`demo_clean` config), 10 episodes, `max_steps=400`, `action_chunk_exec=50`.
+| Model | Data | Base | Success |
+|---|---|---|---|
+| SmolVLA (`smolvla_base`) | 300 ep | smolvla_base | **0/10 (0%)** |
+| **SmolVLA (this model)** | **50 ep** | **smolvla_robotwin** | **6/10 (60%)** |
+| X-VLA (`xvla-base`) | 300 ep | xvla-base | 8/10 (80%) |
+Training with the `smolvla_robotwin` base checkpoint enables strong data efficiency: with 6× less data and 3× fewer training steps, this model jumps from 0% to 60% success rate on this task.
+Successful episodes complete in 132–201 environment steps (13–20s); failed episodes time out at 400 steps.
+## Usage
+```python
+from lerobot.policies.smolvla import SmolVLAPolicy
+policy = SmolVLAPolicy.from_pretrained("arrow-hf/smolvla-robotwin-place-bread-skillet-50ep")
+```
+See [LeRobot documentation](https://huggingface.co/docs/lerobot) for inference setup.
+## Citation
+Built on [SmolVLA](https://huggingface.co/lerobot/smolvla_base) and [SmolVLA-RoboTwin](https://huggingface.co/lerobot/smolvla_robotwin), fine-tuned on data collected from [RoboTwin 2.0](https://github.com/TianxingChen/RoboTwin).

config.json ADDED Viewed

	@@ -0,0 +1,101 @@

+{
+    "type": "smolvla",
+    "n_obs_steps": 1,
+    "input_features": {
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                6
+            ]
+        },
+        "observation.images.camera1": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                256,
+                256
+            ]
+        },
+        "observation.images.camera2": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                256,
+                256
+            ]
+        },
+        "observation.images.camera3": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                256,
+                256
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                16
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "use_peft": false,
+    "push_to_hub": false,
+    "repo_id": "pepijn223/smolvla_robotwin",
+    "private": null,
+    "tags": null,
+    "license": null,
+    "pretrained_path": "lerobot/smolvla_robotwin",
+    "chunk_size": 50,
+    "n_action_steps": 50,
+    "normalization_mapping": {
+        "VISUAL": "IDENTITY",
+        "STATE": "MEAN_STD",
+        "ACTION": "MEAN_STD"
+    },
+    "max_state_dim": 32,
+    "max_action_dim": 32,
+    "resize_imgs_with_padding": [
+        512,
+        512
+    ],
+    "empty_cameras": 0,
+    "adapt_to_pi_aloha": false,
+    "use_delta_joint_actions_aloha": false,
+    "tokenizer_max_length": 48,
+    "num_steps": 10,
+    "use_cache": true,
+    "freeze_vision_encoder": false,
+    "train_expert_only": false,
+    "train_state_proj": true,
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.9,
+        0.95
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-10,
+    "optimizer_grad_clip_norm": 10.0,
+    "scheduler_warmup_steps": 300,
+    "scheduler_decay_steps": 6000,
+    "scheduler_decay_lr": 2.5e-06,
+    "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct",
+    "load_vlm_weights": true,
+    "add_image_special_tokens": false,
+    "attention_mode": "cross_attn",
+    "prefix_length": 0,
+    "pad_language_to": "max_length",
+    "num_expert_layers": 0,
+    "num_vlm_layers": 16,
+    "self_attn_every_n_layers": 2,
+    "expert_width_multiplier": 0.75,
+    "min_period": 0.004,
+    "max_period": 4.0,
+    "rtc_config": null,
+    "compile_model": false,
+    "compile_mode": "max-autotune"
+}

model.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:0064b25495c614795d69f73ad37ee0e5164d89e30cd53246d9f3c33a9d0a84f9
+size 906712520

policy_postprocessor.json ADDED Viewed

	@@ -0,0 +1,32 @@

+{
+  "name": "policy_postprocessor",
+  "steps": [
+    {
+      "registry_name": "unnormalizer_processor",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              16
+            ]
+          }
+        },
+        "norm_map": {
+          "VISUAL": "IDENTITY",
+          "STATE": "MEAN_STD",
+          "ACTION": "MEAN_STD"
+        }
+      },
+      "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors"
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cpu",
+        "float_dtype": null
+      }
+    }
+  ]
+}

policy_postprocessor_step_0_unnormalizer_processor.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:e23a7ebd24a01822c72ae131026ab1e2f062edf10b063fad01a694b97e2572bc
+size 9592

policy_preprocessor.json ADDED Viewed

	@@ -0,0 +1,91 @@

+{
+  "name": "policy_preprocessor",
+  "steps": [
+    {
+      "registry_name": "rename_observations_processor",
+      "config": {
+        "rename_map": {
+          "observation.images.dual_cam_global": "observation.images.camera1",
+          "observation.images.cam_wrist_65": "observation.images.camera2",
+          "observation.images.cam_wrist_75": "observation.images.camera3"
+        }
+      }
+    },
+    {
+      "registry_name": "to_batch_processor",
+      "config": {}
+    },
+    {
+      "registry_name": "smolvla_new_line_processor",
+      "config": {}
+    },
+    {
+      "registry_name": "tokenizer_processor",
+      "config": {
+        "max_length": 48,
+        "task_key": "task",
+        "padding_side": "right",
+        "padding": "max_length",
+        "truncation": true,
+        "tokenizer_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct"
+      }
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cuda",
+        "float_dtype": null
+      }
+    },
+    {
+      "registry_name": "normalizer_processor",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "observation.state": {
+            "type": "STATE",
+            "shape": [
+              6
+            ]
+          },
+          "observation.images.camera1": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              256,
+              256
+            ]
+          },
+          "observation.images.camera2": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              256,
+              256
+            ]
+          },
+          "observation.images.camera3": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              256,
+              256
+            ]
+          },
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              16
+            ]
+          }
+        },
+        "norm_map": {
+          "VISUAL": "IDENTITY",
+          "STATE": "MEAN_STD",
+          "ACTION": "MEAN_STD"
+        }
+      },
+      "state_file": "policy_preprocessor_step_5_normalizer_processor.safetensors"
+    }
+  ]
+}

policy_preprocessor_step_5_normalizer_processor.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:e23a7ebd24a01822c72ae131026ab1e2f062edf10b063fad01a694b97e2572bc
+size 9592

train_config.json ADDED Viewed

	@@ -0,0 +1,294 @@

+{
+    "dataset": {
+        "repo_id": "local/robotwin_place_bread_skillet_300ep_A",
+        "root": "data/lerobot_robotwin_place_bread_skillet_300ep_A",
+        "episodes": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49
+        ],
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "affine": {
+                    "weight": 1.0,
+                    "type": "RandomAffine",
+                    "kwargs": {
+                        "degrees": [
+                            -5.0,
+                            5.0
+                        ],
+                        "translate": [
+                            0.05,
+                            0.05
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec",
+        "streaming": false
+    },
+    "env": null,
+    "policy": {
+        "type": "smolvla",
+        "n_obs_steps": 1,
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    6
+                ]
+            },
+            "observation.images.camera1": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    256,
+                    256
+                ]
+            },
+            "observation.images.camera2": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    256,
+                    256
+                ]
+            },
+            "observation.images.camera3": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    256,
+                    256
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    16
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "use_peft": false,
+        "push_to_hub": false,
+        "repo_id": "pepijn223/smolvla_robotwin",
+        "private": null,
+        "tags": null,
+        "license": null,
+        "pretrained_path": "lerobot/smolvla_robotwin",
+        "chunk_size": 50,
+        "n_action_steps": 50,
+        "normalization_mapping": {
+            "VISUAL": "IDENTITY",
+            "STATE": "MEAN_STD",
+            "ACTION": "MEAN_STD"
+        },
+        "max_state_dim": 32,
+        "max_action_dim": 32,
+        "resize_imgs_with_padding": [
+            512,
+            512
+        ],
+        "empty_cameras": 0,
+        "adapt_to_pi_aloha": false,
+        "use_delta_joint_actions_aloha": false,
+        "tokenizer_max_length": 48,
+        "num_steps": 10,
+        "use_cache": true,
+        "freeze_vision_encoder": false,
+        "train_expert_only": false,
+        "train_state_proj": true,
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.9,
+            0.95
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-10,
+        "optimizer_grad_clip_norm": 10.0,
+        "scheduler_warmup_steps": 300,
+        "scheduler_decay_steps": 6000,
+        "scheduler_decay_lr": 2.5e-06,
+        "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct",
+        "load_vlm_weights": true,
+        "add_image_special_tokens": false,
+        "attention_mode": "cross_attn",
+        "prefix_length": 0,
+        "pad_language_to": "max_length",
+        "num_expert_layers": 0,
+        "num_vlm_layers": 16,
+        "self_attn_every_n_layers": 2,
+        "expert_width_multiplier": 0.75,
+        "min_period": 0.004,
+        "max_period": 4.0,
+        "rtc_config": null,
+        "compile_model": false,
+        "compile_mode": "max-autotune"
+    },
+    "output_dir": "training/smolvla/outputs/robotwin_place_bread_skillet_50ep_v1_robotwin_base",
+    "job_name": "smolvla",
+    "resume": false,
+    "seed": 42,
+    "cudnn_deterministic": false,
+    "num_workers": 4,
+    "batch_size": 32,
+    "steps": 6000,
+    "eval_freq": 0,
+    "log_freq": 50,
+    "tolerance_s": 0.0001,
+    "save_checkpoint": true,
+    "save_freq": 1000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adamw",
+        "lr": 0.0001,
+        "weight_decay": 1e-10,
+        "grad_clip_norm": 10.0,
+        "betas": [
+            0.9,
+            0.95
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "cosine_decay_with_warmup",
+        "num_warmup_steps": 300,
+        "num_decay_steps": 6000,
+        "peak_lr": 0.0001,
+        "decay_lr": 2.5e-06
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 42,
+        "use_async_envs": true
+    },
+    "wandb": {
+        "enable": false,
+        "disable_artifact": false,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": null,
+        "mode": null,
+        "add_tags": true
+    },
+    "peft": null,
+    "use_rabc": false,
+    "rabc_progress_path": null,
+    "rabc_kappa": 0.01,
+    "rabc_epsilon": 1e-06,
+    "rabc_head_mode": "sparse",
+    "rename_map": {
+        "observation.images.dual_cam_global": "observation.images.camera1",
+        "observation.images.cam_wrist_65": "observation.images.camera2",
+        "observation.images.cam_wrist_75": "observation.images.camera3"
+    },
+    "checkpoint_path": null
+}