Upload folder using huggingface_hub

README.md

@@ -0,0 +1,78 @@
+---
+license: apache-2.0
+base_model: allenai/MolmoAct2-LIBERO-LeRobot
+tags:
+- lerobot
+- molmoact2
+- robotics
+- vla
+- token-pruning
+- grid-sampler
+- libero
+pipeline_tag: robotics
+---
+
+# MolmoAct2-LIBERO + Grid Sampler (random-init sampler)
+
+[allenai/MolmoAct2-LIBERO-LeRobot](https://huggingface.co/allenai/MolmoAct2-LIBERO-LeRobot) with
+[Grid Sampler (GridS, ICML 2026)](https://github.com/Fediory/Grid-Sampler) visual token pruning
+integrated into the vision backbone.
+
+⚠️ **The `ActiveTokenSampler` weights in this checkpoint are randomly initialized.** This artifact
+is the starting point for fine-tuning; do not expect baseline task performance with pruning enabled.
+For the fine-tuned version see
+[xpuenabler/molmoact2-libero_grid_sampler_fine_tuned](https://huggingface.co/xpuenabler/molmoact2-libero_grid_sampler_fine_tuned).
+
+## What changed vs. the base checkpoint
+
+- Each camera image's pooled visual feature grid (14×14 = 196 tokens for single-crop 256×256) is
+  pruned to **K = 16** tokens by an `ActiveTokenSampler`: a global-pooled feature predicts K
+  normalized 2D coordinates, `F.grid_sample` bilinearly reads features at those locations, and a
+  coordinate MLP injects geometry.
+- The processor emits exactly K image placeholder tokens per image, so the LIBERO 2-camera prompt
+  shrinks from 483 to 123 tokens.
+- New config flags: `use_grid_token_sampler=true`, `grid_token_sampler_num_tokens=16`
+  (stored in `config.json` and in the saved processor pipeline).
+
+## Usage
+
+Requires the `feat/grid-sampler-molmoact2` branch of
+[nota-github/xpu-lerobot](https://github.com/nota-github/xpu-lerobot):
+
+```python
+import torch
+from lerobot.configs.policies import PreTrainedConfig
+from lerobot.policies.factory import make_pre_post_processors
+from lerobot.policies.molmoact2.modeling_molmoact2 import MolmoAct2Policy
+
+path = "xpuenabler/molmoact2-libero_grid_sampler_random_init"
+cfg = PreTrainedConfig.from_pretrained(path)
+cfg.pretrained_path = path
+cfg.device = "cuda"
+cfg.inference_action_mode = "continuous"
+
+policy = MolmoAct2Policy.from_pretrained(path, config=cfg)
+preprocessor, postprocessor = make_pre_post_processors(
+    policy_cfg=cfg, pretrained_path=path,
+    preprocessor_overrides={"device_processor": {"device": "cuda"}},
+)
+
+batch = preprocessor({
+    "observation.images.image": torch.rand(3, 256, 256),
+    "observation.images.wrist_image": torch.rand(3, 256, 256),
+    "observation.state": torch.zeros(8),
+    "task": "pick up the black bowl",
+})
+action = postprocessor(policy.select_action(batch))
+```
+
+## Citation
+
+```bibtex
+@inproceedings{feng2026gridsampler,
+  title     = {See What Matters: Differentiable Grid Sample Pruning for Generalizable Vision-Language-Action Model},
+  author    = {Feng, Yixu and Zhao, Zinan and Ma, Yanxiang and Xia, Chenghao and Du, Chengbin and Wang, Yunke and Xu, Chang},
+  booktitle = {Forty-Third International Conference on Machine Learning (ICML)},
+  year      = {2026}
+}
+```

config.json

@@ -0,0 +1,140 @@
+{
+    "type": "molmoact2",
+    "n_obs_steps": 1,
+    "input_features": {
+        "observation.images.image": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                256,
+                256
+            ]
+        },
+        "observation.images.wrist_image": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                256,
+                256
+            ]
+        },
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                8
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                7
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "use_peft": false,
+    "push_to_hub": false,
+    "repo_id": null,
+    "private": null,
+    "tags": null,
+    "license": null,
+    "pretrained_path": "allenai/MolmoAct2-LIBERO-LeRobot",
+    "checkpoint_path": "allenai/MolmoAct2-LIBERO",
+    "checkpoint_revision": null,
+    "checkpoint_force_download": false,
+    "chunk_size": 10,
+    "n_action_steps": 10,
+    "action_mode": "both",
+    "inference_action_mode": null,
+    "discrete_action_tokenizer": "allenai/MolmoAct2-FAST-Tokenizer",
+    "discrete_generation_max_steps": null,
+    "norm_tag": null,
+    "setup_type": "single franka robotic arm in libero",
+    "control_mode": "delta end-effector pose",
+    "image_keys": [
+        "observation.images.image",
+        "observation.images.wrist_image"
+    ],
+    "normalize_language": true,
+    "add_setup_tokens": true,
+    "add_control_tokens": true,
+    "normalize_gripper": false,
+    "num_state_tokens": 256,
+    "max_sequence_length": null,
+    "use_grid_token_sampler": true,
+    "grid_token_sampler_num_tokens": 16,
+    "expected_max_action_dim": 32,
+    "num_flow_timesteps": 8,
+    "flow_matching_cutoff": 1.0,
+    "flow_matching_time_offset": 0.001,
+    "flow_matching_time_scale": 0.999,
+    "flow_matching_beta_alpha": 1.0,
+    "flow_matching_beta_beta": 1.5,
+    "num_inference_steps": null,
+    "mask_action_dim_padding": true,
+    "enable_inference_cuda_graph": true,
+    "per_episode_seed": false,
+    "eval_seed": null,
+    "rtc_config": null,
+    "enable_lora_vlm": false,
+    "lora_rank": 64,
+    "lora_alpha": 16,
+    "lora_dropout": 0.05,
+    "lora_bias": "none",
+    "enable_lora_action_expert": false,
+    "enable_knowledge_insulation": false,
+    "freeze_embedding": true,
+    "train_action_expert_only": false,
+    "gradient_checkpointing": true,
+    "model_dtype": "bfloat16",
+    "softmax_auxiliary_loss": true,
+    "softmax_auxiliary_loss_scale": 0.0001,
+    "discrete_loss_token_weighting": "root_subsegments_root_tokens",
+    "optimizer_lr": 1e-05,
+    "optimizer_vit_lr": 5e-06,
+    "optimizer_connector_lr": 5e-06,
+    "optimizer_action_expert_lr": 5e-05,
+    "optimizer_betas": [
+        0.9,
+        0.95
+    ],
+    "optimizer_eps": 1e-06,
+    "optimizer_weight_decay": 0.0,
+    "optimizer_grad_clip_norm": 1.0,
+    "scheduler_warmup_steps": 200,
+    "scheduler_decay_steps": null,
+    "scheduler_decay_lr": 1e-06,
+    "normalization_mapping": {
+        "VISUAL": "IDENTITY",
+        "STATE": "QUANTILES",
+        "ACTION": "QUANTILES"
+    },
+    "dataset_feature_names": {
+        "action": {
+            "motors": [
+                "x",
+                "y",
+                "z",
+                "roll",
+                "pitch",
+                "yaw",
+                "gripper"
+            ]
+        },
+        "observation.state": {
+            "motors": [
+                "x",
+                "y",
+                "z",
+                "rx",
+                "ry",
+                "rz",
+                "rw",
+                "gripper"
+            ]
+        }
+    }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db4cc097d4872aef9fab14ab7b8ab22b097038b61313b0d5ca7f106751a81fab
+size 10888547536

policy_postprocessor.json

@@ -0,0 +1,36 @@
+{
+  "name": "policy_postprocessor",
+  "steps": [
+    {
+      "registry_name": "molmoact2_clamp_action",
+      "config": {}
+    },
+    {
+      "registry_name": "molmoact2_masked_unnormalizer",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              7
+            ]
+          }
+        },
+        "norm_map": {
+          "VISUAL": "IDENTITY",
+          "STATE": "QUANTILES",
+          "ACTION": "QUANTILES"
+        }
+      },
+      "state_file": "policy_postprocessor_step_1_molmoact2_masked_unnormalizer.safetensors"
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cpu",
+        "float_dtype": null
+      }
+    }
+  ]
+}

policy_postprocessor_step_1_molmoact2_masked_unnormalizer.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a20b5ab0598507d3a736bc78925dde64c14748440a56df1112b049d55c54c4f6
+size 5060

policy_preprocessor.json

@@ -0,0 +1,95 @@
+{
+  "name": "policy_preprocessor",
+  "steps": [
+    {
+      "registry_name": "rename_observations_processor",
+      "config": {
+        "rename_map": {}
+      }
+    },
+    {
+      "registry_name": "to_batch_processor",
+      "config": {}
+    },
+    {
+      "registry_name": "molmoact2_masked_normalizer",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "observation.images.image": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              256,
+              256
+            ]
+          },
+          "observation.images.wrist_image": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              256,
+              256
+            ]
+          },
+          "observation.state": {
+            "type": "STATE",
+            "shape": [
+              8
+            ]
+          },
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              7
+            ]
+          }
+        },
+        "norm_map": {
+          "VISUAL": "IDENTITY",
+          "STATE": "QUANTILES",
+          "ACTION": "QUANTILES"
+        }
+      },
+      "state_file": "policy_preprocessor_step_2_molmoact2_masked_normalizer.safetensors"
+    },
+    {
+      "registry_name": "molmoact2_clamp_normalized",
+      "config": {}
+    },
+    {
+      "registry_name": "molmoact2_pack_inputs",
+      "config": {
+        "checkpoint_path": "allenai/MolmoAct2-LIBERO",
+        "checkpoint_revision": null,
+        "checkpoint_force_download": false,
+        "action_mode": "both",
+        "discrete_action_tokenizer": "allenai/MolmoAct2-FAST-Tokenizer",
+        "image_keys": [
+          "observation.images.image",
+          "observation.images.wrist_image"
+        ],
+        "allow_image_key_fallback": false,
+        "setup_type": "single franka robotic arm in libero",
+        "control_mode": "delta end-effector pose",
+        "normalize_language": true,
+        "add_setup_tokens": true,
+        "add_control_tokens": true,
+        "num_state_tokens": 256,
+        "max_sequence_length": null,
+        "chunk_size": 10,
+        "max_action_dim": 32,
+        "env_action_dim": 7,
+        "use_grid_token_sampler": true,
+        "grid_token_sampler_num_tokens": 16
+      }
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cuda",
+        "float_dtype": null
+      }
+    }
+  ]
+}

policy_preprocessor_step_2_molmoact2_masked_normalizer.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4070d5a500fb431b51183ce9391c2906a6e5a128d926694606e90299b73817c
+size 5068