Add step60000 config and eval metadata

README.md

@@ -0,0 +1,17 @@
+# VITRA GigaHands all-cam0 keypoints_mano step60000
+
+This repository contains the finetuned VITRA checkpoint at training step 60000 for GigaHands all-cam0 training with `action_type=keypoints` and `keypoints_source=mano`.
+
+## Files
+- `checkpoints/epoch=0-step=60000.ckpt/weights.pt`: model weights
+- `checkpoints/epoch=0-step=60000.ckpt/meta.json`: checkpoint metadata
+- `config/human_pretrain_gigahands_real_all_cam0_keypoints_mano_vitra3b_linked.json`: training config
+- `eval/metrics_comparison.json`: base vs step60000 evaluation on 50 test clips
+
+## Eval summary
+- action_mse: 16.103662 -> 0.670375
+- left_action_mse: 3.385067 -> 0.750329
+- right_action_mse: 45.174740 -> 0.487623
+- dual_hand_action_mse: 16.103662 -> 0.670375
+
+Optimizer state was not uploaded.

checkpoints/epoch=0-step=60000.ckpt/meta.json

@@ -0,0 +1 @@
+{"epoch": 0, "global_step": 60000}

config/human_pretrain_gigahands_real_all_cam0_keypoints_mano_vitra3b_linked.json

@@ -0,0 +1,73 @@
+{
+    "vla_name": "VITRA_Paligemma",
+    "task_name": "gigahands_real_all_cam0_keypoints_mano_vitra3b_linked_stage1",
+    "model": "vitra_paligemma2",
+    "fwd_pred_next_n": 16,
+    "seed": 42,
+    "batch_size": 1,
+    "output_root": "runs/gigahands_real_all_cam0_keypoints_mano_vitra3b_linked_train/checkpoints",
+    "log_root": "runs/gigahands_real_all_cam0_keypoints_mano_vitra3b_linked_train/logs",
+    "cache_root": "runs/gigahands_real_all_cam0_keypoints_mano_vitra3b_linked_train/cache",
+    "model_load_path": null,
+    "pretrain_path": "./checkpoints/vitra-vla-3b.pt",
+    "resume": false,
+    "wandb_project": "vitra-gigahands-real-keypoints",
+    "wandb_entity": "",
+    "save_steps": 20000,
+    "epoch_save_interval": 100000,
+    "total_batch_size": 2,
+    "use_bf16": true,
+    "use_fov": true,
+    "untied_cognition_token": true,
+    "use_state": "DiT",
+    "loss_type": "human",
+    "train_setup": {
+        "freeze_option": "freeze_vision_encoder"
+    },
+    "state_encoder": {
+        "state_dim": 212
+    },
+    "action_model": {
+        "model_type": "DiT-B",
+        "token_size": 2304,
+        "action_dim": 192,
+        "hidden_size": 1024
+    },
+    "vlm": {
+        "type": "PaliGemmaForConditionalGeneration",
+        "name": "paligemma",
+        "pretrained_model_name_or_path": "google/paligemma2-3b-mix-224"
+    },
+    "trainer": {
+        "sharding_strategy": "shard-grad-op",
+        "strategy": "fsdp_paligemma_with_checkpointing",
+        "lr_scheduler_type": "backbone-freeze-warmup",
+        "gradient_clip_val": 1.0,
+        "learning_rate": 1e-05,
+        "weight_decay": 0.1,
+        "max_epochs": 100000,
+        "max_steps": 65000,
+        "reduce_in_full_precision": true,
+        "enable_mixed_precision_training": false,
+        "enable_gradient_checkpointing": true,
+        "action_model_learning_rate": 0.0001,
+        "llm_freeze_step": 5000,
+        "warmup_ratio": null
+    },
+    "train_dataset": {
+        "data_root_dir": "datasets/vitra_gigahands_real_all_cam0_keypoints_mano_linked",
+        "augmentation": false,
+        "set_none_ratio": 0.0,
+        "data_mix": "gigahands_real_train",
+        "num_workers": 2,
+        "prefetch_factor": null,
+        "flip_augmentation": 1.0,
+        "action_type": "keypoints",
+        "use_rel": false,
+        "clip_len": null,
+        "normalization": true,
+        "state_mask_prob": 0.0,
+        "statistics_dataset_name": "gigahands_real_train"
+    },
+    "repeated_diffusion_steps": 8
+}

eval/metrics_comparison.json

@@ -0,0 +1,38 @@
+{
+  "before_label": "base",
+  "after_label": "step60000",
+  "before_checkpoint": "/home/chonghej/scratch/chonghej/VLA-HAND/checkpoints/vitra-vla-3b.pt",
+  "after_checkpoint": "/home/chonghej/scratch/chonghej/VLA-HAND/runs/gigahands_real_all_cam0_keypoints_mano_vitra3b_linked_train/checkpoints/gigahands_real_all_cam0_keypoints_mano_vitra3b_linked_stage1_TB2_B2_bf16True/checkpoints/epoch=0-step=60000.ckpt",
+  "before": {
+    "action_mse": 16.103662490844727,
+    "left_action_mse": 3.3850672245025635,
+    "right_action_mse": 45.174739837646484,
+    "dual_hand_action_mse": 16.103662490844727,
+    "valid_frame_count": 800,
+    "bimanual_frame_count": 800
+  },
+  "after": {
+    "action_mse": 0.6703752875328064,
+    "left_action_mse": 0.750329315662384,
+    "right_action_mse": 0.4876228868961334,
+    "dual_hand_action_mse": 0.6703752875328064,
+    "valid_frame_count": 800,
+    "bimanual_frame_count": 800
+  },
+  "delta_after_minus_before": {
+    "action_mse": -15.43328720331192,
+    "left_action_mse": -2.6347379088401794,
+    "right_action_mse": -44.68711695075035,
+    "dual_hand_action_mse": -15.43328720331192,
+    "valid_frame_count": 0.0,
+    "bimanual_frame_count": 0.0
+  },
+  "relative_improvement": {
+    "action_mse": 0.9583712532528592,
+    "left_action_mse": 0.7783413841145665,
+    "right_action_mse": 0.9892058506889336,
+    "dual_hand_action_mse": 0.9583712532528592,
+    "valid_frame_count": 0.0,
+    "bimanual_frame_count": 0.0
+  }
+}