Hugging Face's logo

LeoJiangOR
/
opentouch-vitra-step140000-adapted

PyTorch
vitra
opentouch
tactile
action-prediction
hand-pose
Model card Files
xet
 Community
opentouch-vitra-step140000-adapted
File size: 2,705 Bytes
136d01d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
{
  "vla_name": "VITRA_Paligemma",
  "task_name": "opentouch_keypoint_from_step140000_large",
  "model": "vitra_paligemma2",
  "fwd_pred_next_n": 16,
  "seed": 42,
  "batch_size": 1,
  "output_root": "/home/chonghej/scratch/chonghej/VLA-HAND/runs/opentouch_step140000_adapt_touch_editor/adapt_checkpoints",
  "log_root": "/home/chonghej/scratch/chonghej/VLA-HAND/runs/opentouch_step140000_adapt_touch_editor/adapt_logs",
  "cache_root": "runs/opentouch_keypoint_from_step140000_adapt/cache/vitra_paligemma2",
  "model_load_path": null,
  "pretrain_path": "runs/gigahands_real_all_cam0_keypoints_mano_vitra3b_linked_train/checkpoints/gigahands_real_all_cam0_keypoints_mano_vitra3b_linked_stage1_TB2_B2_bf16True/checkpoints/epoch=0-step=140000.ckpt",
  "resume": false,
  "wandb_project": "vitra-opentouch-keypoint-adapt",
  "wandb_entity": "",
  "save_steps": 5000,
  "epoch_save_interval": 100000,
  "total_batch_size": 4,
  "use_bf16": true,
  "use_fov": true,
  "untied_cognition_token": true,
  "use_state": "DiT",
  "loss_type": "human",
  "train_setup": {
    "freeze_option": "freeze_vision_encoder"
  },
  "state_encoder": {
    "state_dim": 212
  },
  "action_model": {
    "model_type": "DiT-B",
    "token_size": 2304,
    "action_dim": 192,
    "hidden_size": 1024
  },
  "vlm": {
    "type": "PaliGemmaForConditionalGeneration",
    "name": "paligemma",
    "pretrained_model_name_or_path": "google/paligemma2-3b-mix-224"
  },
  "trainer": {
    "sharding_strategy": "shard-grad-op",
    "strategy": "fsdp_paligemma_with_checkpointing",
    "lr_scheduler_type": "backbone-freeze-warmup",
    "gradient_clip_val": 1.0,
    "learning_rate": 1e-05,
    "weight_decay": 0.1,
    "max_epochs": 100000,
    "max_steps": 10000,
    "reduce_in_full_precision": true,
    "enable_mixed_precision_training": false,
    "enable_gradient_checkpointing": true,
    "action_model_learning_rate": 0.0001,
    "llm_freeze_step": 5000,
    "warmup_ratio": null
  },
  "train_dataset": {
    "data_root_dir": "datasets/vitra_opentouch_keypoint_full_text_aligned",
    "augmentation": false,
    "set_none_ratio": 0.0,
    "data_mix": "opentouch_keypoint_train",
    "num_workers": 2,
    "prefetch_factor": null,
    "flip_augmentation": 1.0,
    "action_type": "keypoints",
    "use_rel": false,
    "clip_len": null,
    "normalization": true,
    "state_mask_prob": 0.0,
    "statistics_dataset_name": "opentouch_keypoint_train"
  },
  "repeated_diffusion_steps": 8,
  "config": "/home/chonghej/scratch/chonghej/VLA-HAND/vitra/configs/human_pretrain_opentouch_keypoint_from_step140000.json",
  "data_mix": null,
  "debug": false,
  "num_workers": 2,
  "prefetch_factor": null,
  "no_save_checkpoint": false
}