GR00T-N1.6 fine-tune on SO-101 vials_rack_left Cosmos augment v2 (10k steps)

config.json

@@ -0,0 +1,70 @@
+{
+  "action_horizon": 50,
+  "add_pos_embed": true,
+  "apply_sincos_state_encoding": true,
+  "architectures": [
+    "Gr00tN1d6"
+  ],
+  "attn_dropout": 0.2,
+  "attn_implementation": null,
+  "backbone_embedding_dim": 2048,
+  "backbone_model_type": "eagle",
+  "backbone_trainable_params_fp32": true,
+  "collator_overwrite_image_inputs": false,
+  "color_jitter_params": {
+    "brightness": 0.1,
+    "contrast": 0.1,
+    "hue": 0.1,
+    "saturation": 0.1
+  },
+  "crop_fraction": 0.95,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "eagle_collator": true,
+  "formalize_language": true,
+  "gemma_collator": false,
+  "hidden_size": 1024,
+  "image_crop_size": null,
+  "image_target_size": null,
+  "input_embedding_dim": 1536,
+  "load_bf16": true,
+  "max_action_dim": 128,
+  "max_num_embodiments": 32,
+  "max_seq_len": 1024,
+  "max_state_dim": 128,
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Eagle-Block2A-2B-v2",
+  "model_type": "Gr00tN1d6",
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_inference_timesteps": 4,
+  "num_timestep_buckets": 1000,
+  "random_rotation_angle": null,
+  "reproject_vision": false,
+  "select_layer": 16,
+  "shortest_image_edge": 256,
+  "state_dropout_prob": 0.0,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "tune_diffusion_model": true,
+  "tune_llm": false,
+  "tune_projector": true,
+  "tune_top_llm_layers": 4,
+  "tune_visual": false,
+  "tune_vlln": true,
+  "use_albumentations_transforms": true,
+  "use_alternate_vl_dit": true,
+  "use_flash_attention": true,
+  "use_relative_action": true,
+  "use_vlln": true
+}

embodiment_id.json

@@ -0,0 +1,10 @@
+{
+  "robocasa_panda_omron": 13,
+  "gr1": 20,
+  "behavior_r1_pro": 24,
+  "unitree_g1": 8,
+  "oxe_google": 0,
+  "oxe_widowx": 1,
+  "libero_panda": 2,
+  "new_embodiment": 10
+}

experiment_cfg/conf.yaml

@@ -0,0 +1,209 @@
+load_config_path: null
+model:
+  model_type: Gr00tN1d6
+  model_dtype: bfloat16
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  backbone_model_type: eagle
+  model_revision: null
+  tune_top_llm_layers: 4
+  backbone_embedding_dim: 2048
+  tune_llm: false
+  tune_visual: false
+  select_layer: 16
+  reproject_vision: false
+  use_flash_attention: true
+  load_bf16: false
+  collator_overwrite_image_inputs: false
+  eagle_collator: true
+  backbone_trainable_params_fp32: true
+  image_crop_size: null
+  image_target_size: null
+  shortest_image_edge: 256
+  crop_fraction: 0.95
+  random_rotation_angle: null
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    saturation: 0.5
+    hue: 0.08
+  use_albumentations_transforms: true
+  formalize_language: true
+  apply_sincos_state_encoding: false
+  use_relative_action: true
+  max_state_dim: 29
+  max_action_dim: 29
+  action_horizon: 16
+  hidden_size: 1024
+  input_embedding_dim: 1536
+  add_pos_embed: true
+  attn_dropout: 0.2
+  use_vlln: true
+  max_seq_len: 1024
+  use_alternate_vl_dit: true
+  attend_text_every_n_blocks: 2
+  diffusion_model_cfg:
+    positional_embeddings: null
+    num_layers: 32
+    num_attention_heads: 32
+    attention_head_dim: 48
+    norm_type: ada_norm
+    dropout: 0.2
+    final_dropout: true
+    output_dim: 1024
+    interleave_self_attention: true
+  num_inference_timesteps: 4
+  noise_beta_alpha: 1.5
+  noise_beta_beta: 1.0
+  noise_s: 0.999
+  num_timestep_buckets: 1000
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  state_dropout_prob: 0.0
+  state_additive_noise_scale: 0.0
+  max_num_embodiments: 32
+data:
+  datasets:
+  - dataset_paths:
+    - examples/SO100/sreetz-nv/so101_teleop_vials_rack_left_augment_02
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    dataset_type: physical_embodiment
+    val_dataset_path: null
+  modality_configs:
+    new_embodiment:
+      video:
+        delta_indices:
+        - 0
+        modality_keys:
+        - front
+        - wrist
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      state:
+        delta_indices:
+        - 0
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      action:
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs:
+        - rep: RELATIVE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+        - rep: ABSOLUTE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+      language:
+        delta_indices:
+        - 0
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+  download_cache: false
+  shard_size: 1024
+  episode_sampling_rate: 0.1
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  mode: single_turn
+  random_chop: 0.0
+  mock_dataset_mode: false
+  shuffle: true
+  seed: 42
+  multiprocessing_context: fork
+  allow_padding: false
+  subsample_ratio: 1.0
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  video_backend: torchcodec
+training:
+  output_dir: /ephemeral/sreetz-nv-so101_teleop_vials_rack_left_augment_02
+  experiment_name: null
+  max_steps: 10000
+  global_batch_size: 32
+  batch_size: null
+  gradient_accumulation_steps: 1
+  learning_rate: 0.0001
+  lr_scheduler_type: cosine
+  weight_decay: 1.0e-05
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  max_grad_norm: 1.0
+  optim: adamw_torch
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  fp16: false
+  bf16: true
+  eval_bf16: true
+  logging_steps: 10
+  save_steps: 5000
+  save_total_limit: 5
+  save_vl_model: false
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  max_concurrent_uploads: 2
+  eval_strategy: 'no'
+  eval_steps: 500
+  eval_set_split_ratio: 0.1
+  eval_batch_size: 2
+  save_best_eval_metric_name: ''
+  save_best_eval_metric_greater_is_better: true
+  deepspeed_stage: 2
+  gradient_checkpointing: false
+  transformers_trust_remote_code: true
+  transformers_local_files_only: false
+  transformers_cache_dir: null
+  transformers_access_token: null
+  use_ddp: false
+  ddp_bucket_cap_mb: 100
+  num_gpus: 1
+  dataloader_num_workers: 4
+  remove_unused_columns: false
+  use_wandb: false
+  wandb_project: finetune-gr00t-n1d6
+  enable_profiling: false
+  max_retries: 3
+  assert_loss_less_than: null
+  add_rl_callback: false
+  enable_open_loop_eval: false
+  open_loop_eval_traj_ids:
+  - 0
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_plot_indices: null
+max_steps: 10000
+save_steps: 5000

experiment_cfg/config.yaml

@@ -0,0 +1,243 @@
+!!python/object:gr00t.configs.base_config.Config
+data: !!python/object:gr00t.configs.data.data_config.DataConfig
+  allow_padding: false
+  datasets:
+  - !!python/object:gr00t.configs.data.data_config.SingleDatasetConfig
+    dataset_paths:
+    - examples/SO100/sreetz-nv/so101_teleop_vials_rack_left_augment_02
+    dataset_type: physical_embodiment
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    val_dataset_path: null
+  download_cache: false
+  episode_sampling_rate: 0.1
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  mock_dataset_mode: false
+  modality_configs:
+    new_embodiment:
+      action: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs:
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: &id001 !!python/object/apply:gr00t.data.types.ActionFormat
+          - default
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - relative
+          state_key: null
+          type: &id002 !!python/object/apply:gr00t.data.types.ActionType
+          - non_eef
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: *id001
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - absolute
+          state_key: null
+          type: *id002
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      language: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+      state: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      video: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - front
+        - wrist
+        sin_cos_embedding_keys: null
+  mode: single_turn
+  multiprocessing_context: fork
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  random_chop: 0.0
+  seed: 42
+  shard_size: 1024
+  shuffle: true
+  subsample_ratio: 1.0
+  video_backend: torchcodec
+load_config_path: null
+model: !!python/object:gr00t.configs.model.gr00t_n1d6.Gr00tN1d6Config
+  _attn_implementation_autoset: false
+  _attn_implementation_internal: null
+  _commit_hash: null
+  _name_or_path: ''
+  add_cross_attention: false
+  architectures: null
+  backbone_model_type: eagle
+  backbone_trainable_params_fp32: true
+  bad_words_ids: null
+  begin_suppress_tokens: null
+  bos_token_id: null
+  chunk_size_feed_forward: 0
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    hue: 0.08
+    saturation: 0.5
+  cross_attention_hidden_size: null
+  decoder_start_token_id: null
+  diffusion_model_cfg:
+    attention_head_dim: 48
+    dropout: 0.2
+    final_dropout: true
+    interleave_self_attention: true
+    norm_type: ada_norm
+    num_attention_heads: 32
+    num_layers: 32
+    output_dim: 1024
+    positional_embeddings: null
+  diversity_penalty: 0.0
+  do_sample: false
+  eagle_collator: true
+  early_stopping: false
+  encoder_no_repeat_ngram_size: 0
+  eos_token_id: null
+  exponential_decay_length_penalty: null
+  finetuning_task: null
+  forced_bos_token_id: null
+  forced_eos_token_id: null
+  id2label:
+    0: LABEL_0
+    1: LABEL_1
+  is_decoder: false
+  is_encoder_decoder: false
+  label2id:
+    LABEL_0: 0
+    LABEL_1: 1
+  length_penalty: 1.0
+  load_bf16: false
+  max_length: 20
+  min_length: 0
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  no_repeat_ngram_size: 0
+  num_beam_groups: 1
+  num_beams: 1
+  num_return_sequences: 1
+  output_attentions: false
+  output_hidden_states: false
+  output_scores: false
+  pad_token_id: null
+  prefix: null
+  problem_type: null
+  pruned_heads: {}
+  random_rotation_angle: null
+  remove_invalid_values: false
+  repetition_penalty: 1.0
+  reproject_vision: false
+  return_dict: true
+  return_dict_in_generate: false
+  sep_token_id: null
+  state_dropout_prob: 0.0
+  suppress_tokens: null
+  task_specific_params: null
+  temperature: 1.0
+  tf_legacy_loss: false
+  tie_encoder_decoder: false
+  tie_word_embeddings: true
+  tokenizer_class: null
+  top_k: 50
+  top_p: 1.0
+  torch_dtype: null
+  torchscript: false
+  transformers_version: null
+  tune_diffusion_model: true
+  tune_llm: false
+  tune_projector: true
+  tune_visual: false
+  typical_p: 1.0
+  use_bfloat16: false
+  use_relative_action: true
+training: !!python/object:gr00t.configs.training.training_config.TrainingConfig
+  add_rl_callback: false
+  assert_loss_less_than: null
+  batch_size: null
+  bf16: true
+  dataloader_num_workers: 4
+  ddp_bucket_cap_mb: 100
+  deepspeed_stage: 2
+  enable_open_loop_eval: false
+  enable_profiling: false
+  eval_batch_size: 2
+  eval_bf16: true
+  eval_set_split_ratio: 0.1
+  eval_steps: 500
+  eval_strategy: 'no'
+  experiment_name: null
+  fp16: false
+  global_batch_size: 32
+  gradient_accumulation_steps: 1
+  gradient_checkpointing: false
+  learning_rate: 0.0001
+  logging_steps: 10
+  lr_scheduler_type: cosine
+  max_concurrent_uploads: 2
+  max_grad_norm: 1.0
+  max_retries: 3
+  max_steps: 10000
+  num_gpus: 1
+  open_loop_eval_plot_indices: null
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_traj_ids:
+  - 0
+  optim: adamw_torch
+  output_dir: /ephemeral/sreetz-nv-so101_teleop_vials_rack_left_augment_02
+  remove_unused_columns: false
+  save_best_eval_metric_greater_is_better: true
+  save_best_eval_metric_name: ''
+  save_steps: 5000
+  save_total_limit: 5
+  save_vl_model: false
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  transformers_access_token: null
+  transformers_cache_dir: null
+  transformers_local_files_only: false
+  transformers_trust_remote_code: true
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  use_ddp: false
+  use_wandb: false
+  wandb_project: finetune-gr00t-n1d6
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  weight_decay: 1.0e-05

experiment_cfg/dataset_statistics.json

@@ -0,0 +1,824 @@
+{
+  "new_embodiment": {
+    "state": {
+      "single_arm": {
+        "min": [
+          -44.03818893432617,
+          -99.9115982055664,
+          -34.79449462890625,
+          -100.0046157836914,
+          -71.68205261230469
+        ],
+        "max": [
+          35.02424621582031,
+          50.342254638671875,
+          100.00312805175781,
+          99.99986267089844,
+          -34.87779235839844
+        ],
+        "mean": [
+          -5.836079315512441,
+          -32.43699472425867,
+          58.237880016541624,
+          -2.058469300161804,
+          -54.88172398597043
+        ],
+        "std": [
+          18.879953820913162,
+          35.72329565568573,
+          28.352601136667346,
+          45.24627570450343,
+          5.240307440093531
+        ],
+        "q01": [
+          -28.04994649996509,
+          -92.6465602218449,
+          13.209247854973407,
+          -71.39476368305034,
+          -59.632286930802415
+        ],
+        "q99": [
+          16.441536596780434,
+          21.650053209546478,
+          89.8375856833381,
+          47.5234561924644,
+          -48.34047974049414
+        ]
+      },
+      "gripper": {
+        "min": [
+          1.2090978622436523
+        ],
+        "max": [
+          58.09996795654297
+        ],
+        "mean": [
+          24.89359516380481
+        ],
+        "std": [
+          9.748755624478095
+        ],
+        "q01": [
+          15.187353651307305
+        ],
+        "q99": [
+          40.16410101748612
+        ]
+      }
+    },
+    "action": {
+      "single_arm": {
+        "min": [
+          -44.09799575805664,
+          -100.0,
+          -35.0520133972168,
+          -99.82714080810547,
+          -72.86458587646484
+        ],
+        "max": [
+          34.966590881347656,
+          49.93710708618164,
+          99.9095458984375,
+          99.73822021484375,
+          -34.84375
+        ],
+        "mean": [
+          -5.958351950210464,
+          -32.97634184841469,
+          57.362650748618684,
+          -2.560561536132551,
+          -54.904470053884694
+        ],
+        "std": [
+          18.917756436877145,
+          35.473149564045826,
+          28.36427162879512,
+          45.34836372765847,
+          5.341554451679705
+        ],
+        "q01": [
+          -28.149341794719206,
+          -92.84341982051184,
+          12.713704037485476,
+          -71.5873207142509,
+          -59.73696803800649
+        ],
+        "q99": [
+          16.358533002892408,
+          21.42156659449393,
+          89.04983528549377,
+          47.510543748005446,
+          -48.1645772225168
+        ]
+      },
+      "gripper": {
+        "min": [
+          0.0
+        ],
+        "max": [
+          58.69038009643555
+        ],
+        "mean": [
+          19.329132683561966
+        ],
+        "std": [
+          15.315618181174665
+        ],
+        "q01": [
+          3.608805088856866
+        ],
+        "q99": [
+          40.45621313447
+        ]
+      }
+    },
+    "relative_action": {
+      "single_arm": {
+        "min": [
+          [
+            -7.403679847717285,
+            -21.343788146972656,
+            -7.281219482421875,
+            -26.58721160888672,
+            -4.022655487060547
+          ],
+          [
+            -15.338791847229004,
+            -30.557064056396484,
+            -12.155563354492188,
+            -36.63103485107422,
+            -5.078456878662109
+          ],
+          [
+            -17.60955810546875,
+            -33.57593536376953,
+            -15.516799926757812,
+            -39.94593811035156,
+            -6.100704193115234
+          ],
+          [
+            -19.56610107421875,
+            -36.97331619262695,
+            -18.439834594726562,
+            -43.18556213378906,
+            -7.064990997314453
+          ],
+          [
+            -21.609573364257812,
+            -40.83075714111328,
+            -21.334449768066406,
+            -46.67118835449219,
+            -7.955162048339844
+          ],
+          [
+            -23.818679809570312,
+            -44.688201904296875,
+            -24.13861083984375,
+            -50.31035614013672,
+            -8.74277114868164
+          ],
+          [
+            -26.139259338378906,
+            -47.874786376953125,
+            -26.761859893798828,
+            -54.02598571777344,
+            -9.537178039550781
+          ],
+          [
+            -28.473419189453125,
+            -50.6420783996582,
+            -29.38510513305664,
+            -57.71142578125,
+            -10.437923431396484
+          ],
+          [
+            -30.723190307617188,
+            -53.241661071777344,
+            -31.639423370361328,
+            -61.252540588378906,
+            -11.297477722167969
+          ],
+          [
+            -32.8468017578125,
+            -55.673526763916016,
+            -34.24517822265625,
+            -64.55354309082031,
+            -11.978679656982422
+          ],
+          [
+            -34.89494323730469,
+            -58.18925094604492,
+            -37.05697250366211,
+            -67.57942199707031,
+            -12.65576171875
+          ],
+          [
+            -36.79258728027344,
+            -60.70497131347656,
+            -39.81881332397461,
+            -70.3065185546875,
+            -13.115116119384766
+          ],
+          [
+            -38.41820526123047,
+            -63.38841247558594,
+            -42.285457611083984,
+            -72.76057434082031,
+            -13.479698181152344
+          ],
+          [
+            -39.82875061035156,
+            -65.73641967773438,
+            -44.643733978271484,
+            -74.99454498291016,
+            -14.06201171875
+          ],
+          [
+            -41.06019592285156,
+            -67.91671752929688,
+            -46.73720932006836,
+            -77.06887817382812,
+            -14.582843780517578
+          ],
+          [
+            -42.17378234863281,
+            -70.09700775146484,
+            -48.38654708862305,
+            -79.13003540039062,
+            -15.38583755493164
+          ]
+        ],
+        "max": [
+          [
+            2.8082656860351562,
+            15.1451416015625,
+            15.772930145263672,
+            30.926345825195312,
+            4.617164611816406
+          ],
+          [
+            6.609169006347656,
+            24.240886688232422,
+            25.535900115966797,
+            40.97989273071289,
+            5.822669982910156
+          ],
+          [
+            7.277320861816406,
+            26.672752380371094,
+            27.968975067138672,
+            45.56070327758789,
+            6.975105285644531
+          ],
+          [
+            8.466912269592285,
+            28.433757781982422,
+            29.670612335205078,
+            49.64745330810547,
+            7.9514312744140625
+          ],
+          [
+            10.17441177368164,
+            30.027050018310547,
+            31.17824935913086,
+            53.44855499267578,
+            8.888931274414062
+          ],
+          [
+            11.881908416748047,
+            31.36876678466797,
+            32.392520904541016,
+            57.183837890625,
+            9.774349212646484
+          ],
+          [
+            13.589406967163086,
+            32.54277420043945,
+            33.3309440612793,
+            60.89848327636719,
+            10.570022583007812
+          ],
+          [
+            14.927953720092773,
+            35.78352737426758,
+            36.78923416137695,
+            68.95306396484375,
+            11.388931274414062
+          ],
+          [
+            16.190017700195312,
+            38.634681701660156,
+            40.39845657348633,
+            73.93795776367188,
+            12.274349212646484
+          ],
+          [
+            17.22235870361328,
+            41.48583221435547,
+            43.944515228271484,
+            78.80449676513672,
+            13.070022583007812
+          ],
+          [
+            18.84430503845215,
+            45.80079650878906,
+            47.110504150390625,
+            83.459228515625,
+            13.78769302368164
+          ],
+          [
+            20.551803588867188,
+            49.082237243652344,
+            50.16779327392578,
+            87.70958709716797,
+            14.449142456054688
+          ],
+          [
+            22.117164611816406,
+            52.18496322631836,
+            53.71385192871094,
+            91.4458236694336,
+            15.10708236694336
+          ],
+          [
+            23.676183700561523,
+            55.36054611206055,
+            56.553443908691406,
+            94.54940032958984,
+            15.787273406982422
+          ],
+          [
+            25.160964965820312,
+            58.64198684692383,
+            58.729164123535156,
+            97.79720306396484,
+            16.412574768066406
+          ],
+          [
+            26.52654266357422,
+            61.636837005615234,
+            60.90012741088867,
+            102.08735656738281,
+            17.037574768066406
+          ]
+        ],
+        "mean": [
+          [
+            -0.13650745153427124,
+            -0.4677199423313141,
+            -0.8473921418190002,
+            -0.7649565935134888,
+            -0.021094100549817085
+          ],
+          [
+            -0.20539112389087677,
+            -0.3632800877094269,
+            -0.8333888649940491,
+            -0.9195149540901184,
+            -0.027212025597691536
+          ],
+          [
+            -0.2723381519317627,
+            -0.2653297483921051,
+            -0.8210741281509399,
+            -1.0628538131713867,
+            -0.033518265932798386
+          ],
+          [
+            -0.33753377199172974,
+            -0.1744663566350937,
+            -0.8102078437805176,
+            -1.194720983505249,
+            -0.039917171001434326
+          ],
+          [
+            -0.40112099051475525,
+            -0.08965988457202911,
+            -0.800645649433136,
+            -1.3166823387145996,
+            -0.04640614613890648
+          ],
+          [
+            -0.4633120000362396,
+            -0.010710185393691063,
+            -0.7921202778816223,
+            -1.4293442964553833,
+            -0.05297448858618736
+          ],
+          [
+            -0.5243921875953674,
+            0.06191810965538025,
+            -0.7841764688491821,
+            -1.5325887203216553,
+            -0.05967888608574867
+          ],
+          [
+            -0.5849394202232361,
+            0.12821657955646515,
+            -0.7767584323883057,
+            -1.6264628171920776,
+            -0.06637542694807053
+          ],
+          [
+            -0.6457070708274841,
+            0.1892988383769989,
+            -0.7695739269256592,
+            -1.7125898599624634,
+            -0.0730169415473938
+          ],
+          [
+            -0.7074253559112549,
+            0.24505209922790527,
+            -0.7624726295471191,
+            -1.7910035848617554,
+            -0.07966674119234085
+          ],
+          [
+            -0.770281970500946,
+            0.29502570629119873,
+            -0.7553150653839111,
+            -1.8618149757385254,
+            -0.08618798851966858
+          ],
+          [
+            -0.8341434597969055,
+            0.33993807435035706,
+            -0.7477642893791199,
+            -1.9256418943405151,
+            -0.09246616810560226
+          ],
+          [
+            -0.8997676968574524,
+            0.3799499273300171,
+            -0.7391418814659119,
+            -1.9840985536575317,
+            -0.09862944483757019
+          ],
+          [
+            -0.9675423502922058,
+            0.4147157371044159,
+            -0.7288776636123657,
+            -2.037827968597412,
+            -0.1048692837357521
+          ],
+          [
+            -1.037450909614563,
+            0.44359350204467773,
+            -0.7166252136230469,
+            -2.0869860649108887,
+            -0.11122476309537888
+          ],
+          [
+            -1.1097698211669922,
+            0.4664764404296875,
+            -0.701746940612793,
+            -2.132956027984619,
+            -0.1176241859793663
+          ]
+        ],
+        "std": [
+          [
+            0.3108135461807251,
+            2.1086082458496094,
+            1.4324616193771362,
+            3.145540475845337,
+            0.7327245473861694
+          ],
+          [
+            0.8329985737800598,
+            3.7673492431640625,
+            2.448432445526123,
+            4.642838001251221,
+            0.915313720703125
+          ],
+          [
+            1.3731037378311157,
+            5.437337398529053,
+            3.4790890216827393,
+            6.143582344055176,
+            1.0956119298934937
+          ],
+          [
+            1.9086061716079712,
+            7.101789951324463,
+            4.500624656677246,
+            7.635791778564453,
+            1.270045518875122
+          ],
+          [
+            2.438143730163574,
+            8.75395679473877,
+            5.50702428817749,
+            9.112739562988281,
+            1.438236117362976
+          ],
+          [
+            2.9614508152008057,
+            10.391569137573242,
+            6.496376037597656,
+            10.573601722717285,
+            1.6005955934524536
+          ],
+          [
+            3.478393793106079,
+            12.014689445495605,
+            7.468076229095459,
+            12.02019214630127,
+            1.7576541900634766
+          ],
+          [
+            3.987931489944458,
+            13.622572898864746,
+            8.4224271774292,
+            13.456147193908691,
+            1.9097516536712646
+          ],
+          [
+            4.489015579223633,
+            15.213079452514648,
+            9.359692573547363,
+            14.875533103942871,
+            2.0570716857910156
+          ],
+          [
+            4.9812188148498535,
+            16.785816192626953,
+            10.280274391174316,
+            16.279115676879883,
+            2.199845552444458
+          ],
+          [
+            5.464637756347656,
+            18.340810775756836,
+            11.184697151184082,
+            17.667957305908203,
+            2.338259220123291
+          ],
+          [
+            5.9398908615112305,
+            19.876331329345703,
+            12.0739107131958,
+            19.04352378845215,
+            2.472538471221924
+          ],
+          [
+            6.405988693237305,
+            21.39141273498535,
+            12.947680473327637,
+            20.401578903198242,
+            2.602895736694336
+          ],
+          [
+            6.8628830909729,
+            22.884916305541992,
+            13.806445121765137,
+            21.74209213256836,
+            2.7295615673065186
+          ],
+          [
+            7.310882568359375,
+            24.356969833374023,
+            14.650439262390137,
+            23.06608772277832,
+            2.8527798652648926
+          ],
+          [
+            7.749383926391602,
+            25.805896759033203,
+            15.47972583770752,
+            24.371322631835938,
+            2.9727964401245117
+          ]
+        ],
+        "q01": [
+          [
+            -0.9051208305358887,
+            -5.201502494812011,
+            -3.8378103256225584,
+            -8.952060127258301,
+            -2.3931473541259765
+          ],
+          [
+            -2.471837434768677,
+            -8.705605869293214,
+            -6.1647023010253905,
+            -12.89043288230896,
+            -3.008226432800293
+          ],
+          [
+            -4.110925331115722,
+            -12.253431720733643,
+            -8.484132919311524,
+            -16.768682098388673,
+            -3.608476371765137
+          ],
+          [
+            -5.75265643119812,
+            -15.789856986999512,
+            -10.747563400268554,
+            -20.574179191589355,
+            -4.141338157653808
+          ],
+          [
+            -7.352046937942505,
+            -19.271711978912354,
+            -13.014918251037598,
+            -24.250542907714845,
+            -4.661877288818359
+          ],
+          [
+            -8.906445770263671,
+            -22.71067506790161,
+            -15.178997230529784,
+            -27.820487251281737,
+            -5.194190864562988
+          ],
+          [
+            -10.453160247802735,
+            -26.16987955093384,
+            -17.238589096069337,
+            -31.385007781982424,
+            -5.616140365600586
+          ],
+          [
+            -11.949403495788575,
+            -29.50733991622925,
+            -19.241472282409667,
+            -34.830007247924804,
+            -6.126104125976562
+          ],
+          [
+            -13.442130460739136,
+            -32.75207595825195,
+            -21.227107734680175,
+            -38.213056106567386,
+            -6.591048393249512
+          ],
+          [
+            -14.913943681716919,
+            -35.75944507598877,
+            -23.224502639770506,
+            -41.565428504943846,
+            -7.049036369323731
+          ],
+          [
+            -16.337127132415773,
+            -38.877289581298825,
+            -25.1516646194458,
+            -44.82405395507813,
+            -7.5608226013183595
+          ],
+          [
+            -17.74782602310181,
+            -41.74310558319092,
+            -27.112762126922608,
+            -47.945948638916015,
+            -7.970102157592773
+          ],
+          [
+            -19.192581405639647,
+            -44.7731888961792,
+            -28.949325141906737,
+            -51.02447566986084,
+            -8.38568546295166
+          ],
+          [
+            -20.555711765289306,
+            -47.736771774291995,
+            -30.758780307769776,
+            -54.03051551818847,
+            -8.83361312866211
+          ],
+          [
+            -22.00614896774292,
+            -50.76547737121582,
+            -32.48425132751465,
+            -57.05641273498535,
+            -9.113520126342774
+          ],
+          [
+            -23.3553165435791,
+            -53.718587226867676,
+            -34.147215347290036,
+            -59.73561668395996,
+            -9.4493701171875
+          ]
+        ],
+        "q99": [
+          [
+            0.49832933723926537,
+            3.797894344329834,
+            3.2793192672729488,
+            7.872846636772154,
+            2.3757497024536125
+          ],
+          [
+            1.7570628011226654,
+            7.053264045715332,
+            6.01789608001709,
+            12.058475570678711,
+            2.9948218917846665
+          ],
+          [
+            3.0012492847442616,
+            10.504232788085938,
+            9.14252864837646,
+            16.25984489440918,
+            3.6006008529663074
+          ],
+          [
+            4.25918544769287,
+            13.8760725402832,
+            12.175011253356933,
+            20.49592012405395,
+            4.16137584686279
+          ],
+          [
+            5.390107412338254,
+            17.200147171020504,
+            15.157647514343259,
+            24.66618663787841,
+            4.660554695129391
+          ],
+          [
+            6.539512734413146,
+            20.4291674041748,
+            18.09365829467773,
+            28.940540199279784,
+            5.177352981567381
+          ],
+          [
+            7.652539024353027,
+            23.671152973175047,
+            20.983547439575194,
+            33.19656345367431,
+            5.632239151000973
+          ],
+          [
+            8.741564941406242,
+            26.57767150878906,
+            23.824512290954583,
+            36.67815811157226,
+            6.111554946899394
+          ],
+          [
+            9.780613365173334,
+            29.62935256958007,
+            26.606433830261224,
+            40.67216819763182,
+            6.600919151306153
+          ],
+          [
+            10.886824455261227,
+            32.61225460052489,
+            29.2786895751953,
+            44.76439350128169,
+            6.959247703552245
+          ],
+          [
+            11.864203271865835,
+            35.41606624603271,
+            31.92717166900634,
+            48.39228778839111,
+            7.385690155029296
+          ],
+          [
+            12.855848236083979,
+            38.271680412292476,
+            34.57434051513672,
+            52.11393959045408,
+            7.753778991699215
+          ],
+          [
+            13.883966655731202,
+            41.07266147613525,
+            36.89253074645996,
+            55.74001392364501,
+            8.024913902282712
+          ],
+          [
+            14.92360103607177,
+            43.84491764068603,
+            39.170840911865234,
+            59.02492687225339,
+            8.399250564575194
+          ],
+          [
+            15.77400249481201,
+            46.636178932189935,
+            41.60740081787109,
+            62.55514793395993,
+            8.658222274780274
+          ],
+          [
+            16.824822902679443,
+            49.18269191741943,
+            43.65656803131103,
+            65.87823959350585,
+            8.868799972534177
+          ]
+        ]
+      }
+    }
+  }
+}

experiment_cfg/final_model_config.json

@@ -0,0 +1,53 @@
+{
+  "model_type": "Gr00tN1d6",
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Eagle-Block2A-2B-v2",
+  "backbone_model_type": "eagle",
+  "model_revision": null,
+  "tune_top_llm_layers": 4,
+  "backbone_embedding_dim": 2048,
+  "tune_llm": false,
+  "tune_visual": false,
+  "select_layer": 16,
+  "reproject_vision": false,
+  "use_flash_attention": true,
+  "load_bf16": true,
+  "collator_overwrite_image_inputs": false,
+  "eagle_collator": true,
+  "backbone_trainable_params_fp32": true,
+  "apply_sincos_state_encoding": true,
+  "use_relative_action": true,
+  "max_state_dim": 128,
+  "max_action_dim": 128,
+  "action_horizon": 50,
+  "hidden_size": 1024,
+  "input_embedding_dim": 1536,
+  "add_pos_embed": true,
+  "attn_dropout": 0.2,
+  "use_vlln": true,
+  "max_seq_len": 1024,
+  "use_alternate_vl_dit": true,
+  "attend_text_every_n_blocks": 2,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "num_inference_timesteps": 4,
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_timestep_buckets": 1000,
+  "tune_projector": true,
+  "tune_diffusion_model": true,
+  "tune_vlln": true,
+  "state_dropout_prob": 0.0,
+  "state_additive_noise_scale": 0.0,
+  "max_num_embodiments": 32
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b2840079d9ef66d5df386f63ecd931742b955fb162ceb9396fb6899fdcdd274
+size 4990120184

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba0679aa36fbe8c1ee38792f20b78ef395200681cd4cd2196e429415899a324a
+size 4823190320

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21829d4ff2f5c712724f2c719af7180063a3e2cc02ebff525e4855c07a2022ee
+size 12960193762

processor_config.json

@@ -0,0 +1,455 @@
+{
+  "processor_class": "Gr00tN1d6Processor",
+  "processor_kwargs": {
+    "modality_configs": {
+      "behavior_r1_pro": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "observation.images.rgb.head_256_256",
+            "observation.images.rgb.left_wrist_256_256",
+            "observation.images.rgb.right_wrist_256_256"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "robot_pos",
+            "robot_ori_cos",
+            "robot_ori_sin",
+            "robot_2d_ori",
+            "robot_2d_ori_cos",
+            "robot_2d_ori_sin",
+            "robot_lin_vel",
+            "robot_ang_vel",
+            "arm_left_qpos",
+            "arm_left_qpos_sin",
+            "arm_left_qpos_cos",
+            "eef_left_pos",
+            "eef_left_quat",
+            "gripper_left_qpos",
+            "arm_right_qpos",
+            "arm_right_qpos_sin",
+            "arm_right_qpos_cos",
+            "eef_right_pos",
+            "eef_right_quat",
+            "gripper_right_qpos",
+            "trunk_qpos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31
+          ],
+          "modality_keys": [
+            "base",
+            "torso",
+            "left_arm",
+            "left_gripper",
+            "right_arm",
+            "right_gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "trunk_qpos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_left_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_right_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "gr1": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "ego_view_bg_crop_pad_res256_freq20"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "task"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "robocasa_panda_omron": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "res256_image_side_0",
+            "res256_image_side_1",
+            "res256_image_wrist_0"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "end_effector_position_relative",
+            "end_effector_rotation_relative",
+            "gripper_qpos",
+            "base_position",
+            "base_rotation"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "end_effector_position",
+            "end_effector_rotation",
+            "gripper_close",
+            "base_motion",
+            "control_mode"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.action.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "new_embodiment": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "front",
+            "wrist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      }
+    },
+    "image_crop_size": null,
+    "image_target_size": null,
+    "use_albumentations": true,
+    "random_rotation_angle": null,
+    "color_jitter_params": {
+      "brightness": 0.3,
+      "contrast": 0.4,
+      "saturation": 0.5,
+      "hue": 0.08
+    },
+    "shortest_image_edge": 256,
+    "crop_fraction": 0.95,
+    "model_name": "nvidia/Eagle-Block2A-2B-v2",
+    "model_type": "eagle",
+    "formalize_language": true,
+    "max_state_dim": 128,
+    "max_action_dim": 128,
+    "max_action_horizon": 50,
+    "use_percentiles": false,
+    "clip_outliers": true,
+    "apply_sincos_state_encoding": true,
+    "use_relative_action": true
+  }
+}

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f40f14c749486cb921a4172109eec747f70a02e7084acf7a22edd8d56f985a31
+size 14645

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d02be6d8bda4ea9c67040ed89f878acdc986bd4df3fbc60440a9d3eacca02d63
+size 1465

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fb22db84b78dd75cef65691c1f4cdd017de52c2bc9e978195c6c471bdd7c581
+size 5777

wandb_config.json

@@ -0,0 +1 @@
+{"project": "finetune-gr00t-n1d6", "run_id": "sreetz-nv-so101_teleop_vials_rack_left_augment_02"}