Add files using upload-large-folder tool

config.yaml

@@ -0,0 +1,69 @@
+datasets:
+  vla_data:
+    CoT_prompt: Your task is {instruction}. To identify the key objects for your task.
+      Locate their bounding boxes in [x1,y1,x2,y2] format.
+    data_mix: bridge_rt_1_vp
+    data_root_dir: /gpfs/wangzixuan/visual_prompting/data_process/datasets
+    dataset_py: visual_prompt_datasets
+    delete_pause_frame: false
+    feed_both_images: true
+    image_size:
+    - 224
+    - 224
+    per_device_batch_size: 32
+    target_location_prompt_type: box
+    target_object_prompt_type: crosshair
+    use_subtask: false
+    video_backend: pyav
+    visual_prompt_dir: /gpfs/wangzixuan/visual_prompting/starVLA_robocasa/simplerenv_process/visual_prompts_output
+  vp_data:
+    dataset_py: visual_prompt_prediction_datasets
+    extracted_frames_dir: /gpfs/wangzixuan/visual_prompting/data_process/extracted_frames_simplerenv
+    feed_both_images: false
+    per_device_batch_size: 8
+    target_location_prompt_type: box
+    target_object_prompt_type: crosshair
+    use_chunked_frames: true
+    visual_prompt_dir: /gpfs/wangzixuan/visual_prompting/starVLA_robocasa/simplerenv_process/visual_prompts_output
+framework:
+  action_model:
+    action_dim: 7
+    action_hidden_dim: 2560
+    action_model_type: DiT-B
+    future_action_window_size: 15
+    past_action_window_size: 0
+  name: QwenOFT
+  qwenvl:
+    base_vlm: /gpfs/wangzixuan/visual_prompting/starVLA_robocasa/playground/Pretrained_models/Qwen3-VL-4B-Instruct
+output_dir: /gpfs/wangzixuan/visual_prompting/starVLA_robocasa/playground/Checkpoints/oxe_bridge_rt_1_vp_visual_prompt_training_QwenOFT_both_image_no_subtask
+run_id: oxe_bridge_rt_1_vp_visual_prompt_training_QwenOFT_both_image_no_subtask
+run_root_dir: /gpfs/wangzixuan/visual_prompting/starVLA_robocasa/playground/Checkpoints
+seed: 42
+trainer:
+  eval_interval: 100
+  freeze_modules: null
+  gradient_accumulation_steps: 1
+  gradient_clipping: 1.0
+  is_resume: false
+  learning_rate:
+    action_model: 0.0001
+    base: 3.0e-05
+    qwen_vl_interface: 1.0e-05
+  logging_frequency: 10
+  loss_scale:
+    visual_prompt: 0.1
+    vla: 1.0
+  lr_scheduler_type: cosine_with_min_lr
+  max_train_steps: 100000
+  num_warmup_steps: 5000
+  optimizer:
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    weight_decay: 1.0e-08
+  save_interval: 10000
+  scheduler_specific_kwargs:
+    min_lr: 5.0e-07
+wandb_entity: zwanggk
+wandb_project: starVLA_simplerEnv_visual_prompt

dataset_statistics.json

@@ -0,0 +1,264 @@
+{
+  "oxe_bridge": {
+    "action": {
+      "mean": [
+        0.00011365935642970726,
+        6.556110747624189e-05,
+        -6.320965621853247e-05,
+        -7.205353176686913e-05,
+        -0.00019515302847139537,
+        0.0001203166029881686,
+        0.28829458355903625
+      ],
+      "std": [
+        0.006909770731857718,
+        0.009684093232852218,
+        0.00896290498528129,
+        0.020121052930683073,
+        0.021582655517295487,
+        0.054723342223346974,
+        0.4543627821514982
+      ],
+      "max": [
+        0.41691166162490845,
+        0.25864794850349426,
+        0.21218234300613403,
+        3.122201919555664,
+        1.8618112802505493,
+        6.272472858428955,
+        1.0
+      ],
+      "min": [
+        -0.4007510244846344,
+        -0.13874775171279907,
+        -0.22553899884223938,
+        -3.2010786533355713,
+        -1.8618112802505493,
+        -6.279075622558594,
+        0.0
+      ],
+      "q01": [
+        -0.02875255048274994,
+        -0.04170213546603918,
+        -0.026096721179783344,
+        -0.08052874729037285,
+        -0.09249906800687313,
+        -0.20738555490970612,
+        0.0
+      ],
+      "q99": [
+        0.028306663036346436,
+        0.04089853074401617,
+        0.0401805154979229,
+        0.08173403143882751,
+        0.07760760560631752,
+        0.2038465365767479,
+        1.0
+      ],
+      "mask": [
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        false
+      ]
+    },
+    "state": {
+      "mean": [
+        0.15471743047237396,
+        0.015362550504505634,
+        0.03222028166055679,
+        0.0032453201711177826,
+        -0.038600146770477295,
+        0.05382946878671646,
+        0.0,
+        0.35406652092933655
+      ],
+      "std": [
+        0.1605359274864927,
+        0.06677912092232112,
+        0.048657150951527554,
+        0.09275336958940797,
+        0.12566984746914736,
+        0.41222738578801793,
+        0.0,
+        0.4330223535305803
+      ],
+      "max": [
+        0.5862360596656799,
+        0.4034728705883026,
+        0.3568263053894043,
+        1.3517684936523438,
+        1.570796251296997,
+        3.141204357147217,
+        0.0,
+        1.1121242046356201
+      ],
+      "min": [
+        -0.04167502000927925,
+        -0.3563207685947418,
+        -0.15537554025650024,
+        -3.141592502593994,
+        -1.4992541074752808,
+        -3.14153790473938,
+        0.0,
+        0.04637829214334488
+      ],
+      "q01": [
+        0.17102622985839844,
+        -0.1698118858039379,
+        -0.055632163770496845,
+        -0.36493172496557236,
+        -0.541871190071106,
+        -1.354275494813919,
+        0.0,
+        0.052190229296684265
+      ],
+      "q99": [
+        0.45322076976299286,
+        0.2354845181107521,
+        0.19489620998501778,
+        0.3780156075954437,
+        0.27568644285202026,
+        1.8500566184520721,
+        0.0,
+        1.0105689764022827
+      ]
+    },
+    "num_transitions": 1305714,
+    "num_trajectories": 53192
+  },
+  "oxe_rt1": {
+    "action": {
+      "mean": [
+        0.003493750700727105,
+        0.003132961690425873,
+        -0.0063125672750175,
+        0.02166595682501793,
+        -0.0028780836146324873,
+        0.0004565489653032273,
+        0.26771023869514465
+      ],
+      "std": [
+        0.049065014465362655,
+        0.04229853739828572,
+        0.05237628880142378,
+        0.1124860236500875,
+        0.09312952783816872,
+        0.10319098309601091,
+        0.4418448662622395
+      ],
+      "max": [
+        2.9984593391418457,
+        22.09052848815918,
+        2.7507524490356445,
+        1.570636510848999,
+        1.5321086645126343,
+        1.5691522359848022,
+        1.0
+      ],
+      "min": [
+        -2.0204520225524902,
+        -5.497899532318115,
+        -2.031663417816162,
+        -1.569917917251587,
+        -1.569892168045044,
+        -1.570419430732727,
+        0.0
+      ],
+      "q01": [
+        -0.22453527510166169,
+        -0.14820013284683228,
+        -0.231589707583189,
+        -0.3517994859814644,
+        -0.4193011274933815,
+        -0.43643461108207704,
+        0.0
+      ],
+      "q99": [
+        0.17824687153100965,
+        0.14938379630446405,
+        0.21842354819178575,
+        0.5892666035890578,
+        0.35272657424211445,
+        0.44796681255102094,
+        1.0
+      ],
+      "mask": [
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        false
+      ]
+    },
+    "state": {
+      "mean": [
+        0.2799473702907562,
+        -0.04167069122195244,
+        0.38854750990867615,
+        0.21306729316711426,
+        -0.12402277439832687,
+        0.24756911396980286,
+        0.046330634504556656,
+        0.10487449914216995
+      ],
+      "std": [
+        0.29342642876909925,
+        0.09174024655686211,
+        0.42569508885539115,
+        0.38616252611341306,
+        0.38314586427420927,
+        0.44433568806919804,
+        0.1263927443679382,
+        0.22122596673781084
+      ],
+      "max": [
+        1.0534898042678833,
+        0.48018959164619446,
+        1.6896663904190063,
+        1.0,
+        0.9999993443489075,
+        0.9999874830245972,
+        0.9554369449615479,
+        0.9914546012878418
+      ],
+      "min": [
+        -0.4436439275741577,
+        -0.9970501065254211,
+        -0.006579156965017319,
+        0.0,
+        -0.8643477559089661,
+        -0.7079970240592957,
+        -0.7688722014427185,
+        -0.4999994933605194
+      ],
+      "q01": [
+        0.32481380939483645,
+        -0.28334290891885755,
+        0.14107070609927178,
+        0.0,
+        -0.686474204659462,
+        -0.6808923494815826,
+        -0.36045596331357954,
+        -0.454380963742733
+      ],
+      "q99": [
+        0.8750156319141384,
+        0.21247054174542404,
+        1.0727112340927123,
+        1.0,
+        0.9377871316671368,
+        0.9563051050901409,
+        0.45990042358636823,
+        0.7216041100025177
+      ]
+    },
+    "num_transitions": 3786152,
+    "num_trajectories": 87212
+  }
+}

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_0_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c98f3bfad288c3c2c06cc6f0e610a9a799a315e2ff45f953b516fcedf7e9099
+size 294718

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_10_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff9a82413ff2a0714955d6f53bf47fc11807baa28bdc5c1926d3b8b65d8f60c4
+size 299734

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_11_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61f5fbba5515b78bfbe15ec5aae0d6c25bccd457676e5a393fc69bb36d0dc25d
+size 176026

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_12_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfe1f3889d3af71c96315aa835f67a15805918adaecb683986922324699bb72d
+size 267219

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_13_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19b2e8094e38b04b958b6e0477f2df4de4dfed8a8eb04b797c200f8e85409cd2
+size 326723

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_15_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19edf65e24aee9f7f6cfb51ad7bd40ffd36a44aba9b91be4b40262b25c613269
+size 293453

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_18_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:604777ff94fc6115106f8f1e1468b42e275a56a619968b12a336557739ad7ba0
+size 304689

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_19_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8d392802a57337fad14fc083ec20b21031937ae124c32f9f7e97c3c8104df46
+size 250728

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_1_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16f2d4df7c69e64092b61e978382315ee8adabe329d59830495f2066e1c7cc21
+size 246796

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_20_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aae50e84f718c503a0ed2fc1244a2f7d587bd4baf88347b2c9ad7864e6875252
+size 187279

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_21_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c2db2ac62edc687a987cab46cfe0a80d141b8b8c77347489ace1b1baecadd72
+size 275788

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_2_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cee250c02691a76c14428b256b3a585bab23c98809d1ea224e7cb9b1a05098b
+size 240156

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_3_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e790928d4abf58e847f10d9e68e20913979afe7556bd1808dfa3d0e11b40e4c
+size 262170

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_4_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efdbf427acefdda20ac5adfabbfb8354b048f7f2fbc92e34219fe80cc021bc1b
+size 305710

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_5_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df309c7c4d8dd6fa6ae85ec7cf45a01e80e080e349fb0f11e6a53509694bd08b
+size 246548

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_6_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a016770975ce2ed02c956869e371c2a685b602459152d62830552c5fe2438391
+size 269198

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_7_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d1ce53332fcb538f1f3f7dbd881b068fdc50b0837e9fc8b1b14b438f5148570
+size 293609

results/steps_70000_pytorch_model_PutEggplantInBasketScene-v0_run1/steps_70000_pytorch_model.pt/bridge_table_1_v2/arm_pd_ee_target_delta_pose_align2_gripper_pd_joint_pos/PutEggplantInBasketScene-v0/rob_0.127_0.06_rot_0.000_-0.000_3.142_rgb_overlay_bridge_sink/success_obj_episode_8_moved_correct_obj_True_moved_wrong_obj_False_is_src_obj_grasped_True_consecutive_grasp_True_src_on_target_True.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0db4f67ccff58971873cb354a35d1932693940c6b10ce843e94e933f33b96ee5
+size 300961

run_oxe_visual_prompt.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+# Visual Prompt Training Script for SimplerEnv (OXE datasets)
+# export CUDA_VISIBLE_DEVICES=2,3,4,5,6,7
+export NCCL_SOCKET_IFNAME=bond0
+export NCCL_IB_HCA=mlx5_2,mlx5_3
+
+# used for check save when communication
+export NCCL_BLOCKING_WAIT=1
+export NCCL_ASYNC_ERROR_HANDLING=1
+export TORCH_NCCL_BLOCKING_WAIT=1
+export TORCH_NCCL_ASYNC_ERROR_HANDLING=1
+
+# Timeout settings for distributed operations (in seconds)
+export NCCL_TIMEOUT=3600
+export TORCH_DISTRIBUTED_DEBUG=DETAIL
+
+###########################################################################################
+# === Please modify the following paths according to your environment ===
+Framework_name=QwenOFT
+base_vlm=/gpfs/wangzixuan/visual_prompting/starVLA_robocasa/playground/Pretrained_models/Qwen3-VL-4B-Instruct
+freeze_module_list=''
+DIT_TYPE="DiT-B"
+
+# Data paths
+data_root_dir=/gpfs/wangzixuan/visual_prompting/data_process/datasets
+visual_prompt_dir=/gpfs/wangzixuan/visual_prompting/starVLA_robocasa/simplerenv_process/visual_prompts_output
+extracted_frames_dir=/gpfs/wangzixuan/visual_prompting/data_process/extracted_frames_simplerenv
+data_mix=bridge_rt_1_vp
+
+# Config: choose between separate VP dataloader or inline VP
+# For separate VP dataloader mode:
+config_yaml=./examples/SimplerEnv/train_files/starvla_cotrain_oxe_visual_prompt.yaml
+# For inline VP mode (uncomment to use):
+# config_yaml=./examples/SimplerEnv/train_files/starvla_cotrain_oxe_visual_prompt_inline_vp.yaml
+
+# Output
+run_root_dir=/gpfs/wangzixuan/visual_prompting/starVLA_robocasa/playground/Checkpoints
+run_id=oxe_${data_mix}_visual_prompt_training_${Framework_name}_both_image_no_subtask
+# === End of environment variable configuration ===
+###########################################################################################
+
+output_dir=${run_root_dir}/${run_id}
+mkdir -p ${output_dir}
+# Save this script to the output dir
+cp $0 ${output_dir}/
+
+accelerate launch \
+  --config_file starVLA/config/deepseeds/deepspeed_zero2.yaml \
+  --num_processes 8 \
+  starVLA/training/train_starvla_visual_prompt.py \
+  --config_yaml ${config_yaml} \
+  --framework.name ${Framework_name} \
+  --framework.qwenvl.base_vlm ${base_vlm} \
+  --framework.action_model.action_model_type ${DIT_TYPE} \
+  --datasets.vla_data.data_root_dir ${data_root_dir} \
+  --datasets.vla_data.visual_prompt_dir ${visual_prompt_dir} \
+  --datasets.vla_data.data_mix ${data_mix} \
+  --datasets.vla_data.per_device_batch_size 32 \
+  --datasets.vla_data.video_backend pyav \
+  --datasets.vp_data.visual_prompt_dir ${visual_prompt_dir} \
+  --datasets.vp_data.extracted_frames_dir ${extracted_frames_dir} \
+  --datasets.vp_data.per_device_batch_size 8 \
+  --trainer.freeze_modules "${freeze_module_list}" \
+  --trainer.max_train_steps 100000 \
+  --trainer.save_interval 10000 \
+  --trainer.logging_frequency 10 \
+  --trainer.eval_interval 100 \
+  --trainer.learning_rate.base 3e-5 \
+  --trainer.learning_rate.qwen_vl_interface 1e-5 \
+  --trainer.loss_scale.visual_prompt 0.1 \
+  --datasets.vla_data.use_subtask false \
+  --datasets.vla_data.feed_both_images true \
+  --datasets.vp_data.feed_both_images false \
+  --run_root_dir ${run_root_dir} \
+  --run_id ${run_id} \
+  --wandb_project starVLA_simplerEnv_visual_prompt \
+  --wandb_entity zwanggk
+  # --is_debug True

summary.jsonl

@@ -0,0 +1,10 @@
+{"steps": 10000}
+{"steps": 20000}
+{"steps": 30000}
+{"steps": 40000}
+{"steps": 50000}
+{"steps": 60000}
+{"steps": 70000}
+{"steps": 80000}
+{"steps": 90000}
+{"steps": 100000}