Add files using upload-large-folder tool

checkpoints/steps_100000_pytorch_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3006aad361d25e37203bad9e441f4ef84e207b1b62547b5ec2c204f55a426795
+size 9785133528

config.full.yaml

@@ -0,0 +1,62 @@
+run_id: 0615_franka_v3_dual_finetune_qwen3OFT
+run_root_dir: /gpfs/wangzixuan/Octopus/workspace/starVLA/results/Checkpoints
+seed: 42
+trackers:
+- jsonl
+- wandb
+wandb_entity: zwanggk
+wandb_project: franka_dual
+is_debug: false
+framework:
+  name: QwenOFT
+  qwenvl:
+    base_vlm: /gpfs/wangzixuan/Octopus/workspace/MODEL/Qwen3-VL-4B-Instruct
+    attn_implementation: flash_attention_2
+  action_model:
+    action_model_type: DiT-B
+    action_dim: 14
+    action_hidden_dim: 2560
+    future_action_window_size: 15
+    past_action_window_size: 0
+    state_dim: 56
+  obs_image_size: null
+datasets:
+  vla_data:
+    dataset_py: lerobot_datasets
+    data_root_dir: /gpfs/wangzixuan/Octopus/workspace/data/lerobot/v3
+    data_mix: real_franka_dual_v3
+    image_size:
+    - 224
+    - 224
+    per_device_batch_size: 16
+    sequential_step_sampling: false
+    video_backend: torchvision_av
+trainer:
+  pretrained_checkpoint: ''
+  reload_modules: qwen_vl_interface
+  enable_gradient_checkpointing: true
+  enable_mixed_precision_training: true
+  eval_interval: 1000
+  gradient_accumulation_steps: 1
+  gradient_clipping: 1.0
+  is_resume: false
+  learning_rate:
+    action_model: 0.0001
+    base: 1.0e-05
+    qwen_vl_interface: 1.0e-05
+  logging_frequency: 100
+  lr_scheduler_type: cosine_with_min_lr
+  max_train_steps: 100000
+  num_warmup_steps: 5000
+  optimizer:
+    name: AdamW
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    weight_decay: 1.0e-08
+  save_interval: 10000
+  scheduler_specific_kwargs:
+    min_lr: 1.0e-06
+config_yaml: /gpfs/wangzixuan/Octopus/workspace/starVLA/examples/Franka/train_files/starvla_cotrain_franka_dual.yaml
+output_dir: /gpfs/wangzixuan/Octopus/workspace/starVLA/results/Checkpoints/0615_franka_v3_dual_finetune_qwen3OFT

config.yaml

@@ -0,0 +1,49 @@
+datasets:
+  vla_data:
+    data_mix: real_franka_dual_v3
+    data_root_dir: /gpfs/wangzixuan/Octopus/workspace/data/lerobot/v3
+    dataset_py: lerobot_datasets
+    per_device_batch_size: 16
+    sequential_step_sampling: false
+    video_backend: torchvision_av
+framework:
+  action_model:
+    action_dim: 14
+    action_hidden_dim: 2560
+    action_model_type: DiT-B
+    future_action_window_size: 15
+    past_action_window_size: 0
+  name: QwenOFT
+  obs_image_size: null
+  qwenvl:
+    attn_implementation: flash_attention_2
+    base_vlm: /gpfs/wangzixuan/Octopus/workspace/MODEL/Qwen3-VL-4B-Instruct
+output_dir: /gpfs/wangzixuan/Octopus/workspace/starVLA/results/Checkpoints/0615_franka_v3_dual_finetune_qwen3OFT
+run_id: 0615_franka_v3_dual_finetune_qwen3OFT
+run_root_dir: /gpfs/wangzixuan/Octopus/workspace/starVLA/results/Checkpoints
+seed: 42
+trainer:
+  eval_interval: 1000
+  gradient_accumulation_steps: 1
+  gradient_clipping: 1.0
+  is_resume: false
+  learning_rate:
+    action_model: 0.0001
+    base: 1.0e-05
+    qwen_vl_interface: 1.0e-05
+  logging_frequency: 100
+  lr_scheduler_type: cosine_with_min_lr
+  max_train_steps: 100000
+  num_warmup_steps: 5000
+  optimizer:
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    weight_decay: 1.0e-08
+  pretrained_checkpoint: ''
+  save_interval: 10000
+  scheduler_specific_kwargs:
+    min_lr: 1.0e-06
+wandb_entity: zwanggk
+wandb_project: franka_dual

dataset_statistics.json

@@ -0,0 +1,470 @@
+{
+  "franka": {
+    "action": {
+      "mean": [
+        4.888830943627909e-05,
+        7.275665848283097e-05,
+        0.00013938211486674845,
+        -3.9834317249187734e-05,
+        -0.000177113339304924,
+        0.00040537364839110523,
+        0.6170129179954529,
+        2.4617933377157897e-05,
+        0.00018119297237717547,
+        -6.615781421714928e-05,
+        -0.00015037893172120675,
+        -0.00018984398047905415,
+        -9.954750930774026e-05,
+        0.4076392352581024
+      ],
+      "std": [
+        0.0017061373818413443,
+        0.0034916955899380363,
+        0.004133028393974942,
+        0.011171126635678132,
+        0.007544300738580864,
+        0.006390515878212535,
+        0.4595007406568692,
+        0.0031428786768311024,
+        0.004179461820412356,
+        0.005624060150572623,
+        0.0071649092285841235,
+        0.007370721607707996,
+        0.010550570481416506,
+        0.48159860261886167
+      ],
+      "max": [
+        0.04028011113405228,
+        0.03804168477654457,
+        0.09224456548690796,
+        0.10170083492994308,
+        0.07464238256216049,
+        0.08590518683195114,
+        1.0,
+        0.09474976360797882,
+        0.048625245690345764,
+        0.08866537362337112,
+        0.1009439155459404,
+        0.06910572201013565,
+        0.12226438522338867,
+        1.0
+      ],
+      "min": [
+        -0.019486553966999054,
+        -0.218281552195549,
+        -0.07298208028078079,
+        -0.09791475534439087,
+        -0.09798312187194824,
+        -0.46734997630119324,
+        0.03999999165534973,
+        -0.039379335939884186,
+        -0.047175657004117966,
+        -0.0655299574136734,
+        -0.07799048721790314,
+        -0.08318188786506653,
+        -0.2118947058916092,
+        0.0
+      ],
+      "q01": [
+        -0.00567118511069566,
+        -0.010600759731605649,
+        -0.01443211618810892,
+        -0.047052656076848505,
+        -0.02393171044066548,
+        -0.018354202341288327,
+        0.04159999266266823,
+        -0.015083145005628466,
+        -0.011790631618350745,
+        -0.02013110037893057,
+        -0.033742899186909196,
+        -0.02382699530571699,
+        -0.035279906056821346,
+        0.0
+      ],
+      "q99": [
+        0.006479248367249972,
+        0.015182033153250845,
+        0.01331135814078152,
+        0.05234060395508999,
+        0.029672082737088247,
+        0.02366182932630184,
+        1.0,
+        0.00892832356505096,
+        0.018233739994466303,
+        0.01716135745868087,
+        0.027879812866449406,
+        0.02402543496340515,
+        0.04583118740469217,
+        1.0
+      ],
+      "mask": [
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        false,
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        false
+      ]
+    },
+    "state": {
+      "mean": [
+        0.5225273072719574,
+        -0.2753419876098633,
+        0.3952372074127197,
+        0.9655096530914307,
+        -0.07091884687542915,
+        -0.004599429666996002,
+        0.1402406208217144,
+        0.00028598474455066025,
+        -8.446181891486049e-05,
+        -0.0014079651446081698,
+        -0.004435568116605282,
+        -0.003073592553846538,
+        -0.0004460049094632268,
+        0.061698880046606064,
+        -0.183668352663517,
+        0.2383909523487091,
+        -0.2848377451300621,
+        -1.8863731026649475,
+        -0.19816451519727707,
+        2.118559956550598,
+        -0.22095515578985214,
+        -0.0010318580316379666,
+        0.0004890492127742618,
+        0.0011612097077886574,
+        -0.0034881510073319077,
+        -0.0026354988222010434,
+        0.005212518735788763,
+        0.006081253290176392,
+        0.5530485212802887,
+        0.040946973487734795,
+        0.3687969744205475,
+        0.9534865021705627,
+        0.23540794849395752,
+        0.006937553407624364,
+        0.007410785066895187,
+        -0.00045751509605906904,
+        -0.0008233745029428974,
+        0.0009706763667054474,
+        -0.0006119427853263915,
+        0.0033826023573055863,
+        0.004026530077680945,
+        0.0407597403973341,
+        0.22351545095443726,
+        0.16116031259298325,
+        -0.15174874663352966,
+        -2.0403687357902527,
+        0.003178662620484829,
+        2.2143040895462036,
+        -0.41307882964611053,
+        0.0006673439929727465,
+        0.0002969679771922529,
+        -0.0017078065138775855,
+        0.0033627886441536248,
+        -0.0023021422093734145,
+        -0.005433517508208752,
+        -0.003912093234248459
+      ],
+      "std": [
+        0.0459858861114274,
+        0.08786273148759441,
+        0.11814868721339057,
+        0.04907379958898412,
+        0.06940315989170948,
+        0.09826192287376934,
+        0.1618157967506166,
+        0.026058795996946413,
+        0.0431997866695789,
+        0.0576576229375073,
+        0.15725628320150634,
+        0.11905288798465276,
+        0.08948043528504611,
+        0.04594897311829558,
+        0.1864428513668673,
+        0.20083759837423176,
+        0.1308488572348898,
+        0.19852703778860156,
+        0.4043345148767002,
+        0.35048550519173743,
+        0.3838108328910694,
+        0.06755830719334678,
+        0.09419935040899868,
+        0.044146057836964146,
+        0.09376314945090951,
+        0.20234677794738953,
+        0.18477665987340677,
+        0.17760595910236127,
+        0.0458421555609997,
+        0.11256782948141902,
+        0.10799144059430654,
+        0.05147097729229804,
+        0.15094419975797935,
+        0.057787092866158435,
+        0.0810733693609441,
+        0.029013505749157998,
+        0.06568357850162049,
+        0.07890041707204189,
+        0.10504244301492946,
+        0.10793723033103485,
+        0.1487499443164598,
+        0.0481651565249639,
+        0.0994093677694189,
+        0.1932992786479136,
+        0.17453431307818992,
+        0.20165773712657134,
+        0.18298748526172526,
+        0.2777869663066203,
+        0.2522774296886355,
+        0.03866925314961016,
+        0.13174063766679756,
+        0.10212352767872454,
+        0.1442669254294898,
+        0.12228395644526865,
+        0.20096513460519833,
+        0.11287980530672598
+      ],
+      "max": [
+        0.6608633399009705,
+        0.0536913238465786,
+        0.7144683003425598,
+        0.9999862909317017,
+        0.30269643664360046,
+        0.2477765828371048,
+        0.64363694190979,
+        0.2690843343734741,
+        0.3668709695339203,
+        0.6386856436729431,
+        1.3017550706863403,
+        1.2691222429275513,
+        0.945663571357727,
+        0.10000000149011612,
+        0.5626583695411682,
+        0.7449550628662109,
+        0.0038487620186060667,
+        -1.3651877641677856,
+        0.9716548323631287,
+        3.0503945350646973,
+        0.817148745059967,
+        0.6857859492301941,
+        1.393710970878601,
+        0.6017487049102783,
+        1.1624363660812378,
+        1.3030656576156616,
+        1.2630635499954224,
+        1.3022202253341675,
+        0.7067452073097229,
+        0.5060268044471741,
+        0.6526498794555664,
+        0.9999632835388184,
+        0.7836890816688538,
+        0.24925369024276733,
+        0.1915421336889267,
+        0.38790804147720337,
+        0.5545313358306885,
+        0.8613619208335876,
+        0.9833499789237976,
+        1.1220041513442993,
+        1.5539097785949707,
+        0.10000000149011612,
+        0.6370941400527954,
+        0.743476927280426,
+        0.3269256353378296,
+        -1.2877432107925415,
+        0.905453622341156,
+        3.130293130874634,
+        0.5279216170310974,
+        0.6512355804443359,
+        0.9333639144897461,
+        0.9793274402618408,
+        1.2702125310897827,
+        1.2589616775512695,
+        1.256200909614563,
+        1.1229089498519897
+      ],
+      "min": [
+        0.34517064690589905,
+        -0.4875949025154114,
+        0.19975720345973969,
+        0.7139463424682617,
+        -0.3356277644634247,
+        -0.3633454740047455,
+        -0.2499207705259323,
+        -0.3003971576690674,
+        -0.5880947709083557,
+        -1.2908244132995605,
+        -1.50347900390625,
+        -1.393227458000183,
+        -0.8665279746055603,
+        0.003999999258667231,
+        -0.7179145216941833,
+        -0.2891462445259094,
+        -0.7342656850814819,
+        -2.4332010746002197,
+        -1.3178842067718506,
+        1.3762195110321045,
+        -1.4870926141738892,
+        -0.8194271326065063,
+        -1.087807536125183,
+        -0.9683480262756348,
+        -1.1869137287139893,
+        -1.2923732995986938,
+        -1.2708184719085693,
+        -1.271432876586914,
+        0.3698102831840515,
+        -0.21009254455566406,
+        0.1893705427646637,
+        0.6000288724899292,
+        -0.2756439745426178,
+        -0.2144123762845993,
+        -0.39419710636138916,
+        -0.48916691541671753,
+        -0.6300472617149353,
+        -0.6790470480918884,
+        -1.097592830657959,
+        -1.0350394248962402,
+        -1.4383206367492676,
+        0.0,
+        -0.28041553497314453,
+        -0.335645854473114,
+        -0.6683352589607239,
+        -2.7101190090179443,
+        -0.49113473296165466,
+        1.3856040239334106,
+        -1.369844675064087,
+        -0.6505377292633057,
+        -0.9660952687263489,
+        -1.0358529090881348,
+        -1.1253758668899536,
+        -1.311380386352539,
+        -1.3351207971572876,
+        -1.0486830472946167
+      ],
+      "q01": [
+        0.4097293671965599,
+        -0.4635429382324219,
+        0.21975476577877998,
+        0.7808281183242798,
+        -0.25570258736610413,
+        -0.27340927720069885,
+        -0.11834334656596183,
+        -0.09727564752101898,
+        -0.20413877785205842,
+        -0.17199540361762047,
+        -0.6440033316612244,
+        -0.46073482662439347,
+        -0.3259446558356285,
+        0.004159999080002308,
+        -0.6456817388534546,
+        -0.24669814124703407,
+        -0.6108640700578689,
+        -2.358955194950104,
+        -1.2011451721191406,
+        1.482018573284149,
+        -1.2001782655715942,
+        -0.29006130903959276,
+        -0.36152840286493304,
+        -0.20169847205281258,
+        -0.3347303569316864,
+        -0.8705300104618072,
+        -0.6337909603118896,
+        -0.6745864909887314,
+        0.43251982033252717,
+        -0.13683506697416306,
+        0.194706242531538,
+        0.7582097870111465,
+        -0.1655534391105175,
+        -0.1440018041431904,
+        -0.29267518609762194,
+        -0.08764012560248374,
+        -0.29326584190130234,
+        -0.23429986074566841,
+        -0.5299651724100113,
+        -0.3441405379772186,
+        -0.6200219964981079,
+        0.0,
+        -0.18088871702551843,
+        -0.23185264557600022,
+        -0.5314850044250489,
+        -2.4956492042541503,
+        -0.37103135347366334,
+        1.6116769516468048,
+        -1.107481871843338,
+        -0.13140810817480086,
+        -0.49210676550865173,
+        -0.49522090584039685,
+        -0.464592759013176,
+        -0.5899561047554016,
+        -0.7216029566526413,
+        -0.42101414799690245
+      ],
+      "q99": [
+        0.6263421881198883,
+        -0.06764019623398786,
+        0.6210219025611877,
+        0.9997426354885102,
+        0.12973926678299888,
+        0.21129418790340446,
+        0.5704471629858017,
+        0.08815646916627884,
+        0.13553596735000623,
+        0.221183642745018,
+        0.6697821569442747,
+        0.38852209091186485,
+        0.3150095698237406,
+        0.10000000149011612,
+        0.3036853578686692,
+        0.6112922430038452,
+        -0.015440911054611206,
+        -1.51060919046402,
+        0.8029000127315531,
+        2.977726147174836,
+        0.7535334932804107,
+        0.23914075970649692,
+        0.30444157361984264,
+        0.1573584648966803,
+        0.36670864224433974,
+        0.7684417146444318,
+        0.7544230473041537,
+        0.7980052524805044,
+        0.6604236829280853,
+        0.35345613956451416,
+        0.5950973784923554,
+        0.9990113133192062,
+        0.6438277727365493,
+        0.15520899236202254,
+        0.14765383303165436,
+        0.09180136024951935,
+        0.1676537938416008,
+        0.2759536805748944,
+        0.4311562266945844,
+        0.30535165339708426,
+        0.491670161485672,
+        0.10000000149011612,
+        0.5448997020721436,
+        0.6174134093523026,
+        0.27567989051342007,
+        -1.4706526839733123,
+        0.6419919651746753,
+        2.852680335044861,
+        0.22519413024187088,
+        0.14636461555957833,
+        0.4183126360178006,
+        0.2628422331810009,
+        0.5384424775838856,
+        0.5450347787141835,
+        0.6221371710300452,
+        0.3984075739979745
+      ]
+    },
+    "num_transitions": 58302,
+    "num_trajectories": 150
+  }
+}

run_franka_v3_dual_finetune_qwenoft.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+set -eo pipefail
+
+# Activate the starVLA conda environment.
+CONDA_ENV=${CONDA_ENV:-/gpfs/wangzixuan/conda_envs/starVLA}
+# shellcheck disable=SC1091
+source "$(conda info --base)/etc/profile.d/conda.sh"
+conda activate "${CONDA_ENV}"
+
+export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-bond0}
+export NCCL_IB_HCA=${NCCL_IB_HCA:-mlx5_2,mlx5_3}
+export NCCL_BLOCKING_WAIT=${NCCL_BLOCKING_WAIT:-1}
+export NCCL_ASYNC_ERROR_HANDLING=${NCCL_ASYNC_ERROR_HANDLING:-1}
+export NCCL_TIMEOUT=${NCCL_TIMEOUT:-10000}
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+REPO_ROOT=$(cd -- "${SCRIPT_DIR}/../../.." && pwd)
+cd "${REPO_ROOT}"
+export PYTHONPATH="${REPO_ROOT}:${PYTHONPATH:-}"
+
+# v3 dual-arm Franka datasets (converted with --arm-mode dual).
+config_yaml=${config_yaml:-"${REPO_ROOT}/examples/Franka/train_files/starvla_cotrain_franka_dual.yaml"}
+franka_data_root=${franka_data_root:-/gpfs/wangzixuan/Octopus/workspace/data/lerobot/v3}
+data_mix=${data_mix:-real_franka_dual_v3}
+# Leave empty to train the action head from scratch (action_dim=14 differs from
+# the single-arm pretrain, so the action head will not load anyway).
+pretrained_checkpoint=${pretrained_checkpoint:-}
+base_vlm=${base_vlm:-/gpfs/wangzixuan/Octopus/workspace/MODEL/Qwen3-VL-4B-Instruct}
+reload_modules=${reload_modules:-'qwen_vl_interface'}
+
+run_root_dir=${run_root_dir:-"${REPO_ROOT}/results/Checkpoints"}
+run_id=${run_id:-0615_franka_v3_dual_finetune_qwen3OFT}
+num_processes=${num_processes:-8}
+per_device_batch_size=${per_device_batch_size:-16}
+max_train_steps=${max_train_steps:-100000}
+save_interval=${save_interval:-10000}
+logging_frequency=${logging_frequency:-100}
+eval_interval=${eval_interval:-1000}
+
+if [[ ! -f "${config_yaml}" ]]; then
+  echo "Config yaml not found: ${config_yaml}" >&2
+  exit 1
+fi
+
+if [[ -n "${pretrained_checkpoint}" && ! -f "${pretrained_checkpoint}" ]]; then
+  echo "Pretrained checkpoint not found: ${pretrained_checkpoint}" >&2
+  exit 1
+fi
+
+output_dir="${run_root_dir}/${run_id}"
+mkdir -p "${output_dir}"
+cp "$0" "${output_dir}/"
+
+echo "============================================"
+echo "Conda env             : ${CONDA_ENV}"
+echo "Config yaml           : ${config_yaml}"
+echo "Base VLM              : ${base_vlm}"
+echo "Pretrained checkpoint : ${pretrained_checkpoint:-<none>}"
+echo "Franka data root      : ${franka_data_root}"
+echo "Data mix              : ${data_mix}"
+echo "Batch size / process  : ${per_device_batch_size}"
+echo "Num processes         : ${num_processes}"
+echo "Max train steps       : ${max_train_steps}"
+echo "Run ID                : ${run_id}"
+echo "============================================"
+
+launch_args=(
+  --config_file "${REPO_ROOT}/starVLA/config/deepseeds/deepspeed_zero2.yaml"
+  --num_processes "${num_processes}"
+  "${REPO_ROOT}/starVLA/training/train_starvla.py"
+  --config_yaml "${config_yaml}"
+  --framework.name QwenOFT
+  --framework.qwenvl.base_vlm "${base_vlm}"
+  --datasets.vla_data.data_root_dir "${franka_data_root}"
+  --datasets.vla_data.data_mix "${data_mix}"
+  --datasets.vla_data.per_device_batch_size "${per_device_batch_size}"
+  --datasets.vla_data.sequential_step_sampling false
+  --trainer.max_train_steps "${max_train_steps}"
+  --trainer.save_interval "${save_interval}"
+  --trainer.logging_frequency "${logging_frequency}"
+  --trainer.eval_interval "${eval_interval}"
+  --trainer.reload_modules "${reload_modules}"
+  --run_root_dir "${run_root_dir}"
+  --run_id "${run_id}"
+  --wandb_project franka_dual
+  --wandb_entity zwanggk
+)
+
+if [[ -n "${pretrained_checkpoint}" ]]; then
+  launch_args+=(--trainer.pretrained_checkpoint "${pretrained_checkpoint}")
+fi
+
+accelerate launch "${launch_args[@]}"

summary.jsonl

@@ -0,0 +1,10 @@
+{"steps": 10000}
+{"steps": 20000}
+{"steps": 30000}
+{"steps": 40000}
+{"steps": 50000}
+{"steps": 60000}
+{"steps": 70000}
+{"steps": 80000}
+{"steps": 90000}
+{"steps": 100000}