checkpoint:
  broadcast_via_filesystem: false
  dcp_async_mode_enabled: false
  enable_gcs_patch_in_boto3: true
  hf_export:
    enabled: false
    export_every_n: 1
    hf_repo_id: null
    upload_to_object_store:
      bucket: ''
      credentials: ''
      enabled: false
  jit:
    device: cuda
    dtype: bfloat16
    enabled: false
    input_shape: null
    strict: true
  keys_not_to_resume: []
  keys_to_skip_loading:
  - net_ema.
  load_ema_to_reg: false
  load_from_object_store:
    bucket: ''
    credentials: ''
    enabled: false
  load_path: /workspace/cosmos/checkpoints/Cosmos3-Nano-DCP
  load_training_state: false
  only_load_scheduler_state: false
  save_iter: 100
  save_to_object_store:
    bucket: ''
    credentials: ''
    enabled: false
  strict_resume: true
  type:
    _target_: cosmos_framework.checkpoint.dcp.DistributedCheckpointer
    callbacks: null
    disable_async: false
  verbose: true
data_setting:
  qwen_max_video_token_length: 8192
dataloader_train:
  _target_: cosmos_framework.data.vfm.joint_dataloader.PackingDataLoader
  audio_sample_rate: 48000
  dataloader:
    _target_: cosmos_framework.data.vfm.joint_dataloader.RankPartitionedDataLoader
    batch_size: 1
    datasets:
      dvrk_openh:
        dataset:
          _target_: cosmos3_nanoh_surgical.dataset.OpenHDVRKLeRobotDataset
          apply_cosmos_transform: true
          cfg_dropout_rate: 0.1
          chunk_length: 16
          fps: null
          load_video: true
          local_root: /workspace/cosmos-3-NanoH-Surgical/data/openh_snapshot
          manifest_path: /workspace/cosmos-3-NanoH-Surgical/artifacts/manifests/openh_dvrk_manifest.train.json
          max_action_dim: 64
          max_roots: 0
          max_windows_per_root: 0
          mode: all
          mode_weights:
            forward_dynamics: 1.0
            inverse_dynamics: 1.0
            policy: 1.0
          normalization_path: /workspace/cosmos-3-NanoH-Surgical/artifacts/manifests/dvrk_action_stats.json
          normalize_action: true
          resolution: '256'
          shard_rank: 0
          shard_world_size: 1
          tokenizer_config:
            _target_: cosmos_framework.configs.base.defaults.vlm.create_qwen2_tokenizer_with_download
            config_variant: hf
            pretrained_model_name: Qwen/Qwen3-VL-8B-Instruct
          video_key: auto
        ratio: 1
    in_order: true
    num_workers: 4
    persistent_workers: true
    pin_memory: true
    prefetch_factor: 4
    sampler: null
  dataset_name: dvrk_openh
  lookahead_limit: 10
  max_samples_per_batch: null
  max_sequence_length: 32768
  patch_spatial: 2
  sound_latent_fps: 0
  tokenizer_spatial_compression_factor: 16
  tokenizer_temporal_compression_factor: 4
dataloader_val: null
defaults:
- _self_
- model: mot_fsdp
- data_train: null
- data_val: null
- optimizer: adamw
- scheduler: warmup_cosine_lr
- checkpoint: s3
- callbacks:
  - basic
  - optimization
  - job_monitor
  - generation
- ema: power
- tokenizer: wan2pt2_tokenizer
- sound_tokenizer: null
- cluster: default
- vlm_config: null
- ckpt_type: dcp
- experiment: null
job:
  cluster: null
  group: dvrk-openh
  name: cosmos3_nanoh_dvrk_all_objective_chunk16_singlecam
  project: cosmos3-nanoh-surgical
  wandb_mode: online
model:
  _recursive_: false
  _target_: cosmos_framework.model.vfm.omni_mot_model.OmniMoTModel
  config:
    action_gen: true
    activation_checkpointing:
      determinism_check: default
      mode: full
      preserve_rng_state: true
      save_ops_regex:
      - fmha
    causal_training_strategy: none
    compile:
      compile_dynamic: true
      compiled_region: language
      coordinate_descent_tuning: false
      enabled: false
      max_autotune_pointwise: false
      use_cuda_graphs: false
    diffusion_expert_config:
      base_fps: 24
      enable_fps_modulation: true
      load_weights_from_pretrained: true
      max_vae_latent_side_after_patchify: 20
      patch_spatial: 2
      position_embedding_type: unified_3d_mrope
      rope_h_extrapolation_ratio: 1.0
      rope_t_extrapolation_ratio: 1.0
      rope_w_extrapolation_ratio: 1.0
      timestep_range: 1.0
      unified_3d_mrope_reset_spatial_ids: true
      unified_3d_mrope_temporal_modality_margin: 15000
    ema:
      enabled: false
      iteration_shift: 0
      rate: 0.1
    fixed_step_sampler_config: null
    input_caption_key: ai_caption
    input_image_key: images
    input_video_key: video
    joint_attn_implementation: flex
    latent_downsample_factor: 16
    lbl:
      coeff_gen: null
      coeff_und: null
      method: local
    log_enc_time_every_n: 100
    lora_alpha: 32
    lora_enabled: false
    lora_rank: 16
    lora_target_modules: q_proj_moe_gen,k_proj_moe_gen,v_proj_moe_gen,o_proj_moe_gen
    max_action_dim: 64
    max_num_tokens_after_packing: 32768
    natten_parameter_list: null
    net: null
    num_embodiment_domains: 32
    parallelism:
      cfg_parallel_shard_degree: 1
      context_parallel_shard_degree: 1
      data_parallel_replicate_degree: 1
      data_parallel_shard_degree: -1
      enable_inference_mode: false
      fsdp_master_dtype: float32
    precision: bfloat16
    rectified_flow_inference_config:
      num_train_timesteps: 1000
      scheduler_type: unipc
      shift: 1
      use_dynamic_shifting: false
    rectified_flow_training_config:
      action_loss_weight: 10.0
      high_sigma_ratio: 0.05
      high_sigma_timesteps_max: 1000
      high_sigma_timesteps_min: 995
      image_loss_scale: 1.0
      independent_action_schedule: false
      independent_sound_schedule: false
      loss_scale: 1.0
      normalize_loss_by_active: false
      shift:
        '256': 3
        '480': 5
        '720': 10
      shift_action: null
      shift_sound: null
      sound_loss_scale: null
      train_time_action_distribution: logitnormal
      train_time_image_distribution: logitnormal
      train_time_sound_distribution: logitnormal
      train_time_video_distribution: waver
      train_time_weight: uniform
      use_discrete_rf: false
      use_dynamic_shift: false
      use_high_sigma_strategy: false
      use_high_sigma_strategy_action: false
      use_high_sigma_strategy_sound: false
    resolution: '256'
    sound_dim: null
    sound_gen: false
    sound_latent_fps: 25
    sound_tokenizer: null
    state_ch: 48
    state_t: 300
    tokenizer:
      _target_: cosmos_framework.model.vfm.tokenizers.wan2pt2_vae_4x16x16.Wan2pt2VAEInterface
      bucket_name: ''
      chunk_duration: 93
      encode_bucket_multiple: null
      encode_chunk_frames:
        '256': 68
        '480': 24
        '720': 12
      encode_exact_durations: null
      keep_decoder_cache: false
      object_store_credential_path_pretrained: ''
      spatial_compression_factor: 16
      temporal_compression_factor: 4
      temporal_window: null
      use_streaming_encode: false
      vae_path: /workspace/cosmos/checkpoints/wan22_vae/Wan2.2_VAE.pth
    video_temporal_causal: false
    vision_gen: true
    vlm_config:
      layer_module: Qwen2MoTDecoderLayer
      model_instance:
        _target_: cosmos_framework.model.vfm.mot.unified_mot.Qwen3VLTextForCausalLM
        config:
          _target_: cosmos_framework.configs.base.defaults.vlm.create_vlm_config
          base_config:
            _target_: cosmos_framework.model.vfm.mot.unified_mot.Qwen3VLMoTConfig.from_json_file
            json_file: cosmos_framework/model/vfm/vlm/qwen3_vl/configs/Qwen3-VL-8B-Instruct.json
          freeze_und: false
          layer_module: MoTDecoderLayer
          qk_norm_for_text: true
          tie_word_embeddings: true
      model_name: Qwen/Qwen3-VL-8B-Instruct
      pretrained_weights:
        backbone_path: s3://bucket0/cosmos3/pretrained/huggingface/Qwen/Qwen3-VL-8B-Instruct/
        checkpoint_format: null
        credentials_path: ''
        enable_gcs_patch_in_boto3: true
        enabled: false
      qk_norm: false
      safetensors_path: ''
      tie_word_embeddings: false
      tokenizer:
        _target_: cosmos_framework.configs.base.defaults.vlm.create_qwen2_tokenizer_with_download
        config_variant: hf
        pretrained_model_name: Qwen/Qwen3-VL-8B-Instruct
      use_system_prompt: false
model_parallel:
  _cpu_offloading_context: null
  async_tensor_model_parallel_allreduce: false
  autocast_dtype: torch.float32
  barrier_with_L1_time: true
  batch_p2p_comm: true
  batch_p2p_sync: true
  bf16: false
  context_parallel_size: 1
  cpu_offloading: false
  cpu_offloading_activations: true
  cpu_offloading_double_buffering: false
  cpu_offloading_num_layers: 0
  cpu_offloading_weights: false
  cross_entropy_fusion_impl: native
  cross_entropy_loss_fusion: false
  deallocate_pipeline_outputs: false
  defer_embedding_wgrad_compute: false
  delay_wgrad_compute: false
  deterministic_mode: false
  enable_autocast: false
  ep_overlap_early_attn_memory_release: false
  expert_model_parallel_size: 1
  expert_tensor_parallel_size: 1
  finalize_model_grads_func: null
  fp16: false
  grad_scale_func: null
  grad_sync_func: null
  gradient_accumulation_fusion: false
  hierarchical_context_parallel_sizes: null
  microbatch_group_size_per_vp_stage: 1
  moe_extended_tp: false
  no_sync_func: null
  num_microbatches_with_partial_activation_checkpoints: null
  overlap_moe_expert_parallel_comm: false
  overlap_p2p_comm: false
  overlap_p2p_comm_warmup_flush: false
  param_sync_func: null
  params_dtype: torch.float32
  perform_initialization: true
  pipeline_dtype: null
  pipeline_model_parallel_comm_backend: null
  pipeline_model_parallel_size: 1
  sequence_parallel: false
  tensor_model_parallel_size: 1
  timers: null
  tp_comm_atomic_ag: false
  tp_comm_atomic_rs: false
  tp_comm_bootstrap_backend: nccl
  tp_comm_bulk_dgrad: true
  tp_comm_bulk_wgrad: true
  tp_comm_overlap: false
  tp_comm_overlap_ag: true
  tp_comm_overlap_disable_fc1: false
  tp_comm_overlap_disable_qkv: false
  tp_comm_overlap_rs: true
  tp_comm_overlap_rs_dgrad: false
  tp_comm_split_ag: true
  tp_comm_split_rs: true
  use_cpu_initialization: false
  use_ring_exchange_p2p: false
  use_te_rng_tracker: false
  variable_seq_lengths: false
  virtual_pipeline_model_parallel_size: null
  wgrad_deferral_limit: 0
optimizer:
  _target_: cosmos_framework.utils.vfm.optimizer.build_optimizer
  betas:
  - 0.9
  - 0.95
  disable_weight_decay_for_1d_params: false
  eps: 1.0e-06
  fused: true
  keys_to_select:
  - moe_gen
  - time_embedder
  - vae2llm
  - llm2vae
  - action2llm
  - llm2action
  - action_modality_embed
  - q_proj_moe_gen
  - k_proj_moe_gen
  - v_proj_moe_gen
  - o_proj_moe_gen
  lr: 2.0e-05
  lr_multipliers: {}
  model: null
  optimizer_type: AdamW
  weight_decay: 0
scheduler:
  _target_: cosmos_framework.utils.vfm.optimizer.build_lr_scheduler
  cycle_lengths:
  - 1000
  f_max:
  - 1.0
  f_min:
  - 0.0
  f_start:
  - 0.0
  lr_scheduler_type: LambdaCosine
  optimizer: null
  verbosity_interval: 0
  warm_up_steps:
  - 50
trainer:
  callbacks:
    compile_tokenizer:
      _target_: cosmos_framework.callbacks.compile_tokenizer.CompileTokenizer
      compile_after_iterations: 3
      enabled: false
      warmup_resolutions: null
    dataloader_speed:
      every_n: 100
      save_s3: false
      step_size: 1
    device_monitor:
      _target_: cosmos_framework.callbacks.device_monitor.DeviceMonitor
      every_n: 100
      log_memory_detail: true
      save_s3: false
      step_size: 1
      upload_every_n_mul: 5
    every_n_sample_ema:
      _target_: cosmos_framework.callbacks.every_n_draw_sample.EveryNDrawSample
      do_x0_prediction: false
      every_n: 100
      fps: 16
      guidance:
      - 0.0
      - 3.0
      - 7.0
      is_ema: true
      n_sample_to_save: 128
      n_sigmas_for_x0_prediction: 4
      n_viz_sample: 2
      num_sampling_step: 35
      prompt_type: t5_xxl
      run_at_start: false
      save_local: false
      save_s3: false
      step_size: 1
      use_negative_prompt: false
    every_n_sample_reg:
      _target_: cosmos_framework.callbacks.every_n_draw_sample.EveryNDrawSample
      do_x0_prediction: false
      every_n: 100
      fps: 16
      guidance:
      - 3.0
      is_ema: false
      n_sample_to_save: 8
      n_sigmas_for_x0_prediction: 4
      n_viz_sample: 1
      num_sampling_step: 16
      prompt_type: t5_xxl
      run_at_start: false
      save_local: true
      save_s3: false
      step_size: 1
      use_negative_prompt: false
    expert_heatmap:
      _target_: cosmos_framework.callbacks.expert_heatmap.ExpertHeatmap
      every_n: 1000
    grad_clip:
      _target_: cosmos_framework.callbacks.grad_clip.GradClip
      clip_norm: 0.1
      force_finite: true
      track_per_modality: true
    heart_beat:
      _target_: cosmos_framework.callbacks.heart_beat.HeartBeat
      every_n: 100
      save_s3: false
      step_size: 1
      update_interval_in_minute: 10
    iter_speed:
      _target_: cosmos_framework.callbacks.iter_speed.IterSpeed
      every_n: 1
      hit_thres: 50
      save_s3: false
      save_s3_every_log_n: 500
    load_pretrained:
      _target_: cosmos_framework.callbacks.load_pretrained.LoadPretrained
      config: null
      trainer: null
    low_precision:
      _target_: cosmos_framework.utils.callback.LowPrecisionCallback
      config: null
      trainer: null
      update_iter: 1
    manual_gc:
      _target_: cosmos_framework.callbacks.manual_gc.ManualGarbageCollection
      every_n: 5
      gc_level: 1
      warm_up: 1
    mfu:
      _target_: cosmos_framework.callbacks.mfu.MFUCallback
      backwardpass_ratio: 2.0
      every_n: 1
      grad_accum_iter: 2
      hit_thres: 5
      include_padding: true
      include_vae_encoder: true
    moe_specialization:
      _target_: cosmos_framework.callbacks.moe_specialization_callback.MoESpecializationCallback
      every_n: 250
    moe_stability:
      _target_: cosmos_framework.callbacks.moe_stability_callback.MoEStabilityCallback
      every_n: 250
    norm_monitor:
      _target_: cosmos_framework.callbacks.norm_monitor.NormMonitor
      every_n: 100
      layer_norm_only: false
      log_stat_wandb: false
      model_key: null
      save_s3: false
      step_size: 1
      track_activations: false
    ofu:
      _target_: cosmos_framework.callbacks.ofu.OFUCallback
      every_n: 1
      hit_thres: 5
    param_count:
      _target_: cosmos_framework.callbacks.param_count.ParamCount
      save_s3: false
    sequence_packing_padding:
      _target_: cosmos_framework.callbacks.sequence_packing_padding.SequencePackingPadding
      every_n: 50
    sigma_loss_analysis:
      _target_: cosmos_framework.callbacks.sigma_loss_analysis.SigmaLossAnalysis
      every_n: 250
      every_n_viz: 250
      save_s3: false
    skip_nan_step:
      _target_: cosmos_framework.callbacks.skip_nan_step.SkipNaNStep
      max_consecutive_nan: 20
    termination_signal_checkpoint:
      _target_: cosmos_framework.callbacks.termination_signal_checkpoint.TerminationSignalCheckpoint
      min_save_fraction: 0.3333333333333333
    training_stats:
      log_freq: 50
    wandb:
      _target_: cosmos_framework.utils.callback.WandBCallback
      config: null
      trainer: null
    wandb_2x:
      _target_: cosmos_framework.callbacks.wandb_log.WandbCallback
      logging_iter_multipler: 2
      save_logging_iter_multipler: 1
      save_s3: false
    wandb_val:
      _target_: cosmos_framework.callbacks.wandb_log_eval.WandbCallback
      save_s3: false
  compile_config:
    recompile_limit: 8
    use_duck_shape: false
  cudnn:
    benchmark: true
    deterministic: false
  ddp:
    broadcast_buffers: true
    find_unused_parameters: false
    static_graph: true
  distributed_parallelism: fsdp
  grad_accum_iter: 2
  grad_scaler_args:
    enabled: false
  logging_iter: 1
  max_iter: 500
  max_val_iter: null
  memory_format: torch.preserve_format
  profiling:
    enable_memory_snapshot: false
    enable_nsys: false
    enable_profiling: false
    profile_freq: 1
    profile_memory: false
    profile_warmup: 3
    record_shape: false
    save_s3: false
    target_ranks:
    - 0
    - 1
    - 2
    - 3
    - 4
    - 5
    - 6
    - 7
    with_modules: true
    with_stack: true
  run_validation: false
  run_validation_on_start: false
  save_zero_checkpoint: false
  seed: 42
  straggler_detection:
    analyze_backward: true
    analyze_dataloading: true
    analyze_forward: true
    analyze_optimizer: true
    enabled: false
    max_diff: 2.0
    profile_freq: 1
    raise_error: true
    report_freq: 100
    save_s3: false
  timeout_period: 999999999
  type: cosmos_framework.trainer.ImaginaireTrainer
  validation_iter: 100
upload_reproducible_setup: false