File size: 16,433 Bytes
a456468 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | * model: {'config': {'tokenizer': {'bucket_name': '', 'object_store_credential_path_pretrained': '', 'vae_path': '/workspace/cosmos/checkpoints/wan22_vae/Wan2.2_VAE.pth', 'chunk_duration': 93, 'keep_decoder_cache': False, 'use_streaming_encode': False, 'encode_chunk_frames': {'256': 68, '480': 24, '720': 12}, 'encode_exact_durations': None, 'spatial_compression_factor': 16, 'temporal_compression_factor': 4, 'temporal_window': None, 'encode_bucket_multiple': None, '_target_': <class 'cosmos_framework.model.vfm.tokenizers.wan2pt2_vae_4x16x16.Wan2pt2VAEInterface'>}, 'net': None, 'ema': {'enabled': False, 'rate': 0.1, 'iteration_shift': 0}, 'parallelism': {'data_parallel_shard_degree': -1, 'data_parallel_replicate_degree': 1, 'context_parallel_shard_degree': 1, 'cfg_parallel_shard_degree': 1, 'enable_inference_mode': False, 'fsdp_master_dtype': 'float32'}, 'compile': {'enabled': False, 'compiled_region': 'language', 'compile_dynamic': True, 'use_cuda_graphs': False, 'max_autotune_pointwise': False, 'coordinate_descent_tuning': False}, 'activation_checkpointing': {'mode': 'full', 'save_ops_regex': ['fmha'], 'preserve_rng_state': True, 'determinism_check': 'default'}, 'precision': 'bfloat16', 'lora_enabled': False, 'lora_rank': 16, 'lora_alpha': 32, 'lora_target_modules': 'q_proj_moe_gen,k_proj_moe_gen,v_proj_moe_gen,o_proj_moe_gen', 'rectified_flow_training_config': {'shift': {'256': 3, '480': 5, '720': 10}, 'use_dynamic_shift': False, 'train_time_image_distribution': 'logitnormal', 'train_time_video_distribution': 'waver', 'train_time_action_distribution': 'logitnormal', 'train_time_sound_distribution': 'logitnormal', 'train_time_weight': 'uniform', 'loss_scale': 1.0, 'image_loss_scale': 1.0, 'sound_loss_scale': None, 'use_high_sigma_strategy': False, 'high_sigma_ratio': 0.05, 'high_sigma_timesteps_min': 995, 'high_sigma_timesteps_max': 1000, 'use_discrete_rf': False, 'action_loss_weight': 10.0, 'independent_action_schedule': False, 'shift_action': None, 'use_high_sigma_strategy_action': False, 'independent_sound_schedule': False, 'shift_sound': None, 'use_high_sigma_strategy_sound': False, 'normalize_loss_by_active': False}, 'rectified_flow_inference_config': {'scheduler_type': 'unipc', 'num_train_timesteps': 1000, 'shift': 1, 'use_dynamic_shifting': False}, 'fixed_step_sampler_config': None, 'vlm_config': {'model_name': 'Qwen/Qwen3-VL-8B-Instruct', 'safetensors_path': '', 'pretrained_weights': {'enabled': False, 'backbone_path': 's3://bucket0/cosmos3/pretrained/huggingface/Qwen/Qwen3-VL-8B-Instruct/', 'credentials_path': '', 'enable_gcs_patch_in_boto3': True, 'checkpoint_format': None}, 'model_instance': {'config': {'base_config': {'json_file': 'cosmos_framework/model/vfm/vlm/qwen3_vl/configs/Qwen3-VL-8B-Instruct.json', '_target_': <bound method _MoTConfigBase.from_json_file of <class 'cosmos_framework.model.vfm.mot.unified_mot.Qwen3VLMoTConfig'>>}, 'freeze_und': False, 'layer_module': 'MoTDecoderLayer', 'qk_norm_for_text': True, 'tie_word_embeddings': True, '_target_': <function create_vlm_config at 0x7fc4c2ecc860>}, '_target_': <class 'cosmos_framework.model.vfm.mot.unified_mot.Qwen3VLTextForCausalLM'>}, 'tokenizer': {'config_variant': 'hf', 'pretrained_model_name': 'Qwen/Qwen3-VL-8B-Instruct', '_target_': <function create_qwen2_tokenizer_with_download at 0x7fc4608ffba0>}, 'layer_module': 'Qwen2MoTDecoderLayer', 'qk_norm': False, 'tie_word_embeddings': False, 'use_system_prompt': False}, 'diffusion_expert_config': {'timestep_range': 1.0, 'load_weights_from_pretrained': True, 'patch_spatial': 2, 'max_vae_latent_side_after_patchify': 20, 'position_embedding_type': 'unified_3d_mrope', 'rope_h_extrapolation_ratio': 1.0, 'rope_w_extrapolation_ratio': 1.0, 'rope_t_extrapolation_ratio': 1.0, 'enable_fps_modulation': True, 'base_fps': 24, 'unified_3d_mrope_reset_spatial_ids': True, 'unified_3d_mrope_temporal_modality_margin': 15000}, 'input_video_key': 'video', 'input_image_key': 'images', 'input_caption_key': 'ai_caption', 'state_ch': 48, 'state_t': 300, 'latent_downsample_factor': 16, 'resolution': '256', 'max_num_tokens_after_packing': 32768, 'joint_attn_implementation': 'flex', 'natten_parameter_list': None, 'video_temporal_causal': False, 'causal_training_strategy': 'none', 'lbl': {'method': 'local', 'coeff_und': None, 'coeff_gen': None}, 'vision_gen': True, 'action_gen': True, 'max_action_dim': 64, 'num_embodiment_domains': 32, 'sound_gen': False, 'sound_tokenizer': None, 'sound_dim': None, 'sound_latent_fps': 25, 'log_enc_time_every_n': 100}, '_recursive_': False, '_target_': <class 'cosmos_framework.model.vfm.omni_mot_model.OmniMoTModel'>}
* optimizer: {'model': None, 'optimizer_type': 'AdamW', 'lr': 2e-05, 'weight_decay': 0, 'betas': [0.9, 0.95], 'eps': 1e-06, 'fused': True, 'keys_to_select': ['moe_gen', 'time_embedder', 'vae2llm', 'llm2vae', 'action2llm', 'llm2action', 'action_modality_embed', 'q_proj_moe_gen', 'k_proj_moe_gen', 'v_proj_moe_gen', 'o_proj_moe_gen'], 'lr_multipliers': {}, 'disable_weight_decay_for_1d_params': False, '_target_': <function build_optimizer at 0x7fc45f832700>}
* scheduler: {'optimizer': None, 'lr_scheduler_type': 'LambdaCosine', 'warm_up_steps': [50], 'cycle_lengths': [1000], 'f_start': [0.0], 'f_max': [1.0], 'f_min': [0.0], 'verbosity_interval': 0, '_target_': <function build_lr_scheduler at 0x7fc45f832ca0>}
* dataloader_train: {'max_sequence_length': 32768, 'max_samples_per_batch': None, 'sound_latent_fps': 0, 'audio_sample_rate': 48000, 'dataset_name': 'dvrk_openh', 'lookahead_limit': 10, 'patch_spatial': 2, 'tokenizer_spatial_compression_factor': 16, 'tokenizer_temporal_compression_factor': 4, 'dataloader': {'batch_size': 1, 'in_order': True, 'num_workers': 4, 'persistent_workers': True, 'pin_memory': True, 'prefetch_factor': 4, 'sampler': None, 'datasets': {'dvrk_openh': {'ratio': 1, 'dataset': {'fps': None, 'chunk_length': 16, 'mode': 'all', 'resolution': '256', 'video_key': 'auto', 'normalization_path': '/workspace/cosmos-3-NanoH-Surgical/artifacts/manifests/dvrk_action_stats.json', 'normalize_action': True, 'mode_weights': {'forward_dynamics': 1.0, 'inverse_dynamics': 1.0, 'policy': 1.0}, 'load_video': True, 'apply_cosmos_transform': True, 'tokenizer_config': {'config_variant': 'hf', 'pretrained_model_name': 'Qwen/Qwen3-VL-8B-Instruct', '_target_': <function create_qwen2_tokenizer_with_download at 0x7fc4608ffba0>}, 'cfg_dropout_rate': 0.1, 'max_action_dim': 64, 'max_roots': 0, 'max_windows_per_root': 0, 'shard_world_size': 1, 'shard_rank': 0, 'local_root': '/workspace/cosmos-3-NanoH-Surgical/data/openh_snapshot', 'manifest_path': '/workspace/cosmos-3-NanoH-Surgical/artifacts/manifests/openh_dvrk_manifest.train.json', '_target_': <class 'cosmos3_nanoh_surgical.dataset.OpenHDVRKLeRobotDataset'>}}}, '_target_': <class 'cosmos_framework.data.vfm.joint_dataloader.RankPartitionedDataLoader'>}, '_target_': <class 'cosmos_framework.data.vfm.joint_dataloader.PackingDataLoader'>}
* dataloader_val: None
* job:
* project: cosmos3-nanoh-surgical
* group: dvrk-openh
* name: cosmos3_nanoh_dvrk_all_objective_chunk16_singlecam
* wandb_mode: online
* cluster: None
* trainer:
* type: <class 'cosmos_framework.trainer.ImaginaireTrainer'>
* callbacks: {'iter_speed': {'hit_thres': 50, 'save_s3': False, 'save_s3_every_log_n': 500, 'every_n': 1, '_target_': <class 'cosmos_framework.callbacks.iter_speed.IterSpeed'>}, 'manual_gc': {'warm_up': 1, 'gc_level': 1, 'every_n': 5, '_target_': <class 'cosmos_framework.callbacks.manual_gc.ManualGarbageCollection'>}, 'wandb': {'config': None, 'trainer': None, '_target_': <class 'cosmos_framework.utils.callback.WandBCallback'>}, 'wandb_2x': {'logging_iter_multipler': 2, 'save_logging_iter_multipler': 1, 'save_s3': False, '_target_': <class 'cosmos_framework.callbacks.wandb_log.WandbCallback'>}, 'param_count': {'save_s3': False, '_target_': <class 'cosmos_framework.callbacks.param_count.ParamCount'>}, 'wandb_val': {'save_s3': False, '_target_': <class 'cosmos_framework.callbacks.wandb_log_eval.WandbCallback'>}, 'moe_stability': {'every_n': 250, '_target_': <class 'cosmos_framework.callbacks.moe_stability_callback.MoEStabilityCallback'>}, 'moe_specialization': {'every_n': 250, '_target_': <class 'cosmos_framework.callbacks.moe_specialization_callback.MoESpecializationCallback'>}, 'expert_heatmap': {'every_n': 1000, '_target_': <class 'cosmos_framework.callbacks.expert_heatmap.ExpertHeatmap'>}, 'load_pretrained': {'config': None, 'trainer': None, '_target_': <class 'cosmos_framework.callbacks.load_pretrained.LoadPretrained'>}, 'compile_tokenizer': {'enabled': False, 'compile_after_iterations': 3, 'warmup_resolutions': None, '_target_': <class 'cosmos_framework.callbacks.compile_tokenizer.CompileTokenizer'>}, 'norm_monitor': {'every_n': 100, 'step_size': 1, 'layer_norm_only': False, 'model_key': None, 'log_stat_wandb': False, 'save_s3': False, 'track_activations': False, '_target_': <class 'cosmos_framework.callbacks.norm_monitor.NormMonitor'>}, 'sigma_loss_analysis': {'every_n': 250, 'every_n_viz': 250, 'save_s3': False, '_target_': <class 'cosmos_framework.callbacks.sigma_loss_analysis.SigmaLossAnalysis'>}, 'sequence_packing_padding': {'every_n': 50, '_target_': <class 'cosmos_framework.callbacks.sequence_packing_padding.SequencePackingPadding'>}, 'mfu': {'backwardpass_ratio': 2.0, 'hit_thres': 5, 'include_vae_encoder': True, 'include_padding': True, 'grad_accum_iter': 2, 'every_n': 1, '_target_': <class 'cosmos_framework.callbacks.mfu.MFUCallback'>}, 'ofu': {'hit_thres': 5, 'every_n': 1, '_target_': <class 'cosmos_framework.callbacks.ofu.OFUCallback'>}, 'skip_nan_step': {'max_consecutive_nan': 20, '_target_': <class 'cosmos_framework.callbacks.skip_nan_step.SkipNaNStep'>}, 'grad_clip': {'clip_norm': 0.1, 'force_finite': True, 'track_per_modality': True, '_target_': <class 'cosmos_framework.callbacks.grad_clip.GradClip'>}, 'low_precision': {'update_iter': 1, 'config': None, 'trainer': None, '_target_': <class 'cosmos_framework.utils.callback.LowPrecisionCallback'>}, 'heart_beat': {'step_size': 1, 'update_interval_in_minute': 10, 'save_s3': False, 'every_n': 100, '_target_': <class 'cosmos_framework.callbacks.heart_beat.HeartBeat'>}, 'device_monitor': {'every_n': 100, 'step_size': 1, 'save_s3': False, 'upload_every_n_mul': 5, 'log_memory_detail': True, '_target_': <class 'cosmos_framework.callbacks.device_monitor.DeviceMonitor'>}, 'termination_signal_checkpoint': {'min_save_fraction': 0.3333333333333333, '_target_': <class 'cosmos_framework.callbacks.termination_signal_checkpoint.TerminationSignalCheckpoint'>}, 'every_n_sample_reg': {'step_size': 1, 'n_viz_sample': 1, 'n_sample_to_save': 8, 'num_sampling_step': 16, 'guidance': [3.0], 'do_x0_prediction': False, 'n_sigmas_for_x0_prediction': 4, 'save_s3': False, 'save_local': True, 'is_ema': False, 'use_negative_prompt': False, 'prompt_type': 't5_xxl', 'fps': 16, 'run_at_start': False, 'every_n': 100, '_target_': <class 'cosmos_framework.callbacks.every_n_draw_sample.EveryNDrawSample'>}, 'every_n_sample_ema': {'step_size': 1, 'n_viz_sample': 2, 'n_sample_to_save': 128, 'num_sampling_step': 35, 'guidance': [0.0, 3.0, 7.0], 'do_x0_prediction': False, 'n_sigmas_for_x0_prediction': 4, 'save_s3': False, 'save_local': False, 'is_ema': True, 'use_negative_prompt': False, 'prompt_type': 't5_xxl', 'fps': 16, 'run_at_start': False, 'every_n': 100, '_target_': <class 'cosmos_framework.callbacks.every_n_draw_sample.EveryNDrawSample'>}, 'dataloader_speed': {'every_n': 100, 'save_s3': False, 'step_size': 1}, 'training_stats': {'log_freq': 50}}
* distributed_parallelism: fsdp
* ddp:
* find_unused_parameters: False
* static_graph: True
* broadcast_buffers: True
* cudnn:
* deterministic: False
* benchmark: True
* seed: 42
* grad_scaler_args: {'enabled': False}
* max_iter: 500
* max_val_iter: None
* logging_iter: 1
* run_validation: False
* validation_iter: 100
* run_validation_on_start: False
* timeout_period: 999999999
* memory_format: torch.preserve_format
* grad_accum_iter: 2
* straggler_detection:
* enabled: False
* report_freq: 100
* profile_freq: 1
* max_diff: 2.0
* raise_error: True
* analyze_forward: True
* analyze_backward: True
* analyze_optimizer: True
* analyze_dataloading: True
* save_s3: False
* profiling:
* enable_profiling: False
* enable_nsys: False
* enable_memory_snapshot: False
* save_s3: False
* profile_freq: 1
* profile_warmup: 3
* target_ranks: [0, 1, 2, 3, 4, 5, 6, 7]
* record_shape: False
* profile_memory: False
* with_stack: True
* with_modules: True
* compile_config:
* recompile_limit: 8
* use_duck_shape: False
* save_zero_checkpoint: False
* model_parallel: ModelParallelConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=False, fp16=False, bf16=False, params_dtype=torch.float32, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.float32, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=False, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, cross_entropy_fusion_impl='native', tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', overlap_moe_expert_parallel_comm=False, delay_wgrad_compute=False, ep_overlap_early_attn_memory_release=False, pipeline_dtype=None, variable_seq_lengths=False, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=False, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=False, cpu_offloading_double_buffering=False, barrier_with_L1_time=True)
* checkpoint:
* type: {'callbacks': None, 'disable_async': False, '_target_': <class 'cosmos_framework.checkpoint.dcp.DistributedCheckpointer'>}
* dcp_async_mode_enabled: False
* save_to_object_store:
* enabled: False
* credentials:
* bucket:
* save_iter: 100
* strict_resume: True
* load_from_object_store:
* enabled: False
* credentials:
* bucket:
* load_path: /workspace/cosmos/checkpoints/Cosmos3-Nano-DCP
* load_training_state: False
* only_load_scheduler_state: False
* keys_to_skip_loading: ['net_ema.']
* jit:
* enabled: False
* input_shape: None
* device: cuda
* dtype: bfloat16
* strict: True
* verbose: True
* keys_not_to_resume: []
* broadcast_via_filesystem: False
* load_ema_to_reg: False
* enable_gcs_patch_in_boto3: True
* hf_export:
* enabled: False
* hf_repo_id: None
* upload_to_object_store:
* enabled: False
* credentials:
* bucket:
* export_every_n: 1
* upload_reproducible_setup: False
* data_setting:
* qwen_max_video_token_length: 8192
* defaults: ['_self_', {'model': 'mot_fsdp'}, {'data_train': None}, {'data_val': None}, {'optimizer': 'adamw'}, {'scheduler': 'warmup_cosine_lr'}, {'checkpoint': 's3'}, {'callbacks': ['basic', 'optimization', 'job_monitor', 'generation']}, {'ema': 'power'}, {'tokenizer': 'wan2pt2_tokenizer'}, {'sound_tokenizer': None}, {'cluster': 'default'}, {'vlm_config': None}, {'ckpt_type': 'dcp'}, {'experiment': None}]
/workspace/cosmos-3-NanoH-Surgical/outputs/cosmos3-nanoh-surgical/dvrk-openh/cosmos3_nanoh_dvrk_all_objective_chunk16_singlecam/config.dryrun.txt
|