Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,41 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__368_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__269_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__251_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__263_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__26_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__221_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__2_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__154_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__17_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__337_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__133_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__508_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__40_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__372_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__289_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__496_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__385_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__180_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__196_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__362_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__64_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__296_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__73_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__277_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__3_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__225_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__55_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__229_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__404_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__209_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__373_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__51_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__472_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__70_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__311_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__257_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__50_0.distcp filter=lfs diff=lfs merge=lfs -text
+megatron/iter_0023842/__468_0.distcp filter=lfs diff=lfs merge=lfs -text

V6_UPLOAD_MANIFEST.json

@@ -0,0 +1,22 @@
+{
+  "chat_model": false,
+  "hf_export": {
+    "apparent_bytes": 1517223216,
+    "file_count": 6,
+    "parameter_count": 756819456,
+    "tensor_count": 218,
+    "weight_dtype": "bfloat16"
+  },
+  "megatron_checkpoint": {
+    "apparent_bytes": 10600000875,
+    "file_count": 517
+  },
+  "model": "d24-v6",
+  "repo_id": "sfanm/d24-v6-midtrain",
+  "schema_version": 1,
+  "source_checkpoint": "/eagle/CausalAlign/yudas/unified-training/checkpoints/midtrain-d24-100b-olmo3-only-wsd/iter_0023842",
+  "source_hf_export": "/eagle/CausalAlign/yudas/unified-training/models/d24-v6/midtrain-hf",
+  "stage": "midtrain",
+  "terminal_iteration": 23842,
+  "train_tokens": 100000595968
+}

config.json

@@ -0,0 +1,33 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 50256,
+  "dtype": "bfloat16",
+  "eos_token_id": 50256,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 12,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.3.0",
+  "use_cache": true,
+  "vocab_size": 50304,
+  "torch_dtype": "bfloat16"
+}

export_manifest.json

@@ -0,0 +1,11 @@
+{
+  "source_checkpoint": "/eagle/CausalAlign/yudas/unified-training/checkpoints/midtrain-d24-100b-olmo3-only-wsd/iter_0023842",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "parameter_count": 756819456,
+  "tensor_count": 218,
+  "weight_dtype": "bfloat16",
+  "model_max_length": 2048,
+  "simple_chatml": false
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.3.0",
+  "use_cache": true
+}

megatron/iter_0023842/__133_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bd9e8d1b75e2bbad781c9c6b91a4b54281c950f9b69cfd4d654835d88c3ea24
+size 17747300

megatron/iter_0023842/__154_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c0dad150935aa2beb000a756cb285a9fa08a2aa32b2eb799281ab3eaab3d122
+size 17747300

megatron/iter_0023842/__17_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be4f7221f622c0d68f600b740a7252afbd440159bfe9318ac99dfb2ca1cfd87b
+size 31909274

megatron/iter_0023842/__180_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33579d5fdf02b88d5adaad96928fe7b78a6c2eef90d6ef4655b9bf88b00c6a4d
+size 17742651

megatron/iter_0023842/__196_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5187dc05749755a498b10fd2b8a47e51ddbdb17c7b118ea959c544b502ffbee
+size 17742651

megatron/iter_0023842/__209_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5af3889d27bf4d48ff905498848e1276b8bc9fd6fd0083e3580996c118b65728
+size 17751921

megatron/iter_0023842/__221_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b236b1bc62ff892df82984812725f70f9c6ab1aa98c67faeebfe4baaa96963d6
+size 17742651

megatron/iter_0023842/__225_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54ddd6deb12e25b18714026e050fbbb85740dd43924d2067f948fed98c93201a
+size 17742651

megatron/iter_0023842/__229_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:127522a536059c64946804f6e4e3714e44b3e599a8d39ab5409ecef79fde4d72
+size 17747382

megatron/iter_0023842/__251_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77519b1e132775171d5942be2272c30a4e08e868372907250adc0388e1af0595
+size 17742651

megatron/iter_0023842/__257_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:190f8da6b0d0afc27c8cc901f3890e1014a6a5f05d44ca28d11e54ea49a47bcc
+size 17742651

megatron/iter_0023842/__263_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fcf3d28e52dbcedc233dc4abd8dcf6d9505a19920f471a105ca5bd798fa4ce5
+size 17742651

megatron/iter_0023842/__269_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a0ee77e7b5e283de742fdc5b5d06f3dfa5b260ece492cd5de8992cbe74b6da3
+size 17742651

megatron/iter_0023842/__26_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac9a2576783e9cba09b7eada20f6582519068db5a9a14a66bf0fba7cf9453bf4
+size 30327140

megatron/iter_0023842/__277_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc4a772dd6cd6544b3b1f6f17b73bf86b294fefd9a48db9539b358b8057cec48
+size 17742651

megatron/iter_0023842/__289_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac3d017cb35f64f8b3f911834294fc772f7263eb65e034dce7de7de09bddd1dd
+size 17742651

megatron/iter_0023842/__296_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:721278fedc45caef07633891f35d22eb7f1239e7a88e7ee1aefda99f8675b6c4
+size 17742651

megatron/iter_0023842/__2_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9d13168e1629bd1782190f6b6214573f3896b4a461c5fe00994837cda13eef4
+size 31900004

megatron/iter_0023842/__311_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:161419ecd08859f4066f9e94868647ecce1c9f41d3a4f16403490adb3849a375
+size 17742651

megatron/iter_0023842/__337_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66d1837fa57fecfb5c7d7db2a21e689a88e3312dfad25725fd75f715389399b1
+size 17742651

megatron/iter_0023842/__362_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f852b665af7961bcefe346ef0c0b0e679f1d8e35f6b5b4b106e08c513ec60527
+size 17752113

megatron/iter_0023842/__368_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c89a7970085e1a7bc016bccf4ae3f0f8192f8f247d55ead607e2bd532ca64c3
+size 17747382

megatron/iter_0023842/__372_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07f2c4e7adb2317aad01fe26d2c70fadc62370e0a3cde812564ca00740558bdd
+size 17742651

megatron/iter_0023842/__373_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:671bff49713f3aa4b4038b93aac298c87f9c19b1f85c231457206d779a2fa7d5
+size 17742651

megatron/iter_0023842/__385_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c1f971c8d086078ad6d79090027fe8e2ee84444bc8f05c43fc7ffc2e09387a5
+size 17742651

megatron/iter_0023842/__3_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc1cff102fa717505e9f156e426824ba6276ca99882b0c8b45fef4b56864daf3
+size 31900004

megatron/iter_0023842/__404_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19453bf568698bf158865d28a44c2d3cf83b59f44a729911b3b9b754006ba841
+size 17742651

megatron/iter_0023842/__40_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97ac36e699af394371263f393eed98184878004ac08360946b97f435619cbf1d
+size 30327140

megatron/iter_0023842/__468_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c803b68fc70fb92a7855de69d0d3df14aca94df06bb6447ee0982626cb04d0ca
+size 17742651

megatron/iter_0023842/__472_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cf0b92978c2d5bad25fc28e69a7b7f68735124f2bbc85257ae38818c662a0b2
+size 17742651

megatron/iter_0023842/__496_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30bbb56650838b3417c3fb99d20c1d84adf03d2ad380b39712f32ecc300f1e45
+size 17742651

megatron/iter_0023842/__508_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52308a4097defd3229aa30f8c20c71273ed61ca65a189e2903c2ea25adb9872c
+size 17742651

megatron/iter_0023842/__50_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ab48d4a184f48e54ba288a3ebf60288d043c976ad0d43ceb96305fdb3164af6
+size 30327140

megatron/iter_0023842/__51_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e8889786f916d00be4cec36da03010931e3882adb6bd5433e4041efc4978ef5
+size 30336410

megatron/iter_0023842/__55_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1117583025516fde45490da57ac1cec7a63808cd55971c76542e27bc50d8803f
+size 30336410

megatron/iter_0023842/__64_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:583f50e4844a58fad495a05453f13bc0faae23a2fa6b9a78a99ed4bedb686fec
+size 30327140

megatron/iter_0023842/__70_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b3cd65fbbb41a09a9827b5448c6949a9007d3bca8d267193a5567a1e6108519
+size 30336410

megatron/iter_0023842/__73_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:317ce31527b18ed988b68e3a4b5db2a6a62b8059cf7812190a6161f9e90212bc
+size 30327140

megatron/iter_0023842/metadata.json

@@ -0,0 +1 @@
+{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1}

megatron/iter_0023842/run_config.yaml

@@ -0,0 +1,890 @@
+_target_: megatron.bridge.training.config.ConfigContainer
+checkpoint:
+  _target_: megatron.bridge.training.config.CheckpointConfig
+  async_ckpt_cpu_priority: 10
+  async_ckpt_io_priority: 3
+  async_save: false
+  async_strategy: mcore
+  auto_detect_ckpt_format: false
+  ckpt_assume_constant_structure: false
+  ckpt_convert_format: null
+  ckpt_convert_save: null
+  ckpt_convert_update_legacy_dist_opt_format: false
+  ckpt_format: torch_dist
+  ckpt_fully_parallel_load_exchange_algo: broadcast
+  ckpt_fully_parallel_load_process_group: dp
+  ckpt_fully_parallel_save_process_group: dp
+  ckpt_step: null
+  custom_manager_class: null
+  dist_ckpt_optim_fully_reshardable: false
+  dist_ckpt_save_pre_mcore_014: false
+  dist_ckpt_strictness: assume_ok_unexpected
+  distrib_optim_fully_reshardable_mem_efficient: false
+  exit_on_missing_checkpoint: false
+  finetune: true
+  fully_parallel_load: false
+  fully_parallel_save: true
+  load: /eagle/CausalAlign/yudas/unified-training/checkpoints/midtrain-d24-100b-olmo3-only-wsd
+  load_main_params_from_ckpt: false
+  load_optim: true
+  load_rng: true
+  most_recent_k: -1
+  non_persistent_ckpt_type: null
+  non_persistent_global_ckpt_dir: null
+  non_persistent_local_ckpt_algo: fully_parallel
+  non_persistent_local_ckpt_dir: null
+  non_persistent_save_interval: null
+  pretrained_checkpoint: /eagle/CausalAlign/yudas/unified-training/checkpoints/pretrain-d24-climbmix-400b-wsd
+  replication: false
+  replication_factor: 2
+  replication_jump: null
+  save: /eagle/CausalAlign/yudas/unified-training/checkpoints/midtrain-d24-100b-olmo3-only-wsd
+  save_dgrads_interval: null
+  save_interval: 2000
+  save_optim: true
+  save_retain_interval: null
+  save_rng: true
+  save_tokenizer_assets: true
+  save_wgrads_interval: null
+  storage_writers_per_rank: 1
+  strict_fsdp_dtensor_load: false
+  use_checkpoint_args: false
+  use_mp_args_from_checkpoint_args: false
+  use_persistent_ckpt_worker: true
+  use_tokenizer_model_from_checkpoint_args: true
+  verify_integrity: false
+comm_overlap: null
+dataset:
+  _target_: megatron.bridge.training.config.GPTDatasetConfig
+  add_extra_token_to_sequence: true
+  allow_ambiguous_pad_tokens: false
+  blend: null
+  blend_per_split:
+  - - - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000007
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000022
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000045
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000033
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000019
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000059
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000046
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000009
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000040
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000018
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000042
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000031
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000016
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000021
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000036
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000041
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000029
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000020
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000011
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000050
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000039
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000048
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000003
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000030
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000024
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000055
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000004
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000057
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000054
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000049
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000010
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000000
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000060
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000028
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000044
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000026
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000052
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000012
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000035
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000053
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000038
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000032
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000058
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000013
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000051
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000062
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000002
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000027
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000037
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000005
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000034
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000056
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000043
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000006
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000061
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000008
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000063
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000015
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000017
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000047
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000001
+      - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000014
+    - - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+      - 1.0
+  - - - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000025
+    - - 1.0
+  - - - /eagle/CausalAlign/ftajwar/d24-100b/data/dolmino_tok/runs/6ce5609402eb3bc2/datasets/dolmino-100b/8fb78c393aa8ecd7/shard_000023
+    - - 1.0
+  context_parallel_size: null
+  create_attention_mask: true
+  data_parallel_size: 1
+  data_path: null
+  data_sharding: true
+  dataloader_type: single
+  defer_npy_index_mmap: false
+  drop_last: true
+  drop_last_partial_validation_sequence: true
+  eod_mask_loss: false
+  fast_cache_load: false
+  full_validation: null
+  hybrid_context_parallel: false
+  mid_level_dataset_surplus: 0.005
+  mmap_bin_files: true
+  mock: false
+  multiple_validation_sets: null
+  num_dataset_builder_threads: 1
+  num_workers: 2
+  object_storage_cache_path: null
+  path_to_cache: /eagle/CausalAlign/yudas/unified-training/checkpoints/midtrain-d24-100b-olmo3-only-wsd/dataset-cache
+  persistent_workers: true
+  pin_memory: true
+  random_seed: 1234
+  reset_attention_mask: false
+  reset_position_ids: false
+  sequence_length: 2048
+  sequence_parallel_size: 0
+  sequences_per_dataset: null
+  split: null
+  split_matrix: null
+  token_dtype_code: null
+  tokenizer:
+    _call_: true
+    _target_: megatron.core.tokenizers.text.models.default_tokenizer.DefaultTokenizerText
+  trust_remote_code: null
+ddp:
+  _target_: megatron.core.distributed.distributed_data_parallel_config.DistributedDataParallelConfig
+  align_param_gather: false
+  average_in_collective: false
+  bucket_size: null
+  check_for_large_grads: false
+  check_for_nan_in_grad: true
+  data_parallel_sharding_strategy: no_shard
+  delay_wgrad_compute: false
+  disable_symmetric_registration: false
+  fp4_param_gather: false
+  fp8_param_gather: false
+  fsdp_all_gather_in_start_param_sync: true
+  fsdp_db_use_persist_buf_on_alloc_fail: false
+  fsdp_double_buffer: false
+  fsdp_manual_registration: false
+  grad_reduce_in_fp32: true
+  gradient_reduce_div_fusion: true
+  keep_fp8_transpose_cache: false
+  megatron_fsdp_grad_comm_dtype: null
+  megatron_fsdp_main_grads_dtype: null
+  megatron_fsdp_main_params_dtype:
+    _call_: false
+    _target_: torch.float32
+  nccl_ub: false
+  num_distributed_optimizer_instances: 1
+  outer_dp_sharding_strategy: no_shard
+  overlap_grad_reduce: false
+  overlap_param_gather: false
+  pad_buckets_for_high_nccl_busbw: false
+  param_name_patterns_for_fp32_local_accumulation: []
+  reduce_scatter_with_fp32_accumulation: false
+  reuse_grad_buf_for_mxfp8_param_ag: false
+  suggested_communication_unit_size: null
+  use_custom_fsdp: false
+  use_distributed_optimizer: true
+  use_megatron_fsdp: false
+dist:
+  _target_: megatron.bridge.training.config.DistributedInitConfig
+  align_grad_reduce: true
+  disable_jit_fuser: false
+  distributed_backend: nccl
+  distributed_timeout_minutes: 10
+  distributed_timeout_seconds_after_init: null
+  enable_megatron_core_experimental: false
+  external_gpu_device_mapping: false
+  flight_recorder_dump_on_timeout: true
+  flight_recorder_dump_path: null
+  flight_recorder_extra_dump_on_exec: true
+  flight_recorder_include_only_active: true
+  flight_recorder_include_stack_trace: false
+  flight_recorder_trace_buffer_size: 2000
+  high_priority_stream_groups: []
+  lazy_mpu_init: false
+  local_rank: 0
+  nccl_communicator_config_path: null
+  sharp_enabled_group: null
+  use_decentralized_pg: false
+  use_gloo_process_groups: true
+  use_megatron_fsdp: false
+  use_sharp: false
+  use_torch_fsdp2: false
+  use_tp_pp_dp_mapping: false
+ft: null
+inprocess_restart: null
+logger:
+  _target_: megatron.bridge.training.config.LoggerConfig
+  barrier_with_L1_time: true
+  comet_api_key: null
+  comet_experiment_name: null
+  comet_project: null
+  comet_tags: null
+  comet_workspace: null
+  filter_warnings: true
+  log_device_memory_used: false
+  log_energy: false
+  log_interval: 10
+  log_l2_norm_grad_to_tensorboard: false
+  log_loss_scale_to_tensorboard: true
+  log_max_attention_logit: false
+  log_memory_interval: null
+  log_memory_to_tensorboard: false
+  log_num_zeros_in_grad: false
+  log_params_norm: false
+  log_progress: false
+  log_runtime_to_tensorboard: false
+  log_throughput: false
+  log_throughput_to_tensorboard: false
+  log_timers_to_tensorboard: false
+  log_validation_ppl_to_tensorboard: false
+  log_world_size_to_tensorboard: false
+  logging_level: 20
+  memory_keys: null
+  mlflow_experiment: null
+  mlflow_run_name: null
+  mlflow_tags: null
+  mlflow_tracking_uri: null
+  modules_to_filter: null
+  runtime_time_unit: hours
+  save_config_filepath: null
+  set_level_for_all_loggers: false
+  skip_train_metrics_log: false
+  tensorboard_dir: /home/yudas/unified-training/nemo_experiments/default/tb_logs
+  tensorboard_log_interval: 1
+  tensorboard_queue_size: 1000
+  throughput_window_size: 100
+  timing_log_level: 0
+  timing_log_option: minmax
+  wandb_entity: null
+  wandb_exp_name: midtrain-d24-100b-olmo3-only-wsd
+  wandb_project: nemotron-polaris
+  wandb_save_dir: null
+mixed_precision:
+  _target_: megatron.bridge.training.mixed_precision.MixedPrecisionConfig
+  autocast_dtype: null
+  autocast_enabled: false
+  bf16: true
+  first_last_layers_bf16: false
+  fp16: false
+  fp32: false
+  fp4: null
+  fp4_param: false
+  fp4_param_gather: false
+  fp4_recipe: nvfp4
+  fp8: null
+  fp8_amax_compute_algo: most_recent
+  fp8_amax_history_len: 1
+  fp8_dot_product_attention: false
+  fp8_margin: 0
+  fp8_multi_head_attention: false
+  fp8_param: false
+  fp8_param_gather: false
+  fp8_recipe: tensorwise
+  fp8_wgrad: true
+  grad_reduce_in_fp32: true
+  hysteresis: 2
+  initial_loss_scale: 4294967296
+  loss_scale: null
+  loss_scale_window: 1000
+  min_loss_scale: 1.0
+  num_layers_at_end_in_bf16: 0
+  num_layers_at_start_in_bf16: 0
+  params_dtype:
+    _call_: false
+    _target_: torch.bfloat16
+  pipeline_dtype:
+    _call_: false
+    _target_: torch.bfloat16
+  reuse_grad_buf_for_mxfp8_param_ag: false
+model:
+  _target_: megatron.bridge.models.gpt_provider.GPTModelProvider
+  account_for_embedding_in_pipeline_split: false
+  account_for_loss_in_pipeline_split: false
+  activation_func:
+    _call_: false
+    _target_: torch.nn.functional.silu
+  activation_func_clamp_value: null
+  activation_func_fp8_input_store: false
+  add_bias_linear: false
+  add_qkv_bias: false
+  apply_query_key_layer_scaling: false
+  apply_residual_connection_post_layernorm: false
+  apply_rope_fusion: false
+  attention_backend:
+    _args_:
+    - 5
+    _call_: true
+    _name_: auto
+    _target_: megatron.core.transformer.enums.AttnBackend
+  attention_dropout: 0.1
+  attention_output_gate: false
+  attention_softmax_in_fp32: false
+  autocast_dtype:
+    _call_: false
+    _target_: torch.bfloat16
+  barrier_with_L1_time: true
+  batch_invariant_mode: false
+  batch_p2p_comm: true
+  batch_p2p_sync: true
+  bf16: true
+  bias_activation_fusion: false
+  bias_dropout_fusion: false
+  calculate_per_token_loss: false
+  clone_scatter_output_in_embedding: true
+  config_logger_dir: ''
+  context_parallel_size: 1
+  cp_comm_type: null
+  cpu_offloading: false
+  cpu_offloading_activations: true
+  cpu_offloading_double_buffering: false
+  cpu_offloading_num_layers: 0
+  cpu_offloading_retain_pinned_cpu_buffers: false
+  cpu_offloading_weights: false
+  cross_entropy_fusion_impl: native
+  cross_entropy_loss_fusion: true
+  cuda_graph_impl: none
+  cuda_graph_retain_backward_graph: false
+  cuda_graph_scope: []
+  cuda_graph_use_single_mempool: false
+  cuda_graph_warmup_steps: 3
+  deallocate_pipeline_outputs: true
+  defer_embedding_wgrad_compute: false
+  delay_wgrad_compute: false
+  deterministic_mode: false
+  disable_bf16_reduced_precision_matmul: false
+  disable_parameter_transpose_cache: false
+  distribute_saved_activations: false
+  dsa_indexer_head_dim: null
+  dsa_indexer_loss_coeff: null
+  dsa_indexer_n_heads: null
+  dsa_indexer_topk: null
+  dsa_indexer_use_sparse_loss: false
+  embedding_init_method:
+    _args_: []
+    _partial_: true
+    _target_: torch.nn.init.normal_
+    mean: 0.0
+    std: 0.02
+  embedding_init_method_std: 0.02
+  enable_autocast: false
+  enable_cuda_graph: false
+  ep_overlap_early_attn_memory_release: false
+  experimental_attention_variant: null
+  expert_model_parallel_size: 1
+  expert_tensor_parallel_size: 1
+  external_cuda_graph: false
+  ffn_hidden_size: 4096
+  finalize_model_grads_func:
+    _args_: []
+    _partial_: true
+    _target_: megatron.core.distributed.finalize_model_grads.finalize_model_grads
+    pg_collection:
+      _call_: true
+      _target_: megatron.core.process_groups_config.ProcessGroupCollection
+  fine_grained_activation_offloading: false
+  first_last_layers_bf16: false
+  flash_decode: false
+  fp16: false
+  fp16_lm_cross_entropy: false
+  fp32_residual_connection: false
+  fp4: null
+  fp4_param: false
+  fp4_quantizer_factory: null
+  fp4_recipe: nvfp4
+  fp8: null
+  fp8_amax_compute_algo: most_recent
+  fp8_amax_history_len: 1
+  fp8_dot_product_attention: false
+  fp8_interval: 1
+  fp8_margin: 0
+  fp8_multi_head_attention: false
+  fp8_param: false
+  fp8_quantizer_factory: null
+  fp8_recipe: tensorwise
+  fp8_wgrad: true
+  fused_residual_rmsnorm: false
+  fused_single_qkv_rope: false
+  gated_linear_unit: true
+  glu_linear_offset: 0.0
+  grad_scale_func:
+    _call_: false
+    _target_: megatron.core.optimizer.optimizer.MegatronOptimizer.scale_loss
+  grad_sync_func: null
+  gradient_accumulation_fusion: true
+  hetereogenous_dist_checkpoint: false
+  heterogeneous_block_specs: false
+  hf_model_id: null
+  hidden_dropout: 0.1
+  hidden_size: 1536
+  hierarchical_context_parallel_sizes: null
+  hybrid_context_parallel: false
+  inference_disable_triton_nvls_kernels: false
+  inference_fuse_tp_communication: false
+  inference_grouped_gemm_backend: auto
+  inference_moe_disable_fused_quant_kernels: false
+  inference_rng_tracker: false
+  inference_sampling_seed: 42
+  init_method:
+    _args_: []
+    _partial_: true
+    _target_: torch.nn.init.normal_
+    mean: 0.0
+    std: 0.02
+  init_method_std: 0.02
+  init_model_with_meta_device: false
+  is_hybrid_model: false
+  kitchen_attention_backend: sdpa
+  kv_channels: 128
+  layernorm_epsilon: 1.0e-05
+  layernorm_zero_centered_gamma: false
+  linear_attention_freq: null
+  linear_conv_kernel_dim: 4
+  linear_key_head_dim: 128
+  linear_num_key_heads: 16
+  linear_num_value_heads: 32
+  linear_value_head_dim: 128
+  log_max_attention_logit: false
+  make_vocab_size_divisible_by: 128
+  mamba_head_dim: 64
+  mamba_num_groups: 8
+  mamba_num_heads: null
+  mamba_state_dim: 128
+  masked_softmax_fusion: true
+  max_seqlen_per_dp_cp_rank: null
+  memory_efficient_layer_norm: false
+  microbatch_group_size_per_vp_stage: 1
+  min_offloaded_tensor_size: 1048576
+  mlp_chunks_for_prefill: 1
+  moe_apply_probs_on_input: false
+  moe_aux_loss_coeff: 0.0
+  moe_deepep_num_sms: 20
+  moe_enable_deepep: false
+  moe_enable_routing_replay: false
+  moe_expert_capacity_factor: null
+  moe_ffn_hidden_size: null
+  moe_flex_dispatcher_backend: deepep
+  moe_grouped_gemm: false
+  moe_hybridep_num_blocks_permute: null
+  moe_hybridep_num_blocks_unpermute: null
+  moe_hybridep_num_sms: null
+  moe_input_jitter_eps: null
+  moe_latent_size: null
+  moe_layer_freq: 1
+  moe_layer_recompute: false
+  moe_pad_expert_input_to_capacity: false
+  moe_pad_experts_for_cuda_graph_inference: false
+  moe_per_layer_logging: false
+  moe_permute_fusion: false
+  moe_permute_fusion_into_hybridep: false
+  moe_router_bias_update_rate: 0.001
+  moe_router_dtype: null
+  moe_router_enable_expert_bias: false
+  moe_router_force_biased: null
+  moe_router_force_load_balancing: false
+  moe_router_fusion: false
+  moe_router_group_topk: null
+  moe_router_load_balancing_type: aux_loss
+  moe_router_num_groups: null
+  moe_router_padding_for_fp8: false
+  moe_router_padding_for_quantization: false
+  moe_router_pre_softmax: false
+  moe_router_score_function: softmax
+  moe_router_topk: 2
+  moe_router_topk_limited_devices: null
+  moe_router_topk_scaling_factor: null
+  moe_shared_expert_gate: false
+  moe_shared_expert_intermediate_size: null
+  moe_shared_expert_overlap: false
+  moe_token_dispatcher_type: allgather
+  moe_token_drop_policy: probs
+  moe_token_dropping: false
+  moe_z_loss_coeff: null
+  mrope_section: null
+  mtp_enabled: false
+  mtp_hybrid_override_pattern: null
+  mtp_loss_scaling_factor: 0.1
+  mtp_num_layers: null
+  mtp_standalone: false
+  mtp_use_repeated_layer: false
+  multi_latent_attention: false
+  mup_attn_scale_power: 1.0
+  mup_base_head_dim: null
+  mup_base_hidden_size: null
+  mup_embedding_mult: 1.0
+  mup_output_mult: 1.0
+  mup_width_mult: 1.0
+  nccl_all_reduce_for_prefill: false
+  no_rope_freq: null
+  no_sync_func: null
+  normalization: RMSNorm
+  num_attention_heads: 12
+  num_layers: 24
+  num_layers_at_end_in_bf16: 0
+  num_layers_at_start_in_bf16: 0
+  num_layers_in_first_pipeline_stage: null
+  num_layers_in_last_pipeline_stage: null
+  num_microbatches_with_partial_activation_checkpoints: null
+  num_moe_experts: null
+  num_query_groups: 12
+  offload_modules: []
+  output_layer_init_method:
+    _args_: []
+    _partial_: true
+    _target_: torch.nn.init.normal_
+    mean: 0.0
+    std: 0.002886751345948129
+  overlap_moe_expert_parallel_comm: false
+  overlap_p2p_comm: false
+  overlap_p2p_comm_warmup_flush: false
+  parallel_output: true
+  param_sync_func: null
+  params_dtype:
+    _call_: false
+    _target_: torch.bfloat16
+  perform_initialization: true
+  persist_layer_norm: false
+  pipeline_dtype:
+    _call_: false
+    _target_: torch.bfloat16
+  pipeline_model_parallel_comm_backend: null
+  pipeline_model_parallel_layout: null
+  pipeline_model_parallel_size: 1
+  position_embedding_type: rope
+  qk_clip: false
+  qk_clip_alpha: 0.5
+  qk_clip_threshold: 100
+  qk_l2_norm: false
+  qk_layernorm: false
+  quant_recipe: null
+  recompute_granularity: null
+  recompute_method: null
+  recompute_modules:
+  - core_attn
+  recompute_num_layers: null
+  restore_modelopt_state: false
+  rope_scaling: false
+  rope_scaling_factor: 1.0
+  rotary_base: 10000
+  rotary_interleaved: false
+  rotary_percent: 1.0
+  rotary_scaling_factor: null
+  scatter_embedding_sequence_parallel: true
+  seq_len_interpolation_factor: null
+  seq_length: 2048
+  sequence_parallel: false
+  share_embeddings_and_output_weights: true
+  should_pad_vocab: true
+  softmax_scale: null
+  softmax_type: vanilla
+  symmetric_ar_type: null
+  tensor_model_parallel_size: 1
+  test_mode: false
+  timers:
+    _call_: true
+    _target_: megatron.core.timers.Timers
+  tp_comm_atomic_ag: false
+  tp_comm_atomic_rs: false
+  tp_comm_bootstrap_backend: nccl
+  tp_comm_bulk_dgrad: true
+  tp_comm_bulk_wgrad: true
+  tp_comm_overlap: false
+  tp_comm_overlap_ag: true
+  tp_comm_overlap_cfg: null
+  tp_comm_overlap_disable_fc1: false
+  tp_comm_overlap_disable_qkv: false
+  tp_comm_overlap_rs: true
+  tp_comm_overlap_rs_dgrad: false
+  tp_comm_split_ag: true
+  tp_comm_split_rs: true
+  tp_only_amax_red: false
+  transformer_impl: transformer_engine
+  transformer_layer_spec:
+    _call_: false
+    _target_: megatron.bridge.models.gpt_provider.default_layer_spec
+  use_arbitrary_attention_mask: null
+  use_cpu_initialization: false
+  use_fused_weighted_squared_relu: false
+  use_inference_optimized_layers: false
+  use_kitchen: false
+  use_kitchen_attention: false
+  use_mamba_mem_eff_path: true
+  use_mup: false
+  use_ring_exchange_p2p: false
+  use_te_activation_func: false
+  use_te_rng_tracker: false
+  use_transformer_engine_full_layer_spec: false
+  use_transformer_engine_op_fuser: false
+  variable_seq_lengths: false
+  virtual_pipeline_model_parallel_size: null
+  vocab_size: 50304
+  wgrad_deferral_limit: 0
+  window_attn_skip_freq: null
+  window_size: null
+  yarn_beta_fast: null
+  yarn_beta_slow: null
+  yarn_correction_range_round_to_int: null
+  yarn_mscale: null
+  yarn_mscale_all_dim: null
+  yarn_original_max_position_embeddings: null
+  yarn_rotary_scaling_factor: null
+nvrx_straggler: null
+optimizer:
+  _target_: megatron.bridge.training.config.OptimizerConfig
+  adam_beta1: 0.9
+  adam_beta2: 0.95
+  adam_eps: 1.0e-08
+  apply_wd_to_qk_layernorm: false
+  barrier_with_L1_time: false
+  bf16: true
+  clip_grad: 1.0
+  config_logger_dir: ''
+  decoupled_lr: null
+  decoupled_min_lr: null
+  decoupled_weight_decay: true
+  exp_avg_dtype:
+    _call_: false
+    _target_: torch.float32
+  exp_avg_sq_dtype:
+    _call_: false
+    _target_: torch.float32
+  fp16: false
+  fp8_recipe: tensorwise
+  hysteresis: 2
+  initial_loss_scale: 4294967296
+  lion_beta1: 0.95
+  lion_beta2: 0.98
+  log_num_zeros_in_grad: false
+  loss_scale: null
+  loss_scale_window: 1000
+  lr: 0.0004
+  main_grads_dtype:
+    _call_: false
+    _target_: torch.float32
+  main_params_dtype:
+    _call_: false
+    _target_: torch.float32
+  min_loss_scale: 1.0
+  min_lr: 4.0e-05
+  muon_coefficient_type: quintic
+  muon_extra_scale_factor: 1.0
+  muon_fp32_matmul_prec: medium
+  muon_momentum: 0.95
+  muon_num_ns_steps: 5
+  muon_scalar_optimizer: adam
+  muon_scale_mode: spectral
+  muon_split_qkv: true
+  muon_tp_mode: blockwise
+  muon_use_nesterov: false
+  optimizer: adam
+  optimizer_cpu_offload: false
+  optimizer_cuda_graph: false
+  optimizer_offload_fraction: 0.0
+  overlap_cpu_optimizer_d2h_h2d: false
+  overlap_param_gather: false
+  overlap_param_gather_with_optimizer_step: false
+  params_dtype:
+    _call_: false
+    _target_: torch.bfloat16
+  pin_cpu_grads: true
+  pin_cpu_params: true
+  reuse_grad_buf_for_mxfp8_param_ag: false
+  sgd_momentum: 0.9
+  store_param_remainders: true
+  timers:
+    _call_: true
+    _target_: megatron.core.timers.Timers
+  use_distributed_optimizer: true
+  use_precision_aware_optimizer: false
+  use_torch_optimizer_for_cpu_offload: false
+  weight_decay: 0.1
+optimizer_config_override_provider:
+  _target_: megatron.bridge.training.config.OptimizerConfigOverrideProvider
+peft: null
+profiling:
+  _target_: megatron.bridge.training.config.ProfilingConfig
+  memory_snapshot_path: /nemo_run/snapshot.pickle
+  nvtx_ranges: false
+  profile_ranks:
+  - 0
+  profile_step_end: 12
+  profile_step_start: 10
+  pytorch_profiler_collect_callstack: false
+  pytorch_profiler_collect_chakra: false
+  pytorch_profiler_collect_shapes: false
+  record_memory_history: false
+  record_shapes: false
+  use_nsys_profiler: false
+  use_pytorch_profiler: false
+rerun_state_machine:
+  _target_: megatron.bridge.training.config.RerunStateMachineConfig
+  check_for_nan_in_loss: true
+  check_for_spiky_loss: false
+  error_injection_rate: 0
+  error_injection_type: transient_error
+  rerun_mode: disabled
+  spiky_loss_factor: 10.0
+rng:
+  _target_: megatron.training.config.common_config.RNGConfig
+  data_parallel_random_init: false
+  inference_rng_tracker: false
+  seed: 1234
+  te_rng_tracker: false
+scheduler:
+  _target_: megatron.bridge.training.config.SchedulerConfig
+  end_weight_decay: 0.1
+  lr_decay_iters: 23842
+  lr_decay_samples: null
+  lr_decay_steps: 48828416
+  lr_decay_style: WSD
+  lr_warmup_fraction: null
+  lr_warmup_init: 0.0
+  lr_warmup_iters: 1000
+  lr_warmup_samples: 0
+  lr_warmup_steps: 2048000
+  lr_wsd_decay_iters: 4768
+  lr_wsd_decay_samples: null
+  lr_wsd_decay_style: cosine
+  no_weight_decay_cond_type: null
+  override_opt_param_scheduler: true
+  start_weight_decay: 0.1
+  use_checkpoint_opt_param_scheduler: false
+  wd_incr_steps: 48828416
+  weight_decay_incr_style: constant
+  wsd_decay_steps: 9764864
+straggler: null
+tensor_inspect: null
+tokenizer:
+  _target_: megatron.bridge.training.tokenizers.config.TokenizerConfig
+  chat_template: null
+  force_system_message: false
+  hf_tokenizer_kwargs: {}
+  image_tag_type: null
+  merge_file: null
+  metadata_path: null
+  sp_tokenizer_kwargs: {}
+  special_tokens: null
+  tiktoken_num_special_tokens: 1000
+  tiktoken_pattern: null
+  tiktoken_special_tokens: null
+  tokenizer_model: null
+  tokenizer_prompt_format: null
+  tokenizer_type: NullTokenizer
+  vocab_extra_ids: 0
+  vocab_file: null
+  vocab_size: 50304
+train:
+  _target_: megatron.bridge.training.config.TrainingConfig
+  check_optimizer_step_success: true
+  check_weight_hash_across_dp_replicas_interval: null
+  decrease_batch_size_if_needed: false
+  empty_unused_memory_level: 0
+  eval_interval: 500
+  eval_iters: 32
+  exit_duration_in_mins: null
+  exit_interval: null
+  exit_signal:
+    _args_:
+    - 15
+    _call_: true
+    _name_: SIGTERM
+    _target_: signal.Signals
+  exit_signal_handler: false
+  exit_signal_handler_for_dataloader: false
+  exit_signal_handler_for_training: false
+  global_batch_size: 2048
+  iterations_to_skip: []
+  manual_gc: false
+  manual_gc_eval: true
+  manual_gc_interval: 0
+  micro_batch_size: 4
+  rampup_batch_size: null
+  skip_sync_grad_norm_across_mp: false
+  skip_train: null
+  train_iters: 23842
+  train_samples: null
+  train_sync_interval: null
+validation:
+  _target_: megatron.training.config.training_config.ValidationConfig
+  eval_interval: 500
+  eval_iters: 32
+  full_validation: false
+  multiple_validation_sets: false
+  skip_train: false
+  test_mode: false

megatron/iter_0023842/train_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f27dc846f7233e4710858bf2e5daf213b343972ebe06f12c1b2231bc588685c4
+size 3405

megatron/latest_checkpointed_iteration.txt

@@ -0,0 +1 @@
+23842

megatron/latest_train_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f27dc846f7233e4710858bf2e5daf213b343972ebe06f12c1b2231bc588685c4
+size 3405

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "cls_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": true,
+  "model_max_length": 2048,
+  "pad_token": null,
+  "sep_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}