Training in progress, step 400, checkpoint

last-checkpoint/global_step400/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f38971061efa68bc69fd662a2d7e39b953b40aa4e5932bc17e6dbe0970058c62
+size 12470656801

last-checkpoint/global_step400/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f59d302c9fad20b43379bed3bce0659d2b6cdc324da948d6d2210bd717aa967
+size 12470656801

last-checkpoint/global_step400/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e7cc2c1a00270c1c86d480e710b62760ec89ad21cf9c77d7cbe930358c84a66
+size 12470656801

last-checkpoint/global_step400/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bba3100657e95d88e2d75428e9e0208366e15c1c18a7f4771abbf81e9b0f323d
+size 12470656801

last-checkpoint/global_step400/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d8da68f17159c5e5968e2b942408eb854e75588b448c5c1a70f7fa444926de3
+size 12470656801

last-checkpoint/global_step400/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:748ac3ca30607398d5bdb92e7fc6ba5f167f3676a126f62c1516b7ebcf1ce3e4
+size 12470656801

last-checkpoint/global_step400/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f83891b75a3613949ce939b30d4d32b88090636f463c86edcb1240b6903a3f5
+size 12470656801

last-checkpoint/global_step400/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b860faa80945191a065608381715637ff80f919be5379e043a31968a4c218b0e
+size 12470656801

last-checkpoint/global_step400/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4ea8b50ccde8a559a136579c3ba33bf3f0bb837410c93491cabcceb1c19bc65
+size 113836509

last-checkpoint/global_step400/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2434ec964ada76c052e93f70c2a94184cf74bf8004d24e656e8f7bbea74bed8
+size 113836509

last-checkpoint/global_step400/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0f5a360885bf4162c227292700ee0abfd8f1bbac88b8bc6251369e45d132194
+size 113836509

last-checkpoint/global_step400/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:211a8a9e3686abbfff1299d808ccc350ec640c6b0a89435a64911b1b35eb8664
+size 113836509

last-checkpoint/global_step400/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeabe82f5dec57a8ae444a333077d6e0ea1d1cb7424cff6bab8082cdf7e6d8ee
+size 113836509

last-checkpoint/global_step400/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03ba379cd282c015cbaf87b64251c4ba2d194a6830010bfb803d0be504d6f0b5
+size 113836509

last-checkpoint/global_step400/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ada4314f4ef4174afd864c39603dd0c20c13bb87d3480687bffe6d54a9c1da64
+size 113836509

last-checkpoint/global_step400/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61ef3f366aa2e0473893615068a14af99579f072d3db557b0ca895520de33c0e
+size 113836509

last-checkpoint/latest

@@ -1 +1 @@
-global_step350
+global_step400

last-checkpoint/model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fea1c1bcb724cfa28b3c5bee1020fcfab6683a4f456f95ab057ad3b7340c9a86
+oid sha256:93bbcc6fd62b942e8c6b4c65c878cbb94690ceafac2f4c766061cec171e1335d
 size 4998056552

last-checkpoint/model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e739edc6cbdc0898de2ea6bc868e0a05f4cb5858714800c29e6265feb1e9d813
+oid sha256:1f54cdd981f0ce3eeacbc91899c54f02cb2b1966b230f57399b57af704d9cdc8
 size 4915962464

last-checkpoint/model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6e17ccbbe2715ef6ae77d85f92591557c13e5b42c7efd7731bd24e3885db7c8
+oid sha256:2711d7e2ac79ca2962a8fe98a9fe1ff83b305013ef01d8a50dad34009a5848f2
 size 4915962496

last-checkpoint/model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0323dac437dd16044bd39db21c19b5fa6c55ec201e51b688f053ba00b2515702
+oid sha256:26eca38c7697c00dcbe65005ec5b71a1be5aba9c3e8af4278b5455b7ac8ffccc
 size 2704357976

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8571131fb4a32ab565c6a96e31f776258df5327f36f0d1fb191dfeda9c62f04d
+oid sha256:b14bad8c6fdbeac617c6a4d3974aa0e6e6168ad9eaf2eff0678014d7d51782d3
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.24831500532103584,
+  "epoch": 0.2837885775097552,
   "eval_steps": 500,
-  "global_step": 350,
+  "global_step": 400,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2458,6 +2458,356 @@
       "learning_rate": 2.9555994873507992e-05,
       "loss": 0.1423,
       "step": 350
+    },
+    {
+      "epoch": 0.2490244767648102,
+      "grad_norm": 3.879641589013307,
+      "learning_rate": 2.955173677376284e-05,
+      "loss": 0.1276,
+      "step": 351
+    },
+    {
+      "epoch": 0.2497339482085846,
+      "grad_norm": 1.3769005993971457,
+      "learning_rate": 2.954745866300356e-05,
+      "loss": 0.1163,
+      "step": 352
+    },
+    {
+      "epoch": 0.250443419652359,
+      "grad_norm": 1.8376354282810619,
+      "learning_rate": 2.9543160547113254e-05,
+      "loss": 0.153,
+      "step": 353
+    },
+    {
+      "epoch": 0.25115289109613337,
+      "grad_norm": 1.9844824326325108,
+      "learning_rate": 2.953884243200253e-05,
+      "loss": 0.0909,
+      "step": 354
+    },
+    {
+      "epoch": 0.2518623625399078,
+      "grad_norm": 1.8353955039268464,
+      "learning_rate": 2.9534504323609504e-05,
+      "loss": 0.134,
+      "step": 355
+    },
+    {
+      "epoch": 0.25257183398368216,
+      "grad_norm": 1.0110178319854,
+      "learning_rate": 2.9530146227899787e-05,
+      "loss": 0.1708,
+      "step": 356
+    },
+    {
+      "epoch": 0.2532813054274565,
+      "grad_norm": 1.0238365805291345,
+      "learning_rate": 2.9525768150866465e-05,
+      "loss": 0.141,
+      "step": 357
+    },
+    {
+      "epoch": 0.25399077687123095,
+      "grad_norm": 0.41824371907324315,
+      "learning_rate": 2.952137009853011e-05,
+      "loss": 0.1349,
+      "step": 358
+    },
+    {
+      "epoch": 0.2547002483150053,
+      "grad_norm": 0.41464086641304115,
+      "learning_rate": 2.9516952076938768e-05,
+      "loss": 0.1321,
+      "step": 359
+    },
+    {
+      "epoch": 0.2554097197587797,
+      "grad_norm": 0.7932581870199277,
+      "learning_rate": 2.9512514092167933e-05,
+      "loss": 0.1424,
+      "step": 360
+    },
+    {
+      "epoch": 0.2561191912025541,
+      "grad_norm": 0.47130216597421964,
+      "learning_rate": 2.9508056150320565e-05,
+      "loss": 0.124,
+      "step": 361
+    },
+    {
+      "epoch": 0.2568286626463285,
+      "grad_norm": 3.5373335369279273,
+      "learning_rate": 2.950357825752706e-05,
+      "loss": 0.1225,
+      "step": 362
+    },
+    {
+      "epoch": 0.2575381340901029,
+      "grad_norm": 2.4843113969349346,
+      "learning_rate": 2.9499080419945243e-05,
+      "loss": 0.1242,
+      "step": 363
+    },
+    {
+      "epoch": 0.25824760553387727,
+      "grad_norm": 0.858679728851391,
+      "learning_rate": 2.9494562643760385e-05,
+      "loss": 0.1068,
+      "step": 364
+    },
+    {
+      "epoch": 0.25895707697765163,
+      "grad_norm": 1.3485967573023745,
+      "learning_rate": 2.9490024935185167e-05,
+      "loss": 0.1185,
+      "step": 365
+    },
+    {
+      "epoch": 0.25966654842142606,
+      "grad_norm": 1.0040234129610184,
+      "learning_rate": 2.9485467300459673e-05,
+      "loss": 0.1098,
+      "step": 366
+    },
+    {
+      "epoch": 0.2603760198652004,
+      "grad_norm": 1.0006381662096229,
+      "learning_rate": 2.9480889745851396e-05,
+      "loss": 0.1285,
+      "step": 367
+    },
+    {
+      "epoch": 0.2610854913089748,
+      "grad_norm": 1.9738191133612824,
+      "learning_rate": 2.9476292277655227e-05,
+      "loss": 0.151,
+      "step": 368
+    },
+    {
+      "epoch": 0.2617949627527492,
+      "grad_norm": 2.50954516418147,
+      "learning_rate": 2.947167490219343e-05,
+      "loss": 0.1281,
+      "step": 369
+    },
+    {
+      "epoch": 0.2625044341965236,
+      "grad_norm": 2.366897710500404,
+      "learning_rate": 2.946703762581565e-05,
+      "loss": 0.1194,
+      "step": 370
+    },
+    {
+      "epoch": 0.26321390564029795,
+      "grad_norm": 1.5891040342564644,
+      "learning_rate": 2.9462380454898898e-05,
+      "loss": 0.1201,
+      "step": 371
+    },
+    {
+      "epoch": 0.2639233770840724,
+      "grad_norm": 0.7103642176294669,
+      "learning_rate": 2.9457703395847552e-05,
+      "loss": 0.122,
+      "step": 372
+    },
+    {
+      "epoch": 0.26463284852784674,
+      "grad_norm": 1.5882807551220135,
+      "learning_rate": 2.9453006455093324e-05,
+      "loss": 0.132,
+      "step": 373
+    },
+    {
+      "epoch": 0.26534231997162117,
+      "grad_norm": 3.506348182270578,
+      "learning_rate": 2.944828963909528e-05,
+      "loss": 0.1625,
+      "step": 374
+    },
+    {
+      "epoch": 0.26605179141539553,
+      "grad_norm": 2.2167828603365187,
+      "learning_rate": 2.9443552954339807e-05,
+      "loss": 0.1281,
+      "step": 375
+    },
+    {
+      "epoch": 0.2667612628591699,
+      "grad_norm": 0.7834681043304855,
+      "learning_rate": 2.9438796407340624e-05,
+      "loss": 0.1132,
+      "step": 376
+    },
+    {
+      "epoch": 0.2674707343029443,
+      "grad_norm": 1.41517623538854,
+      "learning_rate": 2.9434020004638757e-05,
+      "loss": 0.1227,
+      "step": 377
+    },
+    {
+      "epoch": 0.2681802057467187,
+      "grad_norm": 4.763582671829927,
+      "learning_rate": 2.9429223752802543e-05,
+      "loss": 0.1385,
+      "step": 378
+    },
+    {
+      "epoch": 0.26888967719049306,
+      "grad_norm": 4.517989156270013,
+      "learning_rate": 2.9424407658427605e-05,
+      "loss": 0.1289,
+      "step": 379
+    },
+    {
+      "epoch": 0.2695991486342675,
+      "grad_norm": 1.8668066600612216,
+      "learning_rate": 2.9419571728136866e-05,
+      "loss": 0.0931,
+      "step": 380
+    },
+    {
+      "epoch": 0.27030862007804185,
+      "grad_norm": 2.4308728966392588,
+      "learning_rate": 2.941471596858052e-05,
+      "loss": 0.1127,
+      "step": 381
+    },
+    {
+      "epoch": 0.2710180915218163,
+      "grad_norm": 2.1639101101062224,
+      "learning_rate": 2.9409840386436024e-05,
+      "loss": 0.1401,
+      "step": 382
+    },
+    {
+      "epoch": 0.27172756296559064,
+      "grad_norm": 0.8909030254301304,
+      "learning_rate": 2.94049449884081e-05,
+      "loss": 0.0897,
+      "step": 383
+    },
+    {
+      "epoch": 0.272437034409365,
+      "grad_norm": 1.0477063909729751,
+      "learning_rate": 2.9400029781228722e-05,
+      "loss": 0.1438,
+      "step": 384
+    },
+    {
+      "epoch": 0.27314650585313943,
+      "grad_norm": 0.8902571869086673,
+      "learning_rate": 2.9395094771657108e-05,
+      "loss": 0.1001,
+      "step": 385
+    },
+    {
+      "epoch": 0.2738559772969138,
+      "grad_norm": 1.448303545838668,
+      "learning_rate": 2.9390139966479692e-05,
+      "loss": 0.1141,
+      "step": 386
+    },
+    {
+      "epoch": 0.27456544874068817,
+      "grad_norm": 1.1357321306457262,
+      "learning_rate": 2.938516537251014e-05,
+      "loss": 0.1046,
+      "step": 387
+    },
+    {
+      "epoch": 0.2752749201844626,
+      "grad_norm": 5.65741028667349,
+      "learning_rate": 2.9380170996589345e-05,
+      "loss": 0.155,
+      "step": 388
+    },
+    {
+      "epoch": 0.27598439162823696,
+      "grad_norm": 3.861699101319741,
+      "learning_rate": 2.9375156845585374e-05,
+      "loss": 0.1288,
+      "step": 389
+    },
+    {
+      "epoch": 0.2766938630720113,
+      "grad_norm": 0.67675696886419,
+      "learning_rate": 2.937012292639351e-05,
+      "loss": 0.1302,
+      "step": 390
+    },
+    {
+      "epoch": 0.27740333451578575,
+      "grad_norm": 1.4877319351369076,
+      "learning_rate": 2.936506924593621e-05,
+      "loss": 0.1207,
+      "step": 391
+    },
+    {
+      "epoch": 0.2781128059595601,
+      "grad_norm": 1.2697475227356827,
+      "learning_rate": 2.9359995811163114e-05,
+      "loss": 0.1348,
+      "step": 392
+    },
+    {
+      "epoch": 0.27882227740333454,
+      "grad_norm": 1.8358469663153398,
+      "learning_rate": 2.9354902629051022e-05,
+      "loss": 0.1361,
+      "step": 393
+    },
+    {
+      "epoch": 0.2795317488471089,
+      "grad_norm": 1.5695843972913994,
+      "learning_rate": 2.934978970660389e-05,
+      "loss": 0.1009,
+      "step": 394
+    },
+    {
+      "epoch": 0.2802412202908833,
+      "grad_norm": 0.5297163047032748,
+      "learning_rate": 2.9344657050852825e-05,
+      "loss": 0.142,
+      "step": 395
+    },
+    {
+      "epoch": 0.2809506917346577,
+      "grad_norm": 1.1116725849481155,
+      "learning_rate": 2.9339504668856064e-05,
+      "loss": 0.133,
+      "step": 396
+    },
+    {
+      "epoch": 0.28166016317843207,
+      "grad_norm": 0.7324661972256294,
+      "learning_rate": 2.9334332567698973e-05,
+      "loss": 0.1275,
+      "step": 397
+    },
+    {
+      "epoch": 0.28236963462220643,
+      "grad_norm": 1.304101752615871,
+      "learning_rate": 2.9329140754494035e-05,
+      "loss": 0.1285,
+      "step": 398
+    },
+    {
+      "epoch": 0.28307910606598086,
+      "grad_norm": 1.5052152972197361,
+      "learning_rate": 2.9323929236380846e-05,
+      "loss": 0.1254,
+      "step": 399
+    },
+    {
+      "epoch": 0.2837885775097552,
+      "grad_norm": 1.5007070683932355,
+      "learning_rate": 2.9318698020526084e-05,
+      "loss": 0.136,
+      "step": 400
     }
   ],
   "logging_steps": 1,
@@ -2477,7 +2827,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 226804831944704.0,
+  "total_flos": 259806188339200.0,
   "train_batch_size": 10,
   "trial_name": null,
   "trial_params": null