Training in progress, step 500, checkpoint

last-checkpoint/global_step500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:040f7290634a24ef81e709b64e59f28294b92e84a8f396199e711cd72933b6bc
+size 12470656801

last-checkpoint/global_step500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aebc56bac5033cdca910e7a140dd5a6000d99e4a5dcacdc20ab81ccfb1c563e1
+size 12470656801

last-checkpoint/global_step500/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23d90ba626050587c18777bae9d896857c3e50fe083c313b7df0e557118d2fee
+size 12470656801

last-checkpoint/global_step500/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f3048308785bea9bd2898dbd2b693ad6d552beb2b8ac0937a8c2ecf448e617
+size 12470656801

last-checkpoint/global_step500/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d1e935b758c886be8de2c7fabcff883cf0f66f235b98dbf613e12f45b92e0b6
+size 12470656801

last-checkpoint/global_step500/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d96a07ffe58a3c2f33503e2900b2d9c2da80875b0c543791882379da84b34a09
+size 12470656801

last-checkpoint/global_step500/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0dcbadb7c087bcc4391fb4041b62f8a196b7356792f94a30e008464f4e96ae55
+size 12470656801

last-checkpoint/global_step500/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5867bd53ba89efacbb0269ae7a76d0a3a73af9900d74d0a026bebdef6382c5d
+size 12470656801

last-checkpoint/global_step500/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0b93302a72707f2abd8fb815b22f56da3191ab802500753e873b66071112264
+size 113836509

last-checkpoint/global_step500/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07b1f86abc7ee5df3ea8fd4bb718407e7902c8d8910b85375e4315cf5c7e4478
+size 113836509

last-checkpoint/global_step500/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2b851053179f70f12f222b4b82655d00b659bd6185648b0fd5ecf0c5c3790d4
+size 113836509

last-checkpoint/global_step500/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a77621af2751d477c9a007a83d5ad7d8200e7bfa63cdeb8a2a9cfbaded5f0694
+size 113836509

last-checkpoint/global_step500/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5e38f7706fa115a0c6e4a1ad3726632619f375d20c6d35a3ca3ddfb51877ec6
+size 113836509

last-checkpoint/global_step500/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bd8e8dd2b312950cdf1531fe3db653558dadc513fd9f94e1ee038c2002ae6b2
+size 113836509

last-checkpoint/global_step500/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f62cce8920fd32e5668bd36f48543e9f8204d09c3b20ee9ce560a8d9ccb4f60
+size 113836509

last-checkpoint/global_step500/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd617d6d8203bb2b0a960ab7283afbe71f3a2c8c0b924495f07c4912d4587b17
+size 113836509

last-checkpoint/latest

@@ -1 +1 @@
-global_step450
+global_step500

last-checkpoint/model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc622ee04de8bf59167f927fc7ec1a6a881a124207f8c6dc8c8bafca08f3271c
+oid sha256:b9975009f95cdfd50ca4845f9d7a36764ac8897fa805efeeac5c05832b0bc4d6
 size 4998056552

last-checkpoint/model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5940864693fe3d71adc22988c237898a74ab859b4f78cf253aca72236f7bb882
+oid sha256:8ebd2d2406adf7c2d2194d7728167eafedaf455d23e6c7dee37b29857a4a141c
 size 4915962464

last-checkpoint/model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb27641ad03ca3faec2ece0da58d5bfd53e8ec59b25e0c96e318c458eca1a326
+oid sha256:f0171bab31497f5a0259f6124b3082686ead1e9b164cc4bf3a163b7d64508f1d
 size 4915962496

last-checkpoint/model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:96c7b9e690dd812e379b3abaa1f7f27be2bc58a0203df9dd3b17aa9e9b6a12d1
+oid sha256:402a3d2946b0e96438f4612df8d06724124234a56ef0f0c01416e3708adace7b
 size 2704357976

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45f09859d03e8124982cd231d41c4db6486c58e304d261682b37274496789731
+oid sha256:96ae6c786cb447cd7059a623a9aa428ccaeccad649bb5b94fd950012c0744df9
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.31926214969847466,
+  "epoch": 0.35473572188719404,
   "eval_steps": 500,
-  "global_step": 450,
+  "global_step": 500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3158,6 +3158,356 @@
       "learning_rate": 2.903218889379168e-05,
       "loss": 0.1096,
       "step": 450
+    },
+    {
+      "epoch": 0.31997162114224903,
+      "grad_norm": 0.8441546985189771,
+      "learning_rate": 2.902596322769682e-05,
+      "loss": 0.1269,
+      "step": 451
+    },
+    {
+      "epoch": 0.3206810925860234,
+      "grad_norm": 3.097844971673133,
+      "learning_rate": 2.901971827361231e-05,
+      "loss": 0.1442,
+      "step": 452
+    },
+    {
+      "epoch": 0.3213905640297978,
+      "grad_norm": 1.0097737198624575,
+      "learning_rate": 2.901345404012598e-05,
+      "loss": 0.0995,
+      "step": 453
+    },
+    {
+      "epoch": 0.3221000354735722,
+      "grad_norm": 0.7485286675934367,
+      "learning_rate": 2.9007170535852176e-05,
+      "loss": 0.1243,
+      "step": 454
+    },
+    {
+      "epoch": 0.32280950691734656,
+      "grad_norm": 3.974146052255681,
+      "learning_rate": 2.9000867769431734e-05,
+      "loss": 0.1158,
+      "step": 455
+    },
+    {
+      "epoch": 0.323518978361121,
+      "grad_norm": 5.153830169966646,
+      "learning_rate": 2.8994545749532003e-05,
+      "loss": 0.1243,
+      "step": 456
+    },
+    {
+      "epoch": 0.32422844980489535,
+      "grad_norm": 4.522082737227656,
+      "learning_rate": 2.8988204484846774e-05,
+      "loss": 0.1452,
+      "step": 457
+    },
+    {
+      "epoch": 0.3249379212486697,
+      "grad_norm": 3.816894032844365,
+      "learning_rate": 2.8981843984096338e-05,
+      "loss": 0.1378,
+      "step": 458
+    },
+    {
+      "epoch": 0.32564739269244414,
+      "grad_norm": 1.3357144795947238,
+      "learning_rate": 2.897546425602741e-05,
+      "loss": 0.0992,
+      "step": 459
+    },
+    {
+      "epoch": 0.3263568641362185,
+      "grad_norm": 1.7656239034092869,
+      "learning_rate": 2.8969065309413158e-05,
+      "loss": 0.1159,
+      "step": 460
+    },
+    {
+      "epoch": 0.32706633557999293,
+      "grad_norm": 2.6594704803126126,
+      "learning_rate": 2.8962647153053187e-05,
+      "loss": 0.1298,
+      "step": 461
+    },
+    {
+      "epoch": 0.3277758070237673,
+      "grad_norm": 3.470766271987268,
+      "learning_rate": 2.8956209795773502e-05,
+      "loss": 0.1348,
+      "step": 462
+    },
+    {
+      "epoch": 0.32848527846754166,
+      "grad_norm": 2.426484992688522,
+      "learning_rate": 2.894975324642653e-05,
+      "loss": 0.1102,
+      "step": 463
+    },
+    {
+      "epoch": 0.3291947499113161,
+      "grad_norm": 2.0087837128641763,
+      "learning_rate": 2.894327751389107e-05,
+      "loss": 0.1322,
+      "step": 464
+    },
+    {
+      "epoch": 0.32990422135509045,
+      "grad_norm": 2.1044358389626665,
+      "learning_rate": 2.8936782607072316e-05,
+      "loss": 0.1286,
+      "step": 465
+    },
+    {
+      "epoch": 0.3306136927988648,
+      "grad_norm": 2.9930476942786894,
+      "learning_rate": 2.8930268534901823e-05,
+      "loss": 0.1251,
+      "step": 466
+    },
+    {
+      "epoch": 0.33132316424263925,
+      "grad_norm": 2.1925012098009247,
+      "learning_rate": 2.892373530633751e-05,
+      "loss": 0.1347,
+      "step": 467
+    },
+    {
+      "epoch": 0.3320326356864136,
+      "grad_norm": 0.9158124499144366,
+      "learning_rate": 2.891718293036363e-05,
+      "loss": 0.1087,
+      "step": 468
+    },
+    {
+      "epoch": 0.33274210713018804,
+      "grad_norm": 0.9890067798699078,
+      "learning_rate": 2.8910611415990766e-05,
+      "loss": 0.0855,
+      "step": 469
+    },
+    {
+      "epoch": 0.3334515785739624,
+      "grad_norm": 1.2152282929004945,
+      "learning_rate": 2.890402077225583e-05,
+      "loss": 0.1302,
+      "step": 470
+    },
+    {
+      "epoch": 0.33416105001773677,
+      "grad_norm": 3.1961672858202066,
+      "learning_rate": 2.8897411008222026e-05,
+      "loss": 0.1174,
+      "step": 471
+    },
+    {
+      "epoch": 0.3348705214615112,
+      "grad_norm": 3.4748868366483716,
+      "learning_rate": 2.8890782132978866e-05,
+      "loss": 0.1117,
+      "step": 472
+    },
+    {
+      "epoch": 0.33557999290528556,
+      "grad_norm": 1.3038400253004467,
+      "learning_rate": 2.888413415564213e-05,
+      "loss": 0.1051,
+      "step": 473
+    },
+    {
+      "epoch": 0.33628946434905993,
+      "grad_norm": 0.6255093137268427,
+      "learning_rate": 2.887746708535388e-05,
+      "loss": 0.111,
+      "step": 474
+    },
+    {
+      "epoch": 0.33699893579283435,
+      "grad_norm": 1.34689232058292,
+      "learning_rate": 2.8870780931282412e-05,
+      "loss": 0.1131,
+      "step": 475
+    },
+    {
+      "epoch": 0.3377084072366087,
+      "grad_norm": 2.4879808829860743,
+      "learning_rate": 2.8864075702622295e-05,
+      "loss": 0.1444,
+      "step": 476
+    },
+    {
+      "epoch": 0.3384178786803831,
+      "grad_norm": 2.2842096171823973,
+      "learning_rate": 2.8857351408594303e-05,
+      "loss": 0.116,
+      "step": 477
+    },
+    {
+      "epoch": 0.3391273501241575,
+      "grad_norm": 1.6083058694068209,
+      "learning_rate": 2.885060805844545e-05,
+      "loss": 0.1155,
+      "step": 478
+    },
+    {
+      "epoch": 0.3398368215679319,
+      "grad_norm": 1.1674603168326303,
+      "learning_rate": 2.884384566144893e-05,
+      "loss": 0.1089,
+      "step": 479
+    },
+    {
+      "epoch": 0.3405462930117063,
+      "grad_norm": 0.7719232134916407,
+      "learning_rate": 2.8837064226904145e-05,
+      "loss": 0.1399,
+      "step": 480
+    },
+    {
+      "epoch": 0.34125576445548067,
+      "grad_norm": 1.5218085739053615,
+      "learning_rate": 2.883026376413668e-05,
+      "loss": 0.1114,
+      "step": 481
+    },
+    {
+      "epoch": 0.34196523589925504,
+      "grad_norm": 0.9109862142531445,
+      "learning_rate": 2.8823444282498288e-05,
+      "loss": 0.1179,
+      "step": 482
+    },
+    {
+      "epoch": 0.34267470734302946,
+      "grad_norm": 0.5573121486290471,
+      "learning_rate": 2.8816605791366855e-05,
+      "loss": 0.0988,
+      "step": 483
+    },
+    {
+      "epoch": 0.34338417878680383,
+      "grad_norm": 1.3377823727573568,
+      "learning_rate": 2.880974830014643e-05,
+      "loss": 0.0776,
+      "step": 484
+    },
+    {
+      "epoch": 0.3440936502305782,
+      "grad_norm": 2.018482834568401,
+      "learning_rate": 2.880287181826719e-05,
+      "loss": 0.0855,
+      "step": 485
+    },
+    {
+      "epoch": 0.3448031216743526,
+      "grad_norm": 1.916645708122162,
+      "learning_rate": 2.879597635518541e-05,
+      "loss": 0.1209,
+      "step": 486
+    },
+    {
+      "epoch": 0.345512593118127,
+      "grad_norm": 2.1276787207181584,
+      "learning_rate": 2.8789061920383475e-05,
+      "loss": 0.0953,
+      "step": 487
+    },
+    {
+      "epoch": 0.3462220645619014,
+      "grad_norm": 1.0368175822313435,
+      "learning_rate": 2.878212852336987e-05,
+      "loss": 0.1058,
+      "step": 488
+    },
+    {
+      "epoch": 0.3469315360056758,
+      "grad_norm": 0.9901952589766785,
+      "learning_rate": 2.8775176173679152e-05,
+      "loss": 0.0836,
+      "step": 489
+    },
+    {
+      "epoch": 0.34764100744945015,
+      "grad_norm": 0.8114828297928477,
+      "learning_rate": 2.8768204880871925e-05,
+      "loss": 0.1259,
+      "step": 490
+    },
+    {
+      "epoch": 0.34835047889322457,
+      "grad_norm": 1.0284217348053095,
+      "learning_rate": 2.8761214654534864e-05,
+      "loss": 0.1081,
+      "step": 491
+    },
+    {
+      "epoch": 0.34905995033699894,
+      "grad_norm": 1.848307395122552,
+      "learning_rate": 2.875420550428067e-05,
+      "loss": 0.1098,
+      "step": 492
+    },
+    {
+      "epoch": 0.3497694217807733,
+      "grad_norm": 0.9573856105115914,
+      "learning_rate": 2.8747177439748066e-05,
+      "loss": 0.1065,
+      "step": 493
+    },
+    {
+      "epoch": 0.3504788932245477,
+      "grad_norm": 0.5044734493627655,
+      "learning_rate": 2.874013047060179e-05,
+      "loss": 0.0841,
+      "step": 494
+    },
+    {
+      "epoch": 0.3511883646683221,
+      "grad_norm": 0.7347140554247538,
+      "learning_rate": 2.8733064606532574e-05,
+      "loss": 0.0939,
+      "step": 495
+    },
+    {
+      "epoch": 0.35189783611209646,
+      "grad_norm": 1.374773099981302,
+      "learning_rate": 2.8725979857257138e-05,
+      "loss": 0.0899,
+      "step": 496
+    },
+    {
+      "epoch": 0.3526073075558709,
+      "grad_norm": 0.805310727253721,
+      "learning_rate": 2.8718876232518165e-05,
+      "loss": 0.0809,
+      "step": 497
+    },
+    {
+      "epoch": 0.35331677899964525,
+      "grad_norm": 1.2252311956008417,
+      "learning_rate": 2.8711753742084305e-05,
+      "loss": 0.1218,
+      "step": 498
+    },
+    {
+      "epoch": 0.3540262504434197,
+      "grad_norm": 0.5543857641099807,
+      "learning_rate": 2.8704612395750133e-05,
+      "loss": 0.1312,
+      "step": 499
+    },
+    {
+      "epoch": 0.35473572188719404,
+      "grad_norm": 0.5165937866079755,
+      "learning_rate": 2.869745220333618e-05,
+      "loss": 0.0948,
+      "step": 500
     }
   ],
   "logging_steps": 1,
@@ -3177,7 +3527,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 292284734832640.0,
+  "total_flos": 324169851666432.0,
   "train_batch_size": 10,
   "trial_name": null,
   "trial_params": null