Training in progress, step 600, checkpoint

last-checkpoint/global_step600/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3ddbee87fa30db265d71ad08d494d1c3178c7ca7bf06493ec012e0357a7852d
+size 12470656801

last-checkpoint/global_step600/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83f71a1756208ca44af237b0e389459d1316700d5b2b9e710f284323d49e06dc
+size 12470656801

last-checkpoint/global_step600/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2e3430ad166c68afe3a14019d565bcdd2c6c2c56422ece11729e4ff4a4dcd31
+size 12470656801

last-checkpoint/global_step600/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e618e8fb0c70ed14420b9706d4cbba89989742dc80e1654419f7707e8a93bac0
+size 12470656801

last-checkpoint/global_step600/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef08910125d35f267de15adb38591a2bc744747a110b672eff07b08872b68c95
+size 12470656801

last-checkpoint/global_step600/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f136fd2824fc4c93bfdc64540ae05b0d5fa286f46e352d05d4fb9e5ff7bc1ce8
+size 12470656801

last-checkpoint/global_step600/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fdbf45b17ee979a2d3b8c8f27c71be2623f3415f03f1cf2cf3c57a7e865a14a
+size 12470656801

last-checkpoint/global_step600/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a483a710a971a52af4c12b4ce85d2bf624c86674acba8adc155ea232ab8b18fe
+size 12470656801

last-checkpoint/global_step600/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe75d7c1642d0a1269b2c7d10c3e80a8825d8f65943409d1660ce65140e26770
+size 113836509

last-checkpoint/global_step600/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edf70f4b00c54b0509e07964081fa3ccddb12565b8feefc6750d74b72cba62af
+size 113836509

last-checkpoint/global_step600/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95a87694bb26a7d713db94a27105fd5bd45823a08a36fdcdcea9ac38a9075221
+size 113836509

last-checkpoint/global_step600/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12fcf48b51b40a6b384c1e6ba26132f9aef11e0841102a840ecde033f468e66b
+size 113836509

last-checkpoint/global_step600/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad127ef3bdf37e126ac13033dc13ab3b525dd1ff80166084de26e786eeab5b76
+size 113836509

last-checkpoint/global_step600/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40b6f91079c00a18eb722bb559daf2dfb062d128b8ff6f7f97ec7033da264dff
+size 113836509

last-checkpoint/global_step600/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34320c257b0d119a27df2a4f9fea983c0ee01586586b4129115432e3aeafffb1
+size 113836509

last-checkpoint/global_step600/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e17c94107cc4b363eeab34bca53560a3a40e82677e591cf6e9e5dd232626ad1
+size 113836509

last-checkpoint/latest

@@ -1 +1 @@
-global_step550
+global_step600

last-checkpoint/model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2535b05457abc808cea2652783ae94f84e639e56b602356660712bdee875d1a
+oid sha256:7800117170cf82b55257c1707adcef8daad710f7cb64de8a9ad8e0e76049a110
 size 4998056552

last-checkpoint/model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fec7d43249874447d8b611dd0020321ed100dbf6261db033b84b9fb4d9a6b4db
+oid sha256:1d33bdc13a5ab594e1d6d5d1093fd63f65f19a5bb3833eb85415983802d819cc
 size 4915962464

last-checkpoint/model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3366852135007d1dae99d9449356831d253d31c2a788e54b5b9d3395c650e44a
+oid sha256:ecd844151e6b7f9c65595211ee5132ae0ccdacc12a451cd1dcb4e1f1ea33c3a5
 size 4915962496

last-checkpoint/model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b59986cc61160ed8d12e731899849878039d390bf7b8051e93a990c46c46a15f
+oid sha256:fcf99ddf29be9984cb6fe02fbb76153e12808be74b2fb9c543fe45de1e9a0966
 size 2704357976

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:660e9bc509979b5be135e0fe71637b257ff80c313e117f88ed3b79603659d777
+oid sha256:11c1dce72fa355171a43b7faada7f2e5ee1b14ef3b7e886de17666de4ecebe3d
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.3902092940759134,
+  "epoch": 0.42568286626463286,
   "eval_steps": 500,
-  "global_step": 550,
+  "global_step": 600,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3858,6 +3858,356 @@
       "learning_rate": 2.831563841365169e-05,
       "loss": 0.1077,
       "step": 550
+    },
+    {
+      "epoch": 0.39091876551968785,
+      "grad_norm": 1.3474483961218888,
+      "learning_rate": 2.8307530696226224e-05,
+      "loss": 0.0849,
+      "step": 551
+    },
+    {
+      "epoch": 0.3916282369634622,
+      "grad_norm": 0.5599425920689892,
+      "learning_rate": 2.8299404678773145e-05,
+      "loss": 0.1079,
+      "step": 552
+    },
+    {
+      "epoch": 0.3923377084072366,
+      "grad_norm": 0.8790334152733548,
+      "learning_rate": 2.8291260372467064e-05,
+      "loss": 0.1224,
+      "step": 553
+    },
+    {
+      "epoch": 0.393047179851011,
+      "grad_norm": 0.941455289026995,
+      "learning_rate": 2.8283097788507713e-05,
+      "loss": 0.0981,
+      "step": 554
+    },
+    {
+      "epoch": 0.3937566512947854,
+      "grad_norm": 0.8317601911521915,
+      "learning_rate": 2.8274916938119987e-05,
+      "loss": 0.0984,
+      "step": 555
+    },
+    {
+      "epoch": 0.3944661227385598,
+      "grad_norm": 1.315951573310521,
+      "learning_rate": 2.826671783255389e-05,
+      "loss": 0.098,
+      "step": 556
+    },
+    {
+      "epoch": 0.39517559418233417,
+      "grad_norm": 0.8093031174705544,
+      "learning_rate": 2.8258500483084534e-05,
+      "loss": 0.1232,
+      "step": 557
+    },
+    {
+      "epoch": 0.39588506562610853,
+      "grad_norm": 2.111748407910676,
+      "learning_rate": 2.825026490101212e-05,
+      "loss": 0.1419,
+      "step": 558
+    },
+    {
+      "epoch": 0.39659453706988296,
+      "grad_norm": 0.7546197406653664,
+      "learning_rate": 2.8242011097661904e-05,
+      "loss": 0.1186,
+      "step": 559
+    },
+    {
+      "epoch": 0.3973040085136573,
+      "grad_norm": 1.2792560412735465,
+      "learning_rate": 2.8233739084384228e-05,
+      "loss": 0.1373,
+      "step": 560
+    },
+    {
+      "epoch": 0.3980134799574317,
+      "grad_norm": 0.9387966041624755,
+      "learning_rate": 2.8225448872554454e-05,
+      "loss": 0.1317,
+      "step": 561
+    },
+    {
+      "epoch": 0.3987229514012061,
+      "grad_norm": 1.324051537579158,
+      "learning_rate": 2.8217140473572974e-05,
+      "loss": 0.096,
+      "step": 562
+    },
+    {
+      "epoch": 0.3994324228449805,
+      "grad_norm": 2.2184673432521613,
+      "learning_rate": 2.8208813898865203e-05,
+      "loss": 0.1355,
+      "step": 563
+    },
+    {
+      "epoch": 0.40014189428875485,
+      "grad_norm": 2.2977328985321988,
+      "learning_rate": 2.820046915988153e-05,
+      "loss": 0.1245,
+      "step": 564
+    },
+    {
+      "epoch": 0.4008513657325293,
+      "grad_norm": 0.8970425250647217,
+      "learning_rate": 2.8192106268097336e-05,
+      "loss": 0.116,
+      "step": 565
+    },
+    {
+      "epoch": 0.40156083717630364,
+      "grad_norm": 2.7080964283232833,
+      "learning_rate": 2.818372523501297e-05,
+      "loss": 0.1156,
+      "step": 566
+    },
+    {
+      "epoch": 0.40227030862007807,
+      "grad_norm": 4.297554105324148,
+      "learning_rate": 2.8175326072153714e-05,
+      "loss": 0.1405,
+      "step": 567
+    },
+    {
+      "epoch": 0.40297978006385243,
+      "grad_norm": 3.9335382319654926,
+      "learning_rate": 2.816690879106979e-05,
+      "loss": 0.1165,
+      "step": 568
+    },
+    {
+      "epoch": 0.4036892515076268,
+      "grad_norm": 4.734404491257155,
+      "learning_rate": 2.8158473403336335e-05,
+      "loss": 0.163,
+      "step": 569
+    },
+    {
+      "epoch": 0.4043987229514012,
+      "grad_norm": 1.2641693545150974,
+      "learning_rate": 2.8150019920553386e-05,
+      "loss": 0.0983,
+      "step": 570
+    },
+    {
+      "epoch": 0.4051081943951756,
+      "grad_norm": 0.7388020809360326,
+      "learning_rate": 2.8141548354345855e-05,
+      "loss": 0.1233,
+      "step": 571
+    },
+    {
+      "epoch": 0.40581766583894996,
+      "grad_norm": 3.1179398092120216,
+      "learning_rate": 2.813305871636354e-05,
+      "loss": 0.1234,
+      "step": 572
+    },
+    {
+      "epoch": 0.4065271372827244,
+      "grad_norm": 5.628199527445323,
+      "learning_rate": 2.812455101828107e-05,
+      "loss": 0.1377,
+      "step": 573
+    },
+    {
+      "epoch": 0.40723660872649875,
+      "grad_norm": 3.3535594744498725,
+      "learning_rate": 2.8116025271797923e-05,
+      "loss": 0.1162,
+      "step": 574
+    },
+    {
+      "epoch": 0.4079460801702732,
+      "grad_norm": 3.2499716790640676,
+      "learning_rate": 2.8107481488638392e-05,
+      "loss": 0.1022,
+      "step": 575
+    },
+    {
+      "epoch": 0.40865555161404754,
+      "grad_norm": 2.823878852476875,
+      "learning_rate": 2.8098919680551574e-05,
+      "loss": 0.1196,
+      "step": 576
+    },
+    {
+      "epoch": 0.4093650230578219,
+      "grad_norm": 1.4480570093450607,
+      "learning_rate": 2.809033985931135e-05,
+      "loss": 0.1131,
+      "step": 577
+    },
+    {
+      "epoch": 0.41007449450159633,
+      "grad_norm": 0.8497223022259704,
+      "learning_rate": 2.8081742036716385e-05,
+      "loss": 0.1178,
+      "step": 578
+    },
+    {
+      "epoch": 0.4107839659453707,
+      "grad_norm": 3.970096324971072,
+      "learning_rate": 2.8073126224590073e-05,
+      "loss": 0.1515,
+      "step": 579
+    },
+    {
+      "epoch": 0.41149343738914507,
+      "grad_norm": 4.597521022934096,
+      "learning_rate": 2.806449243478058e-05,
+      "loss": 0.1467,
+      "step": 580
+    },
+    {
+      "epoch": 0.4122029088329195,
+      "grad_norm": 5.45376769391393,
+      "learning_rate": 2.805584067916076e-05,
+      "loss": 0.1382,
+      "step": 581
+    },
+    {
+      "epoch": 0.41291238027669386,
+      "grad_norm": 3.7527216143697766,
+      "learning_rate": 2.8047170969628203e-05,
+      "loss": 0.1224,
+      "step": 582
+    },
+    {
+      "epoch": 0.4136218517204682,
+      "grad_norm": 3.110343863636251,
+      "learning_rate": 2.8038483318105168e-05,
+      "loss": 0.1084,
+      "step": 583
+    },
+    {
+      "epoch": 0.41433132316424265,
+      "grad_norm": 0.8632185334036093,
+      "learning_rate": 2.8029777736538598e-05,
+      "loss": 0.0942,
+      "step": 584
+    },
+    {
+      "epoch": 0.415040794608017,
+      "grad_norm": 1.632466927853409,
+      "learning_rate": 2.802105423690009e-05,
+      "loss": 0.0954,
+      "step": 585
+    },
+    {
+      "epoch": 0.41575026605179144,
+      "grad_norm": 2.85259810688113,
+      "learning_rate": 2.8012312831185875e-05,
+      "loss": 0.1028,
+      "step": 586
+    },
+    {
+      "epoch": 0.4164597374955658,
+      "grad_norm": 4.517740177335467,
+      "learning_rate": 2.8003553531416818e-05,
+      "loss": 0.14,
+      "step": 587
+    },
+    {
+      "epoch": 0.4171692089393402,
+      "grad_norm": 2.983832501458289,
+      "learning_rate": 2.799477634963838e-05,
+      "loss": 0.0995,
+      "step": 588
+    },
+    {
+      "epoch": 0.4178786803831146,
+      "grad_norm": 2.5750949400952554,
+      "learning_rate": 2.798598129792063e-05,
+      "loss": 0.1044,
+      "step": 589
+    },
+    {
+      "epoch": 0.41858815182688897,
+      "grad_norm": 0.7340883922803195,
+      "learning_rate": 2.797716838835819e-05,
+      "loss": 0.0943,
+      "step": 590
+    },
+    {
+      "epoch": 0.41929762327066333,
+      "grad_norm": 3.550188614082691,
+      "learning_rate": 2.7968337633070252e-05,
+      "loss": 0.1404,
+      "step": 591
+    },
+    {
+      "epoch": 0.42000709471443776,
+      "grad_norm": 2.2835904132382807,
+      "learning_rate": 2.7959489044200553e-05,
+      "loss": 0.1078,
+      "step": 592
+    },
+    {
+      "epoch": 0.4207165661582121,
+      "grad_norm": 3.253665348401309,
+      "learning_rate": 2.795062263391734e-05,
+      "loss": 0.1174,
+      "step": 593
+    },
+    {
+      "epoch": 0.42142603760198655,
+      "grad_norm": 3.368908937873198,
+      "learning_rate": 2.7941738414413374e-05,
+      "loss": 0.1252,
+      "step": 594
+    },
+    {
+      "epoch": 0.4221355090457609,
+      "grad_norm": 2.9843700331588243,
+      "learning_rate": 2.793283639790592e-05,
+      "loss": 0.1609,
+      "step": 595
+    },
+    {
+      "epoch": 0.4228449804895353,
+      "grad_norm": 1.0247980106954122,
+      "learning_rate": 2.7923916596636692e-05,
+      "loss": 0.1041,
+      "step": 596
+    },
+    {
+      "epoch": 0.4235544519333097,
+      "grad_norm": 1.883606813121272,
+      "learning_rate": 2.7914979022871874e-05,
+      "loss": 0.0839,
+      "step": 597
+    },
+    {
+      "epoch": 0.4242639233770841,
+      "grad_norm": 3.492624813108691,
+      "learning_rate": 2.790602368890209e-05,
+      "loss": 0.1249,
+      "step": 598
+    },
+    {
+      "epoch": 0.42497339482085844,
+      "grad_norm": 2.3319145331768736,
+      "learning_rate": 2.789705060704239e-05,
+      "loss": 0.0968,
+      "step": 599
+    },
+    {
+      "epoch": 0.42568286626463286,
+      "grad_norm": 3.900032674331407,
+      "learning_rate": 2.7888059789632228e-05,
+      "loss": 0.1286,
+      "step": 600
     }
   ],
   "logging_steps": 1,
@@ -3877,7 +4227,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 356303432417280.0,
+  "total_flos": 388252513599488.0,
   "train_batch_size": 10,
   "trial_name": null,
   "trial_params": null