Training in progress, step 300, checkpoint

last-checkpoint/global_step300/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb3acee7b14d4ffacae8c5d5fcbad34c07ede9be04c9fc902249ea8af39677cf
+size 12470656801

last-checkpoint/global_step300/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:400eed8da71fb73e6540852755395e21428e395d6cee52ca0a2abccf73ccae77
+size 12470656801

last-checkpoint/global_step300/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd8ac9cceeec9fafff48aaea1d6e65683df2da361ff72333b4d826a62f35d48d
+size 12470656801

last-checkpoint/global_step300/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e83a2a980f65e0ea7515657661b488359ad18789396bf624337905fd488582e
+size 12470656801

last-checkpoint/global_step300/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d1b105dc7b828afc7e747253226bc4775a10859832d7c404275b0fa757d2fc4
+size 12470656801

last-checkpoint/global_step300/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6716f190b699c95dee5a9b1a0fbd7daa2168fd8c45f396fa6444353b819b4a31
+size 12470656801

last-checkpoint/global_step300/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f0a39186864e0551829a74780aa01376c2df07b35460604b48ddef95878daff
+size 12470656801

last-checkpoint/global_step300/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17004c02ff3035e82129338f08b3eb031fb3ed339abd5fd3929cbd2089ae029a
+size 12470656801

last-checkpoint/global_step300/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:409e5f5c2cf69bbc90adca82a9e26bd31301eaa668fb81c512a9eefa61753812
+size 113836509

last-checkpoint/global_step300/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b05d3e8f021db7eed9762c1fe0358ff531cf68e7e094aaeb17a3161be766113
+size 113836509

last-checkpoint/global_step300/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d00816af84a9db9a375e5ffa44c5b2b6d9b7abb8048a5bbb0ac7df051ae95c3a
+size 113836509

last-checkpoint/global_step300/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0905e3499c5bf5361e74483857396fb46ea3fdb9073dfc9fe2d0f3095d60dd1d
+size 113836509

last-checkpoint/global_step300/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f40ea42c20e6250701e47cf7cb096649f3768fa839a085cc2c95c74fdb31f51
+size 113836509

last-checkpoint/global_step300/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a27175d19e90ddb5958376cbbd355842655a41b96dc7d5298bbf069306fb5a3
+size 113836509

last-checkpoint/global_step300/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22357e15ea31c0d67880d1022d37050390146d97ac7b166e6dd79dc8a65c7d90
+size 113836509

last-checkpoint/global_step300/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f7d01eb480e4eac76bd18e24b5b749e4345c2a8dd23ccc8960452c6f915e84a
+size 113836509

last-checkpoint/latest

@@ -1 +1 @@
-global_step250
+global_step300

last-checkpoint/model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c546a1e0d24344cf73c804914908af8135ce0b408d2090117863c0c2189eb3d
+oid sha256:3f169a5cf7a218426bd887afae294bc4839c3a9229b597c792398fbc6018e83a
 size 4998056552

last-checkpoint/model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d2e57d936d2d13c55984f3ea377b988da4844266a19ff4d279a8701ff6f107d
+oid sha256:e546047a602208cf99253121ca7936c155134bd167636b7c1a63409bf20121f8
 size 4915962464

last-checkpoint/model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:872d781e5ba9cc7ad0c11029f05d9d5c6147dc528b64ef0f523666426febba39
+oid sha256:4e676b882a8fbc19a333f5bd267b2cbb710f1609e2626e7a20a6b06262a4f4f3
 size 4915962496

last-checkpoint/model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15cc819833634258b81a77094bd99fee92aadbc2966904bbf019149ae2fb2dc5
+oid sha256:a1c6e8d6ba96b7f403634157a75d29507bd38630b12976f5840d1d555880ac9c
 size 2704357976

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d3990cc95427f96a82dc5969935b1cd5b266f5838d1bf06ef51e17330ebe3bf6
+oid sha256:f94e0b44ab1c4011b009f9df0dbd52c7dae895036b39211f5d3e32e618e85d4e
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.17736786094359702,
+  "epoch": 0.21284143313231643,
   "eval_steps": 500,
-  "global_step": 250,
+  "global_step": 300,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1758,6 +1758,356 @@
       "learning_rate": 2.9879861415314855e-05,
       "loss": 0.1391,
       "step": 250
+    },
+    {
+      "epoch": 0.17807733238737142,
+      "grad_norm": 1.1545314991373943,
+      "learning_rate": 2.987762937067796e-05,
+      "loss": 0.1281,
+      "step": 251
+    },
+    {
+      "epoch": 0.17878680383114579,
+      "grad_norm": 1.0356937318996207,
+      "learning_rate": 2.9875376866871404e-05,
+      "loss": 0.1167,
+      "step": 252
+    },
+    {
+      "epoch": 0.17949627527492018,
+      "grad_norm": 2.9560047482944567,
+      "learning_rate": 2.9873103906992745e-05,
+      "loss": 0.1305,
+      "step": 253
+    },
+    {
+      "epoch": 0.18020574671869458,
+      "grad_norm": 1.6347565687572132,
+      "learning_rate": 2.9870810494167677e-05,
+      "loss": 0.1126,
+      "step": 254
+    },
+    {
+      "epoch": 0.18091521816246897,
+      "grad_norm": 3.338203557091556,
+      "learning_rate": 2.9868496631550018e-05,
+      "loss": 0.1433,
+      "step": 255
+    },
+    {
+      "epoch": 0.18162468960624334,
+      "grad_norm": 1.4815978427913965,
+      "learning_rate": 2.9866162322321703e-05,
+      "loss": 0.116,
+      "step": 256
+    },
+    {
+      "epoch": 0.18233416105001773,
+      "grad_norm": 0.5335135172108745,
+      "learning_rate": 2.9863807569692787e-05,
+      "loss": 0.142,
+      "step": 257
+    },
+    {
+      "epoch": 0.18304363249379213,
+      "grad_norm": 0.7726140131518255,
+      "learning_rate": 2.9861432376901444e-05,
+      "loss": 0.1174,
+      "step": 258
+    },
+    {
+      "epoch": 0.18375310393756653,
+      "grad_norm": 1.087347338174722,
+      "learning_rate": 2.9859036747213955e-05,
+      "loss": 0.1226,
+      "step": 259
+    },
+    {
+      "epoch": 0.1844625753813409,
+      "grad_norm": 2.1907412461793307,
+      "learning_rate": 2.9856620683924694e-05,
+      "loss": 0.1433,
+      "step": 260
+    },
+    {
+      "epoch": 0.1851720468251153,
+      "grad_norm": 2.053242402620567,
+      "learning_rate": 2.9854184190356146e-05,
+      "loss": 0.1345,
+      "step": 261
+    },
+    {
+      "epoch": 0.18588151826888968,
+      "grad_norm": 0.6595206308982481,
+      "learning_rate": 2.985172726985889e-05,
+      "loss": 0.1359,
+      "step": 262
+    },
+    {
+      "epoch": 0.18659098971266405,
+      "grad_norm": 2.928310521090971,
+      "learning_rate": 2.984924992581159e-05,
+      "loss": 0.1364,
+      "step": 263
+    },
+    {
+      "epoch": 0.18730046115643845,
+      "grad_norm": 3.1265595749971524,
+      "learning_rate": 2.9846752161620994e-05,
+      "loss": 0.1261,
+      "step": 264
+    },
+    {
+      "epoch": 0.18800993260021284,
+      "grad_norm": 0.9746166109729698,
+      "learning_rate": 2.9844233980721944e-05,
+      "loss": 0.1491,
+      "step": 265
+    },
+    {
+      "epoch": 0.18871940404398724,
+      "grad_norm": 4.2207538483174805,
+      "learning_rate": 2.9841695386577345e-05,
+      "loss": 0.1479,
+      "step": 266
+    },
+    {
+      "epoch": 0.1894288754877616,
+      "grad_norm": 3.0801986262170593,
+      "learning_rate": 2.983913638267818e-05,
+      "loss": 0.1366,
+      "step": 267
+    },
+    {
+      "epoch": 0.190138346931536,
+      "grad_norm": 5.454712342237991,
+      "learning_rate": 2.9836556972543502e-05,
+      "loss": 0.1543,
+      "step": 268
+    },
+    {
+      "epoch": 0.1908478183753104,
+      "grad_norm": 2.5964897962881768,
+      "learning_rate": 2.9833957159720414e-05,
+      "loss": 0.1309,
+      "step": 269
+    },
+    {
+      "epoch": 0.1915572898190848,
+      "grad_norm": 1.6001161689430843,
+      "learning_rate": 2.9831336947784083e-05,
+      "loss": 0.1304,
+      "step": 270
+    },
+    {
+      "epoch": 0.19226676126285916,
+      "grad_norm": 3.636452756146101,
+      "learning_rate": 2.982869634033773e-05,
+      "loss": 0.1372,
+      "step": 271
+    },
+    {
+      "epoch": 0.19297623270663355,
+      "grad_norm": 3.0436634447896096,
+      "learning_rate": 2.9826035341012623e-05,
+      "loss": 0.1276,
+      "step": 272
+    },
+    {
+      "epoch": 0.19368570415040795,
+      "grad_norm": 1.945146608882815,
+      "learning_rate": 2.9823353953468073e-05,
+      "loss": 0.1466,
+      "step": 273
+    },
+    {
+      "epoch": 0.19439517559418235,
+      "grad_norm": 0.9953220714535693,
+      "learning_rate": 2.982065218139142e-05,
+      "loss": 0.1419,
+      "step": 274
+    },
+    {
+      "epoch": 0.1951046470379567,
+      "grad_norm": 4.152907494451121,
+      "learning_rate": 2.981793002849804e-05,
+      "loss": 0.1487,
+      "step": 275
+    },
+    {
+      "epoch": 0.1958141184817311,
+      "grad_norm": 7.202163788627905,
+      "learning_rate": 2.9815187498531348e-05,
+      "loss": 0.1616,
+      "step": 276
+    },
+    {
+      "epoch": 0.1965235899255055,
+      "grad_norm": 7.888111914020001,
+      "learning_rate": 2.9812424595262763e-05,
+      "loss": 0.1892,
+      "step": 277
+    },
+    {
+      "epoch": 0.1972330613692799,
+      "grad_norm": 4.432936829143072,
+      "learning_rate": 2.9809641322491727e-05,
+      "loss": 0.1529,
+      "step": 278
+    },
+    {
+      "epoch": 0.19794253281305427,
+      "grad_norm": 3.506292642573495,
+      "learning_rate": 2.9806837684045697e-05,
+      "loss": 0.1278,
+      "step": 279
+    },
+    {
+      "epoch": 0.19865200425682866,
+      "grad_norm": 1.3127662675754346,
+      "learning_rate": 2.9804013683780137e-05,
+      "loss": 0.1271,
+      "step": 280
+    },
+    {
+      "epoch": 0.19936147570060306,
+      "grad_norm": 4.370928442712123,
+      "learning_rate": 2.98011693255785e-05,
+      "loss": 0.1514,
+      "step": 281
+    },
+    {
+      "epoch": 0.20007094714437743,
+      "grad_norm": 6.4793207972969515,
+      "learning_rate": 2.979830461335225e-05,
+      "loss": 0.1657,
+      "step": 282
+    },
+    {
+      "epoch": 0.20078041858815182,
+      "grad_norm": 3.671625812708481,
+      "learning_rate": 2.9795419551040836e-05,
+      "loss": 0.1394,
+      "step": 283
+    },
+    {
+      "epoch": 0.20148989003192622,
+      "grad_norm": 3.9538699689628647,
+      "learning_rate": 2.9792514142611683e-05,
+      "loss": 0.125,
+      "step": 284
+    },
+    {
+      "epoch": 0.2021993614757006,
+      "grad_norm": 0.5715475190224388,
+      "learning_rate": 2.9789588392060205e-05,
+      "loss": 0.1169,
+      "step": 285
+    },
+    {
+      "epoch": 0.20290883291947498,
+      "grad_norm": 4.741581016171732,
+      "learning_rate": 2.9786642303409785e-05,
+      "loss": 0.1379,
+      "step": 286
+    },
+    {
+      "epoch": 0.20361830436324937,
+      "grad_norm": 3.5228025054256613,
+      "learning_rate": 2.9783675880711783e-05,
+      "loss": 0.1374,
+      "step": 287
+    },
+    {
+      "epoch": 0.20432777580702377,
+      "grad_norm": 3.1024930941820537,
+      "learning_rate": 2.9780689128045504e-05,
+      "loss": 0.1663,
+      "step": 288
+    },
+    {
+      "epoch": 0.20503724725079817,
+      "grad_norm": 3.064924835016314,
+      "learning_rate": 2.9777682049518226e-05,
+      "loss": 0.1604,
+      "step": 289
+    },
+    {
+      "epoch": 0.20574671869457253,
+      "grad_norm": 2.6375689865745677,
+      "learning_rate": 2.9774654649265173e-05,
+      "loss": 0.1263,
+      "step": 290
+    },
+    {
+      "epoch": 0.20645619013834693,
+      "grad_norm": 1.610926099534686,
+      "learning_rate": 2.977160693144952e-05,
+      "loss": 0.1261,
+      "step": 291
+    },
+    {
+      "epoch": 0.20716566158212132,
+      "grad_norm": 4.594815416228411,
+      "learning_rate": 2.976853890026236e-05,
+      "loss": 0.1441,
+      "step": 292
+    },
+    {
+      "epoch": 0.20787513302589572,
+      "grad_norm": 2.957834181803716,
+      "learning_rate": 2.976545055992276e-05,
+      "loss": 0.1336,
+      "step": 293
+    },
+    {
+      "epoch": 0.2085846044696701,
+      "grad_norm": 0.484227879101729,
+      "learning_rate": 2.976234191467767e-05,
+      "loss": 0.1265,
+      "step": 294
+    },
+    {
+      "epoch": 0.20929407591344448,
+      "grad_norm": 4.130668950574084,
+      "learning_rate": 2.9759212968802e-05,
+      "loss": 0.1345,
+      "step": 295
+    },
+    {
+      "epoch": 0.21000354735721888,
+      "grad_norm": 6.451025714470222,
+      "learning_rate": 2.9756063726598554e-05,
+      "loss": 0.1967,
+      "step": 296
+    },
+    {
+      "epoch": 0.21071301880099327,
+      "grad_norm": 4.992745848175137,
+      "learning_rate": 2.9752894192398057e-05,
+      "loss": 0.1506,
+      "step": 297
+    },
+    {
+      "epoch": 0.21142249024476764,
+      "grad_norm": 2.9879713003816724,
+      "learning_rate": 2.974970437055914e-05,
+      "loss": 0.1262,
+      "step": 298
+    },
+    {
+      "epoch": 0.21213196168854204,
+      "grad_norm": 1.0028224046748944,
+      "learning_rate": 2.974649426546832e-05,
+      "loss": 0.1342,
+      "step": 299
+    },
+    {
+      "epoch": 0.21284143313231643,
+      "grad_norm": 2.8480083319182325,
+      "learning_rate": 2.974326388154002e-05,
+      "loss": 0.1305,
+      "step": 300
     }
   ],
   "logging_steps": 1,
@@ -1777,7 +2127,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 161923904765952.0,
+  "total_flos": 194025318318080.0,
   "train_batch_size": 10,
   "trial_name": null,
   "trial_params": null