Training in progress, step 350, checkpoint

last-checkpoint/global_step350/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63fbe7d6007410ffa43c60f12864c357689b1cb1e039fb9059c1a4d09f7fb40b
+size 12470656801

last-checkpoint/global_step350/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e174b2fe2523a9dd4f2056e55126ef9b71858f7bbbb3778b51a2eb5eecb3503c
+size 12470656801

last-checkpoint/global_step350/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a28fd72aeffbd0d61a0c10ebe998b5eedf1716af67653d3a9e363666e8b89937
+size 12470656801

last-checkpoint/global_step350/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f72e907282980ba61a17bed8448c21621c3ac9e17fffe5c9c0e0a08911cc7066
+size 12470656801

last-checkpoint/global_step350/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:828cd5ce043ba536c67b991235473304f4a8998c6616cf06e8aec877551b0fa2
+size 12470656801

last-checkpoint/global_step350/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:818b506447cf63bbfd8014a4de3f849259fde791c41a2b2463c9fdf397503d02
+size 12470656801

last-checkpoint/global_step350/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97fd1f0c871bdcbe81a6963f0feb88ff2814ecfb421efb27f9944daec3c9284a
+size 12470656801

last-checkpoint/global_step350/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a01cd70fe5563aa781d18ee94db7147606fc8fdbe4530dd486db6019d976d26
+size 12470656801

last-checkpoint/global_step350/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1f6794d4362d4ff141efb9475dcbc128a94609ba786c4521f4f007eca97bccd
+size 113836509

last-checkpoint/global_step350/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c3bd1b4ffaaec2ee8adc7e01107c5e230f59ea1a00c19a07ad36092138b8416
+size 113836509

last-checkpoint/global_step350/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f6f04614f732a02d495ad27730a1b5312b63c143cc1ec5ef69a3caaa9908beb
+size 113836509

last-checkpoint/global_step350/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ff6183bcf8d60c6f0c5bf8f46b1216b0281829d5affa10603cee1ff8e6614bf
+size 113836509

last-checkpoint/global_step350/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3b439759d92085c7e18cda33382f8d637689a6da723f94a119c250d3553b019
+size 113836509

last-checkpoint/global_step350/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36168bbd7c23002dc67d8b2eca80f5b7e38af6b2e0c205cf64e72b634d4d8c24
+size 113836509

last-checkpoint/global_step350/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc56b96a01c201abe8f6846d3eb8aaaf0ddff44dac7331a59468641e968f3cd2
+size 113836509

last-checkpoint/global_step350/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2da7b907d3c0850dd22006ac1df7051ef94629e4a804be4cfb0b1764fc967cda
+size 113836509

last-checkpoint/latest

@@ -1 +1 @@
-global_step300
+global_step350

last-checkpoint/model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f169a5cf7a218426bd887afae294bc4839c3a9229b597c792398fbc6018e83a
+oid sha256:fea1c1bcb724cfa28b3c5bee1020fcfab6683a4f456f95ab057ad3b7340c9a86
 size 4998056552

last-checkpoint/model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e546047a602208cf99253121ca7936c155134bd167636b7c1a63409bf20121f8
+oid sha256:e739edc6cbdc0898de2ea6bc868e0a05f4cb5858714800c29e6265feb1e9d813
 size 4915962464

last-checkpoint/model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4e676b882a8fbc19a333f5bd267b2cbb710f1609e2626e7a20a6b06262a4f4f3
+oid sha256:b6e17ccbbe2715ef6ae77d85f92591557c13e5b42c7efd7731bd24e3885db7c8
 size 4915962496

last-checkpoint/model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a1c6e8d6ba96b7f403634157a75d29507bd38630b12976f5840d1d555880ac9c
+oid sha256:0323dac437dd16044bd39db21c19b5fa6c55ec201e51b688f053ba00b2515702
 size 2704357976

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f94e0b44ab1c4011b009f9df0dbd52c7dae895036b39211f5d3e32e618e85d4e
+oid sha256:8571131fb4a32ab565c6a96e31f776258df5327f36f0d1fb191dfeda9c62f04d
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.21284143313231643,
+  "epoch": 0.24831500532103584,
   "eval_steps": 500,
-  "global_step": 300,
+  "global_step": 350,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2108,6 +2108,356 @@
       "learning_rate": 2.974326388154002e-05,
       "loss": 0.1305,
       "step": 300
+    },
+    {
+      "epoch": 0.2135509045760908,
+      "grad_norm": 3.5670682109244027,
+      "learning_rate": 2.9740013223216548e-05,
+      "loss": 0.139,
+      "step": 301
+    },
+    {
+      "epoch": 0.2142603760198652,
+      "grad_norm": 4.212023817298028,
+      "learning_rate": 2.9736742294968088e-05,
+      "loss": 0.13,
+      "step": 302
+    },
+    {
+      "epoch": 0.2149698474636396,
+      "grad_norm": 3.587089019566683,
+      "learning_rate": 2.9733451101292698e-05,
+      "loss": 0.1265,
+      "step": 303
+    },
+    {
+      "epoch": 0.21567931890741399,
+      "grad_norm": 2.45071936937159,
+      "learning_rate": 2.9730139646716308e-05,
+      "loss": 0.1382,
+      "step": 304
+    },
+    {
+      "epoch": 0.21638879035118835,
+      "grad_norm": 4.056735872956379,
+      "learning_rate": 2.9726807935792714e-05,
+      "loss": 0.1686,
+      "step": 305
+    },
+    {
+      "epoch": 0.21709826179496275,
+      "grad_norm": 2.97078857249005,
+      "learning_rate": 2.9723455973103554e-05,
+      "loss": 0.1313,
+      "step": 306
+    },
+    {
+      "epoch": 0.21780773323873714,
+      "grad_norm": 3.5121573353028808,
+      "learning_rate": 2.9720083763258326e-05,
+      "loss": 0.1112,
+      "step": 307
+    },
+    {
+      "epoch": 0.21851720468251154,
+      "grad_norm": 3.18766256760675,
+      "learning_rate": 2.971669131089438e-05,
+      "loss": 0.113,
+      "step": 308
+    },
+    {
+      "epoch": 0.2192266761262859,
+      "grad_norm": 1.0054652212444466,
+      "learning_rate": 2.9713278620676876e-05,
+      "loss": 0.1408,
+      "step": 309
+    },
+    {
+      "epoch": 0.2199361475700603,
+      "grad_norm": 3.0557497330346415,
+      "learning_rate": 2.970984569729883e-05,
+      "loss": 0.1144,
+      "step": 310
+    },
+    {
+      "epoch": 0.2206456190138347,
+      "grad_norm": 3.9053591865675474,
+      "learning_rate": 2.970639254548107e-05,
+      "loss": 0.1267,
+      "step": 311
+    },
+    {
+      "epoch": 0.2213550904576091,
+      "grad_norm": 2.9315526595020285,
+      "learning_rate": 2.970291916997224e-05,
+      "loss": 0.1339,
+      "step": 312
+    },
+    {
+      "epoch": 0.22206456190138346,
+      "grad_norm": 3.537025860153353,
+      "learning_rate": 2.9699425575548802e-05,
+      "loss": 0.1377,
+      "step": 313
+    },
+    {
+      "epoch": 0.22277403334515786,
+      "grad_norm": 0.646576810328437,
+      "learning_rate": 2.9695911767015024e-05,
+      "loss": 0.1005,
+      "step": 314
+    },
+    {
+      "epoch": 0.22348350478893225,
+      "grad_norm": 3.0375309551799776,
+      "learning_rate": 2.9692377749202953e-05,
+      "loss": 0.1321,
+      "step": 315
+    },
+    {
+      "epoch": 0.22419297623270662,
+      "grad_norm": 2.160070387267167,
+      "learning_rate": 2.9688823526972453e-05,
+      "loss": 0.1141,
+      "step": 316
+    },
+    {
+      "epoch": 0.22490244767648102,
+      "grad_norm": 3.6804450262657395,
+      "learning_rate": 2.9685249105211154e-05,
+      "loss": 0.1494,
+      "step": 317
+    },
+    {
+      "epoch": 0.2256119191202554,
+      "grad_norm": 1.6218354531769252,
+      "learning_rate": 2.9681654488834472e-05,
+      "loss": 0.1351,
+      "step": 318
+    },
+    {
+      "epoch": 0.2263213905640298,
+      "grad_norm": 1.0380972366514776,
+      "learning_rate": 2.9678039682785594e-05,
+      "loss": 0.0999,
+      "step": 319
+    },
+    {
+      "epoch": 0.22703086200780417,
+      "grad_norm": 2.369119083792405,
+      "learning_rate": 2.9674404692035458e-05,
+      "loss": 0.1286,
+      "step": 320
+    },
+    {
+      "epoch": 0.22774033345157857,
+      "grad_norm": 1.3776602694633155,
+      "learning_rate": 2.9670749521582778e-05,
+      "loss": 0.1197,
+      "step": 321
+    },
+    {
+      "epoch": 0.22844980489535296,
+      "grad_norm": 1.503611884765329,
+      "learning_rate": 2.9667074176454013e-05,
+      "loss": 0.1108,
+      "step": 322
+    },
+    {
+      "epoch": 0.22915927633912736,
+      "grad_norm": 1.9343104036169902,
+      "learning_rate": 2.9663378661703356e-05,
+      "loss": 0.1386,
+      "step": 323
+    },
+    {
+      "epoch": 0.22986874778290173,
+      "grad_norm": 1.3450282783534633,
+      "learning_rate": 2.965966298241275e-05,
+      "loss": 0.1091,
+      "step": 324
+    },
+    {
+      "epoch": 0.23057821922667612,
+      "grad_norm": 1.1669298363795306,
+      "learning_rate": 2.965592714369185e-05,
+      "loss": 0.1382,
+      "step": 325
+    },
+    {
+      "epoch": 0.23128769067045052,
+      "grad_norm": 0.7340938703579164,
+      "learning_rate": 2.9652171150678055e-05,
+      "loss": 0.1027,
+      "step": 326
+    },
+    {
+      "epoch": 0.23199716211422491,
+      "grad_norm": 1.0767838107063834,
+      "learning_rate": 2.9648395008536464e-05,
+      "loss": 0.1167,
+      "step": 327
+    },
+    {
+      "epoch": 0.23270663355799928,
+      "grad_norm": 1.0586204854666508,
+      "learning_rate": 2.964459872245989e-05,
+      "loss": 0.106,
+      "step": 328
+    },
+    {
+      "epoch": 0.23341610500177368,
+      "grad_norm": 1.7488457522091547,
+      "learning_rate": 2.9640782297668845e-05,
+      "loss": 0.1174,
+      "step": 329
+    },
+    {
+      "epoch": 0.23412557644554807,
+      "grad_norm": 1.447672995304521,
+      "learning_rate": 2.9636945739411533e-05,
+      "loss": 0.1267,
+      "step": 330
+    },
+    {
+      "epoch": 0.23483504788932247,
+      "grad_norm": 2.005280272097454,
+      "learning_rate": 2.9633089052963858e-05,
+      "loss": 0.1238,
+      "step": 331
+    },
+    {
+      "epoch": 0.23554451933309684,
+      "grad_norm": 1.5411679400453127,
+      "learning_rate": 2.9629212243629387e-05,
+      "loss": 0.1141,
+      "step": 332
+    },
+    {
+      "epoch": 0.23625399077687123,
+      "grad_norm": 0.6149683068305483,
+      "learning_rate": 2.9625315316739365e-05,
+      "loss": 0.1137,
+      "step": 333
+    },
+    {
+      "epoch": 0.23696346222064563,
+      "grad_norm": 0.6354682310356766,
+      "learning_rate": 2.9621398277652703e-05,
+      "loss": 0.1523,
+      "step": 334
+    },
+    {
+      "epoch": 0.23767293366442,
+      "grad_norm": 0.4258397242446468,
+      "learning_rate": 2.9617461131755974e-05,
+      "loss": 0.1004,
+      "step": 335
+    },
+    {
+      "epoch": 0.2383824051081944,
+      "grad_norm": 0.845449244365038,
+      "learning_rate": 2.961350388446339e-05,
+      "loss": 0.1508,
+      "step": 336
+    },
+    {
+      "epoch": 0.23909187655196878,
+      "grad_norm": 1.1920803712120955,
+      "learning_rate": 2.9609526541216817e-05,
+      "loss": 0.1069,
+      "step": 337
+    },
+    {
+      "epoch": 0.23980134799574318,
+      "grad_norm": 1.4617256282688038,
+      "learning_rate": 2.960552910748575e-05,
+      "loss": 0.11,
+      "step": 338
+    },
+    {
+      "epoch": 0.24051081943951755,
+      "grad_norm": 1.4313623063982495,
+      "learning_rate": 2.9601511588767314e-05,
+      "loss": 0.1151,
+      "step": 339
+    },
+    {
+      "epoch": 0.24122029088329194,
+      "grad_norm": 0.5953978626552301,
+      "learning_rate": 2.959747399058625e-05,
+      "loss": 0.127,
+      "step": 340
+    },
+    {
+      "epoch": 0.24192976232706634,
+      "grad_norm": 2.977298376039244,
+      "learning_rate": 2.9593416318494916e-05,
+      "loss": 0.1247,
+      "step": 341
+    },
+    {
+      "epoch": 0.24263923377084073,
+      "grad_norm": 2.341990467465989,
+      "learning_rate": 2.9589338578073277e-05,
+      "loss": 0.1257,
+      "step": 342
+    },
+    {
+      "epoch": 0.2433487052146151,
+      "grad_norm": 0.9654180090165114,
+      "learning_rate": 2.9585240774928887e-05,
+      "loss": 0.1399,
+      "step": 343
+    },
+    {
+      "epoch": 0.2440581766583895,
+      "grad_norm": 0.4976154855602877,
+      "learning_rate": 2.9581122914696898e-05,
+      "loss": 0.1194,
+      "step": 344
+    },
+    {
+      "epoch": 0.2447676481021639,
+      "grad_norm": 0.8093574207224504,
+      "learning_rate": 2.957698500304004e-05,
+      "loss": 0.1245,
+      "step": 345
+    },
+    {
+      "epoch": 0.2454771195459383,
+      "grad_norm": 2.4496597624457164,
+      "learning_rate": 2.9572827045648614e-05,
+      "loss": 0.1109,
+      "step": 346
+    },
+    {
+      "epoch": 0.24618659098971266,
+      "grad_norm": 3.831466220403665,
+      "learning_rate": 2.9568649048240496e-05,
+      "loss": 0.1679,
+      "step": 347
+    },
+    {
+      "epoch": 0.24689606243348705,
+      "grad_norm": 1.4317595227825495,
+      "learning_rate": 2.9564451016561107e-05,
+      "loss": 0.1596,
+      "step": 348
+    },
+    {
+      "epoch": 0.24760553387726145,
+      "grad_norm": 0.6483178119650296,
+      "learning_rate": 2.9560232956383433e-05,
+      "loss": 0.1171,
+      "step": 349
+    },
+    {
+      "epoch": 0.24831500532103584,
+      "grad_norm": 2.0583845057044936,
+      "learning_rate": 2.9555994873507992e-05,
+      "loss": 0.1423,
+      "step": 350
     }
   ],
   "logging_steps": 1,
@@ -2127,7 +2477,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 194025318318080.0,
+  "total_flos": 226804831944704.0,
   "train_batch_size": 10,
   "trial_name": null,
   "trial_params": null