Training in progress, step 550, checkpoint

last-checkpoint/global_step550/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fb6b649488521cdf8e992de64a50e89cd7dddb94ddfe37bcbc072c3c4e4117f
+size 12470656801

last-checkpoint/global_step550/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51398ce741c1acaf7b621d0ae82e16bde9b29bf55776f5cc55f9ad632a4d2c79
+size 12470656801

last-checkpoint/global_step550/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68d6573135d1a7eeca778a941aba153473010cccd723c0aacef3a3d90749636c
+size 12470656801

last-checkpoint/global_step550/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aca2bf7dd0a096c6b36e1e54f2e2fdede473f4c8688e0e26e012b4c3e9fad69f
+size 12470656801

last-checkpoint/global_step550/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea1980af95d830b8c4f78ce32731d1fd54ab9f9264598bc22f35e42d97643bfd
+size 12470656801

last-checkpoint/global_step550/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81362a7eaac4b740da987d5cff8e2e59e814e6915e3b5771fc0d9e1b3dd57b5e
+size 12470656801

last-checkpoint/global_step550/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd440c56f66bd83b7d53e98249db7be8d3b5698ff6710f82d199eec078172dfb
+size 12470656801

last-checkpoint/global_step550/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:640b9b2d7e893fb7c09895698575be031c12bc24d9207290599e23274ecc3aee
+size 12470656801

last-checkpoint/global_step550/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03465cba7194add023bcf140a9bb4c3c459f66da27d5ef12fee1d9648e9379c0
+size 113836509

last-checkpoint/global_step550/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:291eb25b5004833025037327bf075e93139e8600a6051e43e2276371e61788d6
+size 113836509

last-checkpoint/global_step550/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9dc53f6e03ce4834cde7981f054c4af6d1909fb9d63038eb4d4de5211591c30
+size 113836509

last-checkpoint/global_step550/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75731418cee3b3b21e1f1af3aef68c22a9deef0096f09d436c53002247031c2e
+size 113836509

last-checkpoint/global_step550/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6c5f5cfc914c32d389fc47ad0f47b700f47ce193426a759f61773670cf4dfe7
+size 113836509

last-checkpoint/global_step550/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0c81440b3f3ef25052631efaec11e1b082c7a20e73afc067f4c285e1f90b42c
+size 113836509

last-checkpoint/global_step550/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3661ffc0fc650a9163f686ef0646df7d53a84780e2a13496724f03b598a0d752
+size 113836509

last-checkpoint/global_step550/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc368d3cd078511936f292037c41b52f9e0f65ee20dfbda01bbf66f9b05f7152
+size 113836509

last-checkpoint/latest

@@ -1 +1 @@
-global_step500
+global_step550

last-checkpoint/model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9975009f95cdfd50ca4845f9d7a36764ac8897fa805efeeac5c05832b0bc4d6
+oid sha256:e2535b05457abc808cea2652783ae94f84e639e56b602356660712bdee875d1a
 size 4998056552

last-checkpoint/model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8ebd2d2406adf7c2d2194d7728167eafedaf455d23e6c7dee37b29857a4a141c
+oid sha256:fec7d43249874447d8b611dd0020321ed100dbf6261db033b84b9fb4d9a6b4db
 size 4915962464

last-checkpoint/model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f0171bab31497f5a0259f6124b3082686ead1e9b164cc4bf3a163b7d64508f1d
+oid sha256:3366852135007d1dae99d9449356831d253d31c2a788e54b5b9d3395c650e44a
 size 4915962496

last-checkpoint/model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:402a3d2946b0e96438f4612df8d06724124234a56ef0f0c01416e3708adace7b
+oid sha256:b59986cc61160ed8d12e731899849878039d390bf7b8051e93a990c46c46a15f
 size 2704357976

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:96ae6c786cb447cd7059a623a9aa428ccaeccad649bb5b94fd950012c0744df9
+oid sha256:660e9bc509979b5be135e0fe71637b257ff80c313e117f88ed3b79603659d777
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.35473572188719404,
+  "epoch": 0.3902092940759134,
   "eval_steps": 500,
-  "global_step": 500,
+  "global_step": 550,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3508,6 +3508,356 @@
       "learning_rate": 2.869745220333618e-05,
       "loss": 0.0948,
       "step": 500
+    },
+    {
+      "epoch": 0.3554451933309684,
+      "grad_norm": 1.5884838029755333,
+      "learning_rate": 2.8690273174688875e-05,
+      "loss": 0.1275,
+      "step": 501
+    },
+    {
+      "epoch": 0.35615466477474284,
+      "grad_norm": 1.1139505346254182,
+      "learning_rate": 2.8683075319680553e-05,
+      "loss": 0.1279,
+      "step": 502
+    },
+    {
+      "epoch": 0.3568641362185172,
+      "grad_norm": 2.235958534086228,
+      "learning_rate": 2.8675858648209442e-05,
+      "loss": 0.1138,
+      "step": 503
+    },
+    {
+      "epoch": 0.35757360766229157,
+      "grad_norm": 1.2953694992928817,
+      "learning_rate": 2.8668623170199644e-05,
+      "loss": 0.1286,
+      "step": 504
+    },
+    {
+      "epoch": 0.358283079106066,
+      "grad_norm": 0.5678186538404929,
+      "learning_rate": 2.8661368895601117e-05,
+      "loss": 0.1394,
+      "step": 505
+    },
+    {
+      "epoch": 0.35899255054984036,
+      "grad_norm": 2.71690689842914,
+      "learning_rate": 2.865409583438968e-05,
+      "loss": 0.1227,
+      "step": 506
+    },
+    {
+      "epoch": 0.3597020219936148,
+      "grad_norm": 0.9839957303416551,
+      "learning_rate": 2.8646803996566975e-05,
+      "loss": 0.1258,
+      "step": 507
+    },
+    {
+      "epoch": 0.36041149343738915,
+      "grad_norm": 0.6000833258292534,
+      "learning_rate": 2.8639493392160473e-05,
+      "loss": 0.0974,
+      "step": 508
+    },
+    {
+      "epoch": 0.3611209648811635,
+      "grad_norm": 2.2142042594567224,
+      "learning_rate": 2.863216403122345e-05,
+      "loss": 0.1388,
+      "step": 509
+    },
+    {
+      "epoch": 0.36183043632493794,
+      "grad_norm": 0.6554134256882517,
+      "learning_rate": 2.8624815923834968e-05,
+      "loss": 0.1015,
+      "step": 510
+    },
+    {
+      "epoch": 0.3625399077687123,
+      "grad_norm": 0.9363173463439146,
+      "learning_rate": 2.861744908009988e-05,
+      "loss": 0.1249,
+      "step": 511
+    },
+    {
+      "epoch": 0.3632493792124867,
+      "grad_norm": 0.5303245395389136,
+      "learning_rate": 2.8610063510148796e-05,
+      "loss": 0.1127,
+      "step": 512
+    },
+    {
+      "epoch": 0.3639588506562611,
+      "grad_norm": 0.7952493442861119,
+      "learning_rate": 2.8602659224138078e-05,
+      "loss": 0.0966,
+      "step": 513
+    },
+    {
+      "epoch": 0.36466832210003547,
+      "grad_norm": 0.7047199263011409,
+      "learning_rate": 2.8595236232249836e-05,
+      "loss": 0.0931,
+      "step": 514
+    },
+    {
+      "epoch": 0.36537779354380984,
+      "grad_norm": 0.8888203524849596,
+      "learning_rate": 2.858779454469189e-05,
+      "loss": 0.1277,
+      "step": 515
+    },
+    {
+      "epoch": 0.36608726498758426,
+      "grad_norm": 0.9081730555446176,
+      "learning_rate": 2.858033417169778e-05,
+      "loss": 0.1367,
+      "step": 516
+    },
+    {
+      "epoch": 0.3667967364313586,
+      "grad_norm": 0.7404612859804995,
+      "learning_rate": 2.8572855123526734e-05,
+      "loss": 0.1271,
+      "step": 517
+    },
+    {
+      "epoch": 0.36750620787513305,
+      "grad_norm": 2.100314961167394,
+      "learning_rate": 2.8565357410463664e-05,
+      "loss": 0.1094,
+      "step": 518
+    },
+    {
+      "epoch": 0.3682156793189074,
+      "grad_norm": 1.1291899766290312,
+      "learning_rate": 2.8557841042819147e-05,
+      "loss": 0.1014,
+      "step": 519
+    },
+    {
+      "epoch": 0.3689251507626818,
+      "grad_norm": 1.24535798498572,
+      "learning_rate": 2.8550306030929425e-05,
+      "loss": 0.1031,
+      "step": 520
+    },
+    {
+      "epoch": 0.3696346222064562,
+      "grad_norm": 0.43112369779790843,
+      "learning_rate": 2.854275238515636e-05,
+      "loss": 0.0978,
+      "step": 521
+    },
+    {
+      "epoch": 0.3703440936502306,
+      "grad_norm": 0.6300528409343545,
+      "learning_rate": 2.8535180115887452e-05,
+      "loss": 0.116,
+      "step": 522
+    },
+    {
+      "epoch": 0.37105356509400494,
+      "grad_norm": 1.097212192645177,
+      "learning_rate": 2.8527589233535814e-05,
+      "loss": 0.1265,
+      "step": 523
+    },
+    {
+      "epoch": 0.37176303653777937,
+      "grad_norm": 0.9501550373910891,
+      "learning_rate": 2.851997974854014e-05,
+      "loss": 0.1111,
+      "step": 524
+    },
+    {
+      "epoch": 0.37247250798155374,
+      "grad_norm": 2.5055877732516585,
+      "learning_rate": 2.8512351671364715e-05,
+      "loss": 0.1262,
+      "step": 525
+    },
+    {
+      "epoch": 0.3731819794253281,
+      "grad_norm": 1.4302636774870898,
+      "learning_rate": 2.8504705012499393e-05,
+      "loss": 0.1036,
+      "step": 526
+    },
+    {
+      "epoch": 0.3738914508691025,
+      "grad_norm": 1.2276458748509624,
+      "learning_rate": 2.8497039782459575e-05,
+      "loss": 0.1162,
+      "step": 527
+    },
+    {
+      "epoch": 0.3746009223128769,
+      "grad_norm": 0.7137109773755391,
+      "learning_rate": 2.8489355991786215e-05,
+      "loss": 0.0956,
+      "step": 528
+    },
+    {
+      "epoch": 0.3753103937566513,
+      "grad_norm": 1.008888417467383,
+      "learning_rate": 2.8481653651045766e-05,
+      "loss": 0.0871,
+      "step": 529
+    },
+    {
+      "epoch": 0.3760198652004257,
+      "grad_norm": 2.0246132601013116,
+      "learning_rate": 2.847393277083021e-05,
+      "loss": 0.13,
+      "step": 530
+    },
+    {
+      "epoch": 0.37672933664420005,
+      "grad_norm": 2.1011438801827764,
+      "learning_rate": 2.8466193361757024e-05,
+      "loss": 0.1288,
+      "step": 531
+    },
+    {
+      "epoch": 0.3774388080879745,
+      "grad_norm": 1.2671506346777859,
+      "learning_rate": 2.8458435434469146e-05,
+      "loss": 0.1102,
+      "step": 532
+    },
+    {
+      "epoch": 0.37814827953174884,
+      "grad_norm": 0.6555487248726319,
+      "learning_rate": 2.8450658999635006e-05,
+      "loss": 0.1077,
+      "step": 533
+    },
+    {
+      "epoch": 0.3788577509755232,
+      "grad_norm": 1.085286850768489,
+      "learning_rate": 2.844286406794846e-05,
+      "loss": 0.1106,
+      "step": 534
+    },
+    {
+      "epoch": 0.37956722241929763,
+      "grad_norm": 0.9792362061459019,
+      "learning_rate": 2.8435050650128824e-05,
+      "loss": 0.1007,
+      "step": 535
+    },
+    {
+      "epoch": 0.380276693863072,
+      "grad_norm": 1.2797079551443928,
+      "learning_rate": 2.8427218756920812e-05,
+      "loss": 0.0733,
+      "step": 536
+    },
+    {
+      "epoch": 0.3809861653068464,
+      "grad_norm": 0.8833476768907377,
+      "learning_rate": 2.8419368399094566e-05,
+      "loss": 0.1084,
+      "step": 537
+    },
+    {
+      "epoch": 0.3816956367506208,
+      "grad_norm": 1.077460291285719,
+      "learning_rate": 2.8411499587445597e-05,
+      "loss": 0.1078,
+      "step": 538
+    },
+    {
+      "epoch": 0.38240510819439516,
+      "grad_norm": 0.5947485052041173,
+      "learning_rate": 2.840361233279482e-05,
+      "loss": 0.0906,
+      "step": 539
+    },
+    {
+      "epoch": 0.3831145796381696,
+      "grad_norm": 0.5399505172653348,
+      "learning_rate": 2.8395706645988488e-05,
+      "loss": 0.09,
+      "step": 540
+    },
+    {
+      "epoch": 0.38382405108194395,
+      "grad_norm": 0.8633163431177144,
+      "learning_rate": 2.838778253789822e-05,
+      "loss": 0.1071,
+      "step": 541
+    },
+    {
+      "epoch": 0.3845335225257183,
+      "grad_norm": 1.6609420512087403,
+      "learning_rate": 2.837984001942095e-05,
+      "loss": 0.1114,
+      "step": 542
+    },
+    {
+      "epoch": 0.38524299396949274,
+      "grad_norm": 0.7600386944333442,
+      "learning_rate": 2.837187910147894e-05,
+      "loss": 0.0989,
+      "step": 543
+    },
+    {
+      "epoch": 0.3859524654132671,
+      "grad_norm": 1.440240990351525,
+      "learning_rate": 2.8363899795019753e-05,
+      "loss": 0.1065,
+      "step": 544
+    },
+    {
+      "epoch": 0.3866619368570415,
+      "grad_norm": 0.8111575848469673,
+      "learning_rate": 2.8355902111016244e-05,
+      "loss": 0.104,
+      "step": 545
+    },
+    {
+      "epoch": 0.3873714083008159,
+      "grad_norm": 0.5681118067015966,
+      "learning_rate": 2.8347886060466523e-05,
+      "loss": 0.1151,
+      "step": 546
+    },
+    {
+      "epoch": 0.38808087974459027,
+      "grad_norm": 1.7155617220772919,
+      "learning_rate": 2.8339851654393978e-05,
+      "loss": 0.1089,
+      "step": 547
+    },
+    {
+      "epoch": 0.3887903511883647,
+      "grad_norm": 1.7465767462695343,
+      "learning_rate": 2.833179890384722e-05,
+      "loss": 0.1214,
+      "step": 548
+    },
+    {
+      "epoch": 0.38949982263213906,
+      "grad_norm": 0.5932015328688405,
+      "learning_rate": 2.832372781990011e-05,
+      "loss": 0.1173,
+      "step": 549
+    },
+    {
+      "epoch": 0.3902092940759134,
+      "grad_norm": 1.4005153635349499,
+      "learning_rate": 2.831563841365169e-05,
+      "loss": 0.1077,
+      "step": 550
     }
   ],
   "logging_steps": 1,
@@ -3527,7 +3877,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 324169851666432.0,
+  "total_flos": 356303432417280.0,
   "train_batch_size": 10,
   "trial_name": null,
   "trial_params": null