Training in progress, step 750, checkpoint

last-checkpoint/global_step750/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a24ec387b10f986e3039043c899629bd837aede38e76cfa16d22d14a44275f4
+size 12470656801

last-checkpoint/global_step750/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2aca23671f7a85c2ab8e76d680827f89196b775ea0ad179e9e8da55423fea09d
+size 12470656801

last-checkpoint/global_step750/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45e05262d2bf58ecbfc142ba26b4fa65bd483bc37177b0c1ade13364a69b8ea8
+size 12470656801

last-checkpoint/global_step750/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8389e008699585ab5ea39988ba40db6a7fe99dfdc2c9f19ac2b6f37b4615258d
+size 12470656801

last-checkpoint/global_step750/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6708265374bf17d41e2ac76329e92882fd7afb9b42749d9ae9a8e71cfe41852d
+size 12470656801

last-checkpoint/global_step750/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33bff7754e46806eb390e25295d06be96286e08efd9219f6d8083f9134397777
+size 12470656801

last-checkpoint/global_step750/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b563bb23266e86b8efd3b5c15542e63c947838a5f9b1888497639ae9c9fd3ed
+size 12470656801

last-checkpoint/global_step750/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:652ae1cf37176f87179b338a550d93c274e3078c03246fb4b06b785c048c5b9d
+size 12470656801

last-checkpoint/global_step750/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0302b64fc01a970da11adadde883b847c22fd8873a93e0a20ce9c74f4195fec
+size 113836509

last-checkpoint/global_step750/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:060ed2d9826cd9d248daf444ac38ffb59841479cbd325838e6e35b1544e6aea1
+size 113836509

last-checkpoint/global_step750/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3d53234c9245acaec0504a8f769aae0d95e7d3d9073f23bc551bbf1df545456
+size 113836509

last-checkpoint/global_step750/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdf8331c46efd3233c794314e165b8055410f1df9abed1d64aedfe53c2c97eaa
+size 113836509

last-checkpoint/global_step750/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfe93ac910987f9b2694b4688d2ec0aa908027b97c8a0225eddbac17a1d5a93e
+size 113836509

last-checkpoint/global_step750/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70f16560c1b2271acdb1cbe15576945e806af97fd9099a2ebe37506d2f6a1e57
+size 113836509

last-checkpoint/global_step750/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c3281fa7ad30efa2ff0b24f2b4e5fea97fbc02e8e3167703f0d55216d83019c
+size 113836509

last-checkpoint/global_step750/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b480eb78c0967304036cd9ee8bfe918a19ad93d576c77593717ed81f7452a504
+size 113836509

last-checkpoint/latest

@@ -1 +1 @@
-global_step700
+global_step750

last-checkpoint/model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c97d58f1e84669748171a823447b74e7b2974d3ae2bab8b751dd97f86148c27
+oid sha256:260b07946f2c8271b4d7b30d6358037c2efc5d127a231c41f7e0339384f4cefb
 size 4998056552

last-checkpoint/model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7a2bc2c62026c3bf6c3d9fe2c3e81aa17cee157b001b7d91c09bef760cce1835
+oid sha256:bd6d9821aa9d0467bb723cfa7c535ba3c37acc58d8b548610026484a35343f9f
 size 4915962464

last-checkpoint/model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c4817478a7f48b3764045b1f047b5002f086931326e6d603d22e08ead8faeb7a
+oid sha256:c425108414329a4e869f0890b27a3b37507b20efab678045e74d2ffc6f06c49e
 size 4915962496

last-checkpoint/model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a1c0177e62308d8b31af9df0528eecbd98dfee1d7f1de43314f049c33e54d138
+oid sha256:ecff6a0c47fc97401f94883daf355a75fb772b942031fa92a60e6544f0370927
 size 2704357976

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:862a68c21eac03c04db020298f10e9079dea2d04447e80ccc9e3f82f7f2e815c
+oid sha256:459ff2213c1cbe9ba261ee169e5dc566603cdf28d36f75c0a4b72c4dbdff5687
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.4966300106420717,
+  "epoch": 0.5321035828307911,
   "eval_steps": 500,
-  "global_step": 700,
+  "global_step": 750,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4908,6 +4908,356 @@
       "learning_rate": 2.690163849865296e-05,
       "loss": 0.0706,
       "step": 700
+    },
+    {
+      "epoch": 0.49733948208584605,
+      "grad_norm": 2.098368571271643,
+      "learning_rate": 2.6890924186363463e-05,
+      "loss": 0.0871,
+      "step": 701
+    },
+    {
+      "epoch": 0.4980489535296204,
+      "grad_norm": 1.5229138474279507,
+      "learning_rate": 2.6880193522111622e-05,
+      "loss": 0.1033,
+      "step": 702
+    },
+    {
+      "epoch": 0.49875842497339484,
+      "grad_norm": 2.530414208783178,
+      "learning_rate": 2.6869446520653842e-05,
+      "loss": 0.1041,
+      "step": 703
+    },
+    {
+      "epoch": 0.4994678964171692,
+      "grad_norm": 1.489347416927812,
+      "learning_rate": 2.6858683196769004e-05,
+      "loss": 0.0967,
+      "step": 704
+    },
+    {
+      "epoch": 0.5001773678609436,
+      "grad_norm": 1.04396886001101,
+      "learning_rate": 2.684790356525844e-05,
+      "loss": 0.118,
+      "step": 705
+    },
+    {
+      "epoch": 0.500886839304718,
+      "grad_norm": 2.593904907774733,
+      "learning_rate": 2.6837107640945904e-05,
+      "loss": 0.0879,
+      "step": 706
+    },
+    {
+      "epoch": 0.5015963107484924,
+      "grad_norm": 1.866562579139544,
+      "learning_rate": 2.6826295438677554e-05,
+      "loss": 0.1339,
+      "step": 707
+    },
+    {
+      "epoch": 0.5023057821922667,
+      "grad_norm": 1.5369853576336128,
+      "learning_rate": 2.681546697332193e-05,
+      "loss": 0.098,
+      "step": 708
+    },
+    {
+      "epoch": 0.5030152536360412,
+      "grad_norm": 1.4868926078314282,
+      "learning_rate": 2.6804622259769942e-05,
+      "loss": 0.1104,
+      "step": 709
+    },
+    {
+      "epoch": 0.5037247250798156,
+      "grad_norm": 0.8181632030731728,
+      "learning_rate": 2.6793761312934843e-05,
+      "loss": 0.0992,
+      "step": 710
+    },
+    {
+      "epoch": 0.5044341965235899,
+      "grad_norm": 0.43046680958879047,
+      "learning_rate": 2.6782884147752206e-05,
+      "loss": 0.0998,
+      "step": 711
+    },
+    {
+      "epoch": 0.5051436679673643,
+      "grad_norm": 0.9321564299679312,
+      "learning_rate": 2.6771990779179916e-05,
+      "loss": 0.0927,
+      "step": 712
+    },
+    {
+      "epoch": 0.5058531394111387,
+      "grad_norm": 0.8541355314610722,
+      "learning_rate": 2.6761081222198126e-05,
+      "loss": 0.0797,
+      "step": 713
+    },
+    {
+      "epoch": 0.506562610854913,
+      "grad_norm": 1.0433472198749454,
+      "learning_rate": 2.675015549180926e-05,
+      "loss": 0.072,
+      "step": 714
+    },
+    {
+      "epoch": 0.5072720822986875,
+      "grad_norm": 0.5203971876763894,
+      "learning_rate": 2.673921360303798e-05,
+      "loss": 0.0695,
+      "step": 715
+    },
+    {
+      "epoch": 0.5079815537424619,
+      "grad_norm": 1.6724417277230796,
+      "learning_rate": 2.6728255570931174e-05,
+      "loss": 0.0919,
+      "step": 716
+    },
+    {
+      "epoch": 0.5086910251862362,
+      "grad_norm": 0.9265002936651544,
+      "learning_rate": 2.6717281410557924e-05,
+      "loss": 0.0806,
+      "step": 717
+    },
+    {
+      "epoch": 0.5094004966300106,
+      "grad_norm": 1.123291846804393,
+      "learning_rate": 2.6706291137009488e-05,
+      "loss": 0.1025,
+      "step": 718
+    },
+    {
+      "epoch": 0.5101099680737851,
+      "grad_norm": 1.7832198327424,
+      "learning_rate": 2.669528476539929e-05,
+      "loss": 0.101,
+      "step": 719
+    },
+    {
+      "epoch": 0.5108194395175594,
+      "grad_norm": 0.9549248781537912,
+      "learning_rate": 2.668426231086289e-05,
+      "loss": 0.108,
+      "step": 720
+    },
+    {
+      "epoch": 0.5115289109613338,
+      "grad_norm": 0.8002956332109035,
+      "learning_rate": 2.667322378855796e-05,
+      "loss": 0.0744,
+      "step": 721
+    },
+    {
+      "epoch": 0.5122383824051082,
+      "grad_norm": 1.411775937448295,
+      "learning_rate": 2.6662169213664273e-05,
+      "loss": 0.1029,
+      "step": 722
+    },
+    {
+      "epoch": 0.5129478538488825,
+      "grad_norm": 3.0358142810774416,
+      "learning_rate": 2.665109860138367e-05,
+      "loss": 0.0904,
+      "step": 723
+    },
+    {
+      "epoch": 0.513657325292657,
+      "grad_norm": 1.4618254132636626,
+      "learning_rate": 2.6640011966940055e-05,
+      "loss": 0.1209,
+      "step": 724
+    },
+    {
+      "epoch": 0.5143667967364314,
+      "grad_norm": 1.9699019369734254,
+      "learning_rate": 2.6628909325579363e-05,
+      "loss": 0.1032,
+      "step": 725
+    },
+    {
+      "epoch": 0.5150762681802058,
+      "grad_norm": 1.1561104662443713,
+      "learning_rate": 2.6617790692569534e-05,
+      "loss": 0.126,
+      "step": 726
+    },
+    {
+      "epoch": 0.5157857396239801,
+      "grad_norm": 1.226127283924977,
+      "learning_rate": 2.6606656083200503e-05,
+      "loss": 0.0709,
+      "step": 727
+    },
+    {
+      "epoch": 0.5164952110677545,
+      "grad_norm": 1.0454229251540932,
+      "learning_rate": 2.6595505512784175e-05,
+      "loss": 0.0877,
+      "step": 728
+    },
+    {
+      "epoch": 0.517204682511529,
+      "grad_norm": 0.7152619511168274,
+      "learning_rate": 2.6584338996654412e-05,
+      "loss": 0.0914,
+      "step": 729
+    },
+    {
+      "epoch": 0.5179141539553033,
+      "grad_norm": 0.7678687910268428,
+      "learning_rate": 2.657315655016699e-05,
+      "loss": 0.1048,
+      "step": 730
+    },
+    {
+      "epoch": 0.5186236253990777,
+      "grad_norm": 0.5302101019819314,
+      "learning_rate": 2.6561958188699604e-05,
+      "loss": 0.0959,
+      "step": 731
+    },
+    {
+      "epoch": 0.5193330968428521,
+      "grad_norm": 0.6005287656785708,
+      "learning_rate": 2.6550743927651826e-05,
+      "loss": 0.0745,
+      "step": 732
+    },
+    {
+      "epoch": 0.5200425682866264,
+      "grad_norm": 1.0619878502071893,
+      "learning_rate": 2.6539513782445097e-05,
+      "loss": 0.0771,
+      "step": 733
+    },
+    {
+      "epoch": 0.5207520397304009,
+      "grad_norm": 0.6249804070305147,
+      "learning_rate": 2.6528267768522698e-05,
+      "loss": 0.1064,
+      "step": 734
+    },
+    {
+      "epoch": 0.5214615111741753,
+      "grad_norm": 1.4627916616815637,
+      "learning_rate": 2.651700590134974e-05,
+      "loss": 0.0856,
+      "step": 735
+    },
+    {
+      "epoch": 0.5221709826179496,
+      "grad_norm": 0.814142555962611,
+      "learning_rate": 2.650572819641313e-05,
+      "loss": 0.1258,
+      "step": 736
+    },
+    {
+      "epoch": 0.522880454061724,
+      "grad_norm": 0.9653069902169457,
+      "learning_rate": 2.6494434669221543e-05,
+      "loss": 0.104,
+      "step": 737
+    },
+    {
+      "epoch": 0.5235899255054984,
+      "grad_norm": 0.7096130309147914,
+      "learning_rate": 2.6483125335305435e-05,
+      "loss": 0.0879,
+      "step": 738
+    },
+    {
+      "epoch": 0.5242993969492727,
+      "grad_norm": 1.0493588114817471,
+      "learning_rate": 2.647180021021698e-05,
+      "loss": 0.0855,
+      "step": 739
+    },
+    {
+      "epoch": 0.5250088683930472,
+      "grad_norm": 1.605499799477196,
+      "learning_rate": 2.6460459309530077e-05,
+      "loss": 0.0797,
+      "step": 740
+    },
+    {
+      "epoch": 0.5257183398368216,
+      "grad_norm": 1.4475212015020267,
+      "learning_rate": 2.6449102648840312e-05,
+      "loss": 0.1189,
+      "step": 741
+    },
+    {
+      "epoch": 0.5264278112805959,
+      "grad_norm": 2.6659082823578704,
+      "learning_rate": 2.6437730243764957e-05,
+      "loss": 0.1087,
+      "step": 742
+    },
+    {
+      "epoch": 0.5271372827243703,
+      "grad_norm": 2.5652982985052617,
+      "learning_rate": 2.6426342109942914e-05,
+      "loss": 0.0966,
+      "step": 743
+    },
+    {
+      "epoch": 0.5278467541681447,
+      "grad_norm": 2.5228507629099397,
+      "learning_rate": 2.6414938263034736e-05,
+      "loss": 0.1214,
+      "step": 744
+    },
+    {
+      "epoch": 0.5285562256119192,
+      "grad_norm": 2.1205431984293095,
+      "learning_rate": 2.6403518718722568e-05,
+      "loss": 0.0888,
+      "step": 745
+    },
+    {
+      "epoch": 0.5292656970556935,
+      "grad_norm": 2.2997587183258075,
+      "learning_rate": 2.639208349271015e-05,
+      "loss": 0.1089,
+      "step": 746
+    },
+    {
+      "epoch": 0.5299751684994679,
+      "grad_norm": 3.4591558890395544,
+      "learning_rate": 2.638063260072279e-05,
+      "loss": 0.1135,
+      "step": 747
+    },
+    {
+      "epoch": 0.5306846399432423,
+      "grad_norm": 2.5281951408322207,
+      "learning_rate": 2.6369166058507332e-05,
+      "loss": 0.0764,
+      "step": 748
+    },
+    {
+      "epoch": 0.5313941113870166,
+      "grad_norm": 1.208412137677219,
+      "learning_rate": 2.635768388183214e-05,
+      "loss": 0.0942,
+      "step": 749
+    },
+    {
+      "epoch": 0.5321035828307911,
+      "grad_norm": 2.9435897965503712,
+      "learning_rate": 2.6346186086487082e-05,
+      "loss": 0.1212,
+      "step": 750
     }
   ],
   "logging_steps": 1,
@@ -4927,7 +5277,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 453272461803520.0,
+  "total_flos": 485441167851520.0,
   "train_batch_size": 10,
   "trial_name": null,
   "trial_params": null