Training in progress, step 900, checkpoint

last-checkpoint/global_step900/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19832f503bf2234588660a0334a158cb204c890bc7ec3f00dc72415b94e83d5c
+size 12470656801

last-checkpoint/global_step900/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b156833371d912d314ab1e047ae6c9e451ad7f80b819a5b7a922335c99ecb7b2
+size 12470656801

last-checkpoint/global_step900/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a600607fa3442e498db21d1a8067c5bda17b6698da37205f6d953fda81daa61
+size 12470656801

last-checkpoint/global_step900/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd866f50174d02af807d95e93fe7a131704e9b84099dce02a4b9880882aed8e7
+size 12470656801

last-checkpoint/global_step900/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb95ce49f002df165a6d2c606ebfaf56349856a2478ef42f9d5cd09070667feb
+size 12470656801

last-checkpoint/global_step900/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26d30bd0d45b673e1304e167c3c74f18da2a4fea8c56e5122c9d590434c0dd09
+size 12470656801

last-checkpoint/global_step900/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:444ecff97517eda678705a5f4f0322e7dbb46f7427f2246a63d9c0e26bd45572
+size 12470656801

last-checkpoint/global_step900/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45fbc9f0d7a0a047d9aa885c7b492b0cf018bc29ac0b6bff5fde6b77658db135
+size 12470656801

last-checkpoint/global_step900/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e320b7dab2a331da99d1fb6459f5fda65b5743c7582804950916544dd38c315
+size 113836509

last-checkpoint/global_step900/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d29b774b35e88ba561f2078c1722b12a593302b0bd3cadeb3e64192de489d4a1
+size 113836509

last-checkpoint/global_step900/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc469c2fcbed507e62d1c7890af30f5a89614700118289aa2b1bbedb50f787f0
+size 113836509

last-checkpoint/global_step900/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ad8cf499d5cb87d0f0cc8fc6b882d551d8f6dbe27000e4b52d512546d274180
+size 113836509

last-checkpoint/global_step900/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e67f11d1534fc00ad80092167a1f8688cb9c9d768458dd72dbf2e2e094b63ea
+size 113836509

last-checkpoint/global_step900/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:748b915e4a740ee308c1f96206d92ac1bad287e5e991ee29f0247e5dc3ea0ff6
+size 113836509

last-checkpoint/global_step900/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae06e677384d80c9062c142bdeb9b7f5c4de032a8d63f5a401d1a7e21395852d
+size 113836509

last-checkpoint/global_step900/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbf84c986f8c6566088f32b2ac4572d98993e1d4f0795854feee512e0c1a057f
+size 113836509

last-checkpoint/latest

@@ -1 +1 @@
-global_step850
+global_step900

last-checkpoint/model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6b4c17dc3982de6d3ce3f41d026885be995dd68d1610ff03d326a7667d3e060
+oid sha256:948ec6fd3d70461cf15f2868d89e2113fdb08df104a010a1947cff17b798e5e2
 size 4998056552

last-checkpoint/model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:712e3cb88681979a7bd7156459cc3f1459c4f5b93b276b486729ac733fa2a9c0
+oid sha256:ce86dd0e3769834d190bf6a79efd805f0ab8e42e0524a8771b73a81597c686a5
 size 4915962464

last-checkpoint/model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:57883a12b5b42f730afd6ea2263ec372b1335f9463e63400bc287ff743535647
+oid sha256:3ecfebf2ad1b0abf38fc2a26363a5ea26ebaabc8af14341227776b6ef4513e31
 size 4915962496

last-checkpoint/model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b391dc9ef7f2e16ea65925e027cbf1685ccd2076749caf90f46e1232af380d8
+oid sha256:707f01124b671b7f6c3abd698074bc1f11fea78f962affdd299a2392a26b40a2
 size 2704357976

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c102b5e9b5d45400f6ab7c948a4ca1d1d779719f7570c858a3c0cba8df1b44a
+oid sha256:c360ee3ef7035c853e37afaed5ae2d209259ad4648b63c90c4f1e90e8f8673d5
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.6030507272082298,
+  "epoch": 0.6385242993969493,
   "eval_steps": 500,
-  "global_step": 850,
+  "global_step": 900,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5958,6 +5958,356 @@
       "learning_rate": 2.5120336405577152e-05,
       "loss": 0.0839,
       "step": 850
+    },
+    {
+      "epoch": 0.6037601986520043,
+      "grad_norm": 1.0624547704910516,
+      "learning_rate": 2.510734615795769e-05,
+      "loss": 0.1098,
+      "step": 851
+    },
+    {
+      "epoch": 0.6044696700957787,
+      "grad_norm": 1.7695216750776055,
+      "learning_rate": 2.5094342011086902e-05,
+      "loss": 0.0917,
+      "step": 852
+    },
+    {
+      "epoch": 0.605179141539553,
+      "grad_norm": 2.0173813017738267,
+      "learning_rate": 2.5081323982847602e-05,
+      "loss": 0.1065,
+      "step": 853
+    },
+    {
+      "epoch": 0.6058886129833274,
+      "grad_norm": 1.452417196311744,
+      "learning_rate": 2.506829209114171e-05,
+      "loss": 0.0627,
+      "step": 854
+    },
+    {
+      "epoch": 0.6065980844271018,
+      "grad_norm": 0.8805815316368366,
+      "learning_rate": 2.50552463538902e-05,
+      "loss": 0.0644,
+      "step": 855
+    },
+    {
+      "epoch": 0.6073075558708761,
+      "grad_norm": 1.987018699939349,
+      "learning_rate": 2.5042186789033095e-05,
+      "loss": 0.1173,
+      "step": 856
+    },
+    {
+      "epoch": 0.6080170273146506,
+      "grad_norm": 1.3761985983204825,
+      "learning_rate": 2.502911341452943e-05,
+      "loss": 0.0886,
+      "step": 857
+    },
+    {
+      "epoch": 0.608726498758425,
+      "grad_norm": 0.8421769441751846,
+      "learning_rate": 2.5016026248357223e-05,
+      "loss": 0.0609,
+      "step": 858
+    },
+    {
+      "epoch": 0.6094359702021993,
+      "grad_norm": 0.787038329096824,
+      "learning_rate": 2.5002925308513475e-05,
+      "loss": 0.1069,
+      "step": 859
+    },
+    {
+      "epoch": 0.6101454416459737,
+      "grad_norm": 1.5520753898161679,
+      "learning_rate": 2.498981061301411e-05,
+      "loss": 0.0954,
+      "step": 860
+    },
+    {
+      "epoch": 0.6108549130897482,
+      "grad_norm": 0.4868249284046237,
+      "learning_rate": 2.4976682179893974e-05,
+      "loss": 0.0844,
+      "step": 861
+    },
+    {
+      "epoch": 0.6115643845335226,
+      "grad_norm": 0.5289063830611047,
+      "learning_rate": 2.4963540027206812e-05,
+      "loss": 0.1219,
+      "step": 862
+    },
+    {
+      "epoch": 0.6122738559772969,
+      "grad_norm": 1.396962891016018,
+      "learning_rate": 2.4950384173025226e-05,
+      "loss": 0.0716,
+      "step": 863
+    },
+    {
+      "epoch": 0.6129833274210713,
+      "grad_norm": 1.2577692839362375,
+      "learning_rate": 2.4937214635440665e-05,
+      "loss": 0.0946,
+      "step": 864
+    },
+    {
+      "epoch": 0.6136927988648457,
+      "grad_norm": 0.6086722079399003,
+      "learning_rate": 2.492403143256339e-05,
+      "loss": 0.0869,
+      "step": 865
+    },
+    {
+      "epoch": 0.61440227030862,
+      "grad_norm": 0.9960610420622015,
+      "learning_rate": 2.4910834582522466e-05,
+      "loss": 0.1012,
+      "step": 866
+    },
+    {
+      "epoch": 0.6151117417523945,
+      "grad_norm": 1.1862531122799593,
+      "learning_rate": 2.489762410346571e-05,
+      "loss": 0.0631,
+      "step": 867
+    },
+    {
+      "epoch": 0.6158212131961689,
+      "grad_norm": 1.7734611678957337,
+      "learning_rate": 2.4884400013559688e-05,
+      "loss": 0.0984,
+      "step": 868
+    },
+    {
+      "epoch": 0.6165306846399432,
+      "grad_norm": 3.96825830012531,
+      "learning_rate": 2.487116233098969e-05,
+      "loss": 0.1226,
+      "step": 869
+    },
+    {
+      "epoch": 0.6172401560837176,
+      "grad_norm": 2.5935895958927286,
+      "learning_rate": 2.4857911073959676e-05,
+      "loss": 0.1,
+      "step": 870
+    },
+    {
+      "epoch": 0.617949627527492,
+      "grad_norm": 0.64578715947487,
+      "learning_rate": 2.4844646260692304e-05,
+      "loss": 0.064,
+      "step": 871
+    },
+    {
+      "epoch": 0.6186590989712664,
+      "grad_norm": 0.7064371198186336,
+      "learning_rate": 2.483136790942885e-05,
+      "loss": 0.0867,
+      "step": 872
+    },
+    {
+      "epoch": 0.6193685704150408,
+      "grad_norm": 0.6456220234365463,
+      "learning_rate": 2.4818076038429214e-05,
+      "loss": 0.0626,
+      "step": 873
+    },
+    {
+      "epoch": 0.6200780418588152,
+      "grad_norm": 1.5889644535933756,
+      "learning_rate": 2.4804770665971894e-05,
+      "loss": 0.1122,
+      "step": 874
+    },
+    {
+      "epoch": 0.6207875133025895,
+      "grad_norm": 2.36229311624623,
+      "learning_rate": 2.4791451810353948e-05,
+      "loss": 0.1218,
+      "step": 875
+    },
+    {
+      "epoch": 0.621496984746364,
+      "grad_norm": 1.1435066597963754,
+      "learning_rate": 2.4778119489890976e-05,
+      "loss": 0.0536,
+      "step": 876
+    },
+    {
+      "epoch": 0.6222064561901384,
+      "grad_norm": 0.6421460597478851,
+      "learning_rate": 2.4764773722917094e-05,
+      "loss": 0.0822,
+      "step": 877
+    },
+    {
+      "epoch": 0.6229159276339127,
+      "grad_norm": 1.462402586369017,
+      "learning_rate": 2.4751414527784918e-05,
+      "loss": 0.1013,
+      "step": 878
+    },
+    {
+      "epoch": 0.6236253990776871,
+      "grad_norm": 1.2871417789368593,
+      "learning_rate": 2.473804192286551e-05,
+      "loss": 0.0978,
+      "step": 879
+    },
+    {
+      "epoch": 0.6243348705214615,
+      "grad_norm": 0.7449031483151409,
+      "learning_rate": 2.47246559265484e-05,
+      "loss": 0.0737,
+      "step": 880
+    },
+    {
+      "epoch": 0.625044341965236,
+      "grad_norm": 1.7189189405930545,
+      "learning_rate": 2.4711256557241507e-05,
+      "loss": 0.073,
+      "step": 881
+    },
+    {
+      "epoch": 0.6257538134090103,
+      "grad_norm": 1.114608101781195,
+      "learning_rate": 2.4697843833371156e-05,
+      "loss": 0.1015,
+      "step": 882
+    },
+    {
+      "epoch": 0.6264632848527847,
+      "grad_norm": 1.9049891229967852,
+      "learning_rate": 2.4684417773382037e-05,
+      "loss": 0.1025,
+      "step": 883
+    },
+    {
+      "epoch": 0.6271727562965591,
+      "grad_norm": 1.238574438484414,
+      "learning_rate": 2.467097839573717e-05,
+      "loss": 0.0802,
+      "step": 884
+    },
+    {
+      "epoch": 0.6278822277403334,
+      "grad_norm": 1.0114207756336435,
+      "learning_rate": 2.4657525718917887e-05,
+      "loss": 0.093,
+      "step": 885
+    },
+    {
+      "epoch": 0.6285916991841078,
+      "grad_norm": 1.8354185266302383,
+      "learning_rate": 2.4644059761423827e-05,
+      "loss": 0.0949,
+      "step": 886
+    },
+    {
+      "epoch": 0.6293011706278823,
+      "grad_norm": 1.0836390869477819,
+      "learning_rate": 2.463058054177288e-05,
+      "loss": 0.0755,
+      "step": 887
+    },
+    {
+      "epoch": 0.6300106420716566,
+      "grad_norm": 1.066045965499429,
+      "learning_rate": 2.4617088078501163e-05,
+      "loss": 0.1139,
+      "step": 888
+    },
+    {
+      "epoch": 0.630720113515431,
+      "grad_norm": 0.828729187302372,
+      "learning_rate": 2.4603582390163023e-05,
+      "loss": 0.1,
+      "step": 889
+    },
+    {
+      "epoch": 0.6314295849592054,
+      "grad_norm": 1.4089547827862339,
+      "learning_rate": 2.459006349533099e-05,
+      "loss": 0.1028,
+      "step": 890
+    },
+    {
+      "epoch": 0.6321390564029797,
+      "grad_norm": 0.5858434642074677,
+      "learning_rate": 2.4576531412595745e-05,
+      "loss": 0.0591,
+      "step": 891
+    },
+    {
+      "epoch": 0.6328485278467542,
+      "grad_norm": 0.6798521381418388,
+      "learning_rate": 2.4562986160566117e-05,
+      "loss": 0.1005,
+      "step": 892
+    },
+    {
+      "epoch": 0.6335579992905286,
+      "grad_norm": 0.6506463919191321,
+      "learning_rate": 2.4549427757869038e-05,
+      "loss": 0.0821,
+      "step": 893
+    },
+    {
+      "epoch": 0.6342674707343029,
+      "grad_norm": 1.882694465032466,
+      "learning_rate": 2.4535856223149525e-05,
+      "loss": 0.0846,
+      "step": 894
+    },
+    {
+      "epoch": 0.6349769421780773,
+      "grad_norm": 0.6150434826675685,
+      "learning_rate": 2.452227157507065e-05,
+      "loss": 0.0866,
+      "step": 895
+    },
+    {
+      "epoch": 0.6356864136218517,
+      "grad_norm": 1.688939520741653,
+      "learning_rate": 2.4508673832313525e-05,
+      "loss": 0.0699,
+      "step": 896
+    },
+    {
+      "epoch": 0.636395885065626,
+      "grad_norm": 0.7121104706825553,
+      "learning_rate": 2.449506301357727e-05,
+      "loss": 0.0952,
+      "step": 897
+    },
+    {
+      "epoch": 0.6371053565094005,
+      "grad_norm": 1.1626310666894542,
+      "learning_rate": 2.4481439137578977e-05,
+      "loss": 0.0882,
+      "step": 898
+    },
+    {
+      "epoch": 0.6378148279531749,
+      "grad_norm": 0.9853285020494227,
+      "learning_rate": 2.446780222305371e-05,
+      "loss": 0.1004,
+      "step": 899
+    },
+    {
+      "epoch": 0.6385242993969493,
+      "grad_norm": 0.5977362751205011,
+      "learning_rate": 2.445415228875444e-05,
+      "loss": 0.0926,
+      "step": 900
     }
   ],
   "logging_steps": 1,
@@ -5977,7 +6327,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 549584096952320.0,
+  "total_flos": 581717677735936.0,
   "train_batch_size": 10,
   "trial_name": null,
   "trial_params": null