Training in progress, step 650, checkpoint

last-checkpoint/global_step650/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e41b218ce0ddc4470005fa07c885754a4bb482096d3220b52ce784589aef29d
+size 12470656801

last-checkpoint/global_step650/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:893d2672da5b8a20a431f3a1b4a9f43b50edc20918ab1858e1625a7f6235bd81
+size 12470656801

last-checkpoint/global_step650/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d831902424db0e3df29bc9714fcd605384bc874f799aadded58797b8c2428280
+size 12470656801

last-checkpoint/global_step650/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74d572fa784625201da5c0620ea9df0806845fc3002e9ca9652aded5fa3b632f
+size 12470656801

last-checkpoint/global_step650/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a13adab6d311109517a8fc88adfc3eb2a5fcd77f09c2573d8571dbf4b80ae357
+size 12470656801

last-checkpoint/global_step650/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62bbfbfc380afbc7ba9b0e123c232f3cb98f42bb550a8895139a780d2b8f305d
+size 12470656801

last-checkpoint/global_step650/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b834ab9acadc5ba0b6b3dcc320b278232fbb91dfb2e7c7864a0eb493190bccd2
+size 12470656801

last-checkpoint/global_step650/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cd416f0346afd8bc722fbb14c8931f6119001e557563584ed146ec33d5eb4dc
+size 12470656801

last-checkpoint/global_step650/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91ce535b28010df3784323c73e9f390e19499a6b44e0986a0b0b76d56b794ace
+size 113836509

last-checkpoint/global_step650/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e11e25c758bf257dacf0cff2f550045c49ed91726f203084bdf1ede83611f146
+size 113836509

last-checkpoint/global_step650/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a87424ff537b85edeb64c8a3ebed56341ef6c117e5c2219c5a26fd1c02f4eb13
+size 113836509

last-checkpoint/global_step650/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04a028264d3b4b831ed21c0634c779a50d5d0bbf4d04592db9e043f721c9fddf
+size 113836509

last-checkpoint/global_step650/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9430d926e4e93e56108904f2e322a7b93fbf98f85f73d9dea17fe38c7a707187
+size 113836509

last-checkpoint/global_step650/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:269b3880aa62456cf99ab67c9763f9ed6f779f6f28048e12f55f62799d3b70c2
+size 113836509

last-checkpoint/global_step650/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:faadb13fac06e7be6fb37b32362b786b45d4bdc376e51f5975a7bd3447307605
+size 113836509

last-checkpoint/global_step650/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9bee828e7046efddcee38643c2f93304d8b25315a216422608397c8fff3459a
+size 113836509

last-checkpoint/latest

@@ -1 +1 @@
-global_step600
+global_step650

last-checkpoint/model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7800117170cf82b55257c1707adcef8daad710f7cb64de8a9ad8e0e76049a110
+oid sha256:7958e836b23cf31719308fe79758225d978ff61c68f5c6ba42a95c48bc70c035
 size 4998056552

last-checkpoint/model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d33bdc13a5ab594e1d6d5d1093fd63f65f19a5bb3833eb85415983802d819cc
+oid sha256:6b7e8ac6567f7eac4e66e8a7eb5f188d774a92150b00b266a4f655eb1b13209b
 size 4915962464

last-checkpoint/model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ecd844151e6b7f9c65595211ee5132ae0ccdacc12a451cd1dcb4e1f1ea33c3a5
+oid sha256:947059082cff5723bcf076a471750bd857f06f1c8a4de114d661f7c302cc9876
 size 4915962496

last-checkpoint/model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fcf99ddf29be9984cb6fe02fbb76153e12808be74b2fb9c543fe45de1e9a0966
+oid sha256:86987bb479f199b019cbda9bc40b08a3fe401649c06d9a4d3a4ba165fb11d4ce
 size 2704357976

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:11c1dce72fa355171a43b7faada7f2e5ee1b14ef3b7e886de17666de4ecebe3d
+oid sha256:5b5ec1deb884fff4c0d9237f0a130431c015a8d00aa4cd5940f16aea4840bcfd
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.42568286626463286,
+  "epoch": 0.46115643845335225,
   "eval_steps": 500,
-  "global_step": 600,
+  "global_step": 650,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4208,6 +4208,356 @@
       "learning_rate": 2.7888059789632228e-05,
       "loss": 0.1286,
       "step": 600
+    },
+    {
+      "epoch": 0.42639233770840723,
+      "grad_norm": 3.732480001308575,
+      "learning_rate": 2.787905124903544e-05,
+      "loss": 0.1133,
+      "step": 601
+    },
+    {
+      "epoch": 0.4271018091521816,
+      "grad_norm": 1.182456635337205,
+      "learning_rate": 2.7870024997640245e-05,
+      "loss": 0.0954,
+      "step": 602
+    },
+    {
+      "epoch": 0.427811280595956,
+      "grad_norm": 0.8826676596585253,
+      "learning_rate": 2.7860981047859206e-05,
+      "loss": 0.0982,
+      "step": 603
+    },
+    {
+      "epoch": 0.4285207520397304,
+      "grad_norm": 1.3708148431705451,
+      "learning_rate": 2.785191941212924e-05,
+      "loss": 0.1016,
+      "step": 604
+    },
+    {
+      "epoch": 0.4292302234835048,
+      "grad_norm": 1.9938187159664562,
+      "learning_rate": 2.784284010291157e-05,
+      "loss": 0.0952,
+      "step": 605
+    },
+    {
+      "epoch": 0.4299396949272792,
+      "grad_norm": 2.549694179989554,
+      "learning_rate": 2.7833743132691728e-05,
+      "loss": 0.0986,
+      "step": 606
+    },
+    {
+      "epoch": 0.43064916637105355,
+      "grad_norm": 1.273947499594232,
+      "learning_rate": 2.7824628513979543e-05,
+      "loss": 0.1046,
+      "step": 607
+    },
+    {
+      "epoch": 0.43135863781482797,
+      "grad_norm": 0.6480794756125879,
+      "learning_rate": 2.7815496259309088e-05,
+      "loss": 0.0965,
+      "step": 608
+    },
+    {
+      "epoch": 0.43206810925860234,
+      "grad_norm": 1.322629548390438,
+      "learning_rate": 2.7806346381238716e-05,
+      "loss": 0.097,
+      "step": 609
+    },
+    {
+      "epoch": 0.4327775807023767,
+      "grad_norm": 1.8914047859341188,
+      "learning_rate": 2.7797178892350997e-05,
+      "loss": 0.1121,
+      "step": 610
+    },
+    {
+      "epoch": 0.43348705214615113,
+      "grad_norm": 1.6872304604984099,
+      "learning_rate": 2.778799380525273e-05,
+      "loss": 0.1122,
+      "step": 611
+    },
+    {
+      "epoch": 0.4341965235899255,
+      "grad_norm": 1.4023364845021251,
+      "learning_rate": 2.7778791132574908e-05,
+      "loss": 0.0937,
+      "step": 612
+    },
+    {
+      "epoch": 0.43490599503369987,
+      "grad_norm": 1.3432790039581282,
+      "learning_rate": 2.776957088697271e-05,
+      "loss": 0.095,
+      "step": 613
+    },
+    {
+      "epoch": 0.4356154664774743,
+      "grad_norm": 1.6450219360687317,
+      "learning_rate": 2.776033308112547e-05,
+      "loss": 0.1179,
+      "step": 614
+    },
+    {
+      "epoch": 0.43632493792124866,
+      "grad_norm": 1.4925111555843509,
+      "learning_rate": 2.775107772773669e-05,
+      "loss": 0.121,
+      "step": 615
+    },
+    {
+      "epoch": 0.4370344093650231,
+      "grad_norm": 0.6091261752379071,
+      "learning_rate": 2.7741804839533995e-05,
+      "loss": 0.1124,
+      "step": 616
+    },
+    {
+      "epoch": 0.43774388080879745,
+      "grad_norm": 1.3348260825825455,
+      "learning_rate": 2.773251442926911e-05,
+      "loss": 0.1123,
+      "step": 617
+    },
+    {
+      "epoch": 0.4384533522525718,
+      "grad_norm": 1.4050404684276534,
+      "learning_rate": 2.772320650971787e-05,
+      "loss": 0.1152,
+      "step": 618
+    },
+    {
+      "epoch": 0.43916282369634624,
+      "grad_norm": 1.5497208983174737,
+      "learning_rate": 2.7713881093680184e-05,
+      "loss": 0.1038,
+      "step": 619
+    },
+    {
+      "epoch": 0.4398722951401206,
+      "grad_norm": 2.9245270126728498,
+      "learning_rate": 2.7704538193980024e-05,
+      "loss": 0.1302,
+      "step": 620
+    },
+    {
+      "epoch": 0.440581766583895,
+      "grad_norm": 1.2975316073853007,
+      "learning_rate": 2.76951778234654e-05,
+      "loss": 0.0888,
+      "step": 621
+    },
+    {
+      "epoch": 0.4412912380276694,
+      "grad_norm": 1.1457202601732912,
+      "learning_rate": 2.7685799995008355e-05,
+      "loss": 0.0892,
+      "step": 622
+    },
+    {
+      "epoch": 0.44200070947144376,
+      "grad_norm": 0.5443262686739241,
+      "learning_rate": 2.7676404721504927e-05,
+      "loss": 0.111,
+      "step": 623
+    },
+    {
+      "epoch": 0.4427101809152182,
+      "grad_norm": 1.141138130325628,
+      "learning_rate": 2.7666992015875152e-05,
+      "loss": 0.1077,
+      "step": 624
+    },
+    {
+      "epoch": 0.44341965235899256,
+      "grad_norm": 1.8625071274478626,
+      "learning_rate": 2.7657561891063042e-05,
+      "loss": 0.1246,
+      "step": 625
+    },
+    {
+      "epoch": 0.4441291238027669,
+      "grad_norm": 0.3468592158620779,
+      "learning_rate": 2.7648114360036557e-05,
+      "loss": 0.0762,
+      "step": 626
+    },
+    {
+      "epoch": 0.44483859524654135,
+      "grad_norm": 0.9503409611669715,
+      "learning_rate": 2.7638649435787596e-05,
+      "loss": 0.0858,
+      "step": 627
+    },
+    {
+      "epoch": 0.4455480666903157,
+      "grad_norm": 2.743854441330861,
+      "learning_rate": 2.7629167131331967e-05,
+      "loss": 0.1308,
+      "step": 628
+    },
+    {
+      "epoch": 0.4462575381340901,
+      "grad_norm": 0.6940548708286597,
+      "learning_rate": 2.7619667459709394e-05,
+      "loss": 0.1041,
+      "step": 629
+    },
+    {
+      "epoch": 0.4469670095778645,
+      "grad_norm": 0.8457511365226246,
+      "learning_rate": 2.7610150433983482e-05,
+      "loss": 0.12,
+      "step": 630
+    },
+    {
+      "epoch": 0.4476764810216389,
+      "grad_norm": 0.44706855167796616,
+      "learning_rate": 2.7600616067241682e-05,
+      "loss": 0.0776,
+      "step": 631
+    },
+    {
+      "epoch": 0.44838595246541324,
+      "grad_norm": 2.21544742425388,
+      "learning_rate": 2.7591064372595315e-05,
+      "loss": 0.1178,
+      "step": 632
+    },
+    {
+      "epoch": 0.44909542390918766,
+      "grad_norm": 0.8699432131758461,
+      "learning_rate": 2.758149536317952e-05,
+      "loss": 0.0758,
+      "step": 633
+    },
+    {
+      "epoch": 0.44980489535296203,
+      "grad_norm": 1.2955684825232407,
+      "learning_rate": 2.7571909052153242e-05,
+      "loss": 0.1148,
+      "step": 634
+    },
+    {
+      "epoch": 0.45051436679673645,
+      "grad_norm": 1.6574835544271427,
+      "learning_rate": 2.756230545269923e-05,
+      "loss": 0.0682,
+      "step": 635
+    },
+    {
+      "epoch": 0.4512238382405108,
+      "grad_norm": 0.8770082079938105,
+      "learning_rate": 2.7552684578024e-05,
+      "loss": 0.1117,
+      "step": 636
+    },
+    {
+      "epoch": 0.4519333096842852,
+      "grad_norm": 1.6069917385521764,
+      "learning_rate": 2.7543046441357816e-05,
+      "loss": 0.0762,
+      "step": 637
+    },
+    {
+      "epoch": 0.4526427811280596,
+      "grad_norm": 0.7516135636403152,
+      "learning_rate": 2.7533391055954703e-05,
+      "loss": 0.0807,
+      "step": 638
+    },
+    {
+      "epoch": 0.453352252571834,
+      "grad_norm": 0.6562967000937064,
+      "learning_rate": 2.752371843509239e-05,
+      "loss": 0.0951,
+      "step": 639
+    },
+    {
+      "epoch": 0.45406172401560835,
+      "grad_norm": 0.656113015480493,
+      "learning_rate": 2.7514028592072304e-05,
+      "loss": 0.0989,
+      "step": 640
+    },
+    {
+      "epoch": 0.45477119545938277,
+      "grad_norm": 1.0869205414112095,
+      "learning_rate": 2.7504321540219566e-05,
+      "loss": 0.0854,
+      "step": 641
+    },
+    {
+      "epoch": 0.45548066690315714,
+      "grad_norm": 1.0508451584907033,
+      "learning_rate": 2.7494597292882957e-05,
+      "loss": 0.1365,
+      "step": 642
+    },
+    {
+      "epoch": 0.45619013834693156,
+      "grad_norm": 0.6067761335725533,
+      "learning_rate": 2.7484855863434902e-05,
+      "loss": 0.0915,
+      "step": 643
+    },
+    {
+      "epoch": 0.45689960979070593,
+      "grad_norm": 0.5291156473760684,
+      "learning_rate": 2.7475097265271463e-05,
+      "loss": 0.093,
+      "step": 644
+    },
+    {
+      "epoch": 0.4576090812344803,
+      "grad_norm": 0.8052577911588888,
+      "learning_rate": 2.74653215118123e-05,
+      "loss": 0.0876,
+      "step": 645
+    },
+    {
+      "epoch": 0.4583185526782547,
+      "grad_norm": 0.7666571788994913,
+      "learning_rate": 2.745552861650067e-05,
+      "loss": 0.0778,
+      "step": 646
+    },
+    {
+      "epoch": 0.4590280241220291,
+      "grad_norm": 0.5675264353523128,
+      "learning_rate": 2.7445718592803408e-05,
+      "loss": 0.1117,
+      "step": 647
+    },
+    {
+      "epoch": 0.45973749556580346,
+      "grad_norm": 0.7353816580307925,
+      "learning_rate": 2.7435891454210892e-05,
+      "loss": 0.0877,
+      "step": 648
+    },
+    {
+      "epoch": 0.4604469670095779,
+      "grad_norm": 0.67985377504387,
+      "learning_rate": 2.742604721423705e-05,
+      "loss": 0.11,
+      "step": 649
+    },
+    {
+      "epoch": 0.46115643845335225,
+      "grad_norm": 0.7259751686604848,
+      "learning_rate": 2.7416185886419314e-05,
+      "loss": 0.1028,
+      "step": 650
     }
   ],
   "logging_steps": 1,
@@ -4227,7 +4577,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 388252513599488.0,
+  "total_flos": 420934415777792.0,
   "train_batch_size": 10,
   "trial_name": null,
   "trial_params": null