{ "best_metric": 0.27069464325904846, "best_model_checkpoint": "models/llama3.2-3b-dpo-coarse/checkpoint-10000", "epoch": 1.0, "eval_steps": 10000, "global_step": 13291, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 7.523888345496953e-05, "grad_norm": 3.1627285728002064, "learning_rate": 3.7593984962406016e-10, "logits/chosen": -0.4609375, "logits/rejected": -0.04931640625, "logps/chosen": -142.0, "logps/rejected": -164.0, "loss": 0.6914, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 1 }, { "epoch": 0.0007523888345496953, "grad_norm": 2.9481424063745734, "learning_rate": 3.759398496240601e-09, "logits/chosen": -0.796875, "logits/rejected": -0.3828125, "logps/chosen": -175.0, "logps/rejected": -169.0, "loss": 0.6925, "rewards/accuracies": 0.2013888955116272, "rewards/chosen": 1.7762184143066406e-05, "rewards/margins": 0.000156402587890625, "rewards/rejected": -0.0001392364501953125, "step": 10 }, { "epoch": 0.0015047776690993906, "grad_norm": 2.4375276480372223, "learning_rate": 7.518796992481202e-09, "logits/chosen": -0.8984375, "logits/rejected": -0.482421875, "logps/chosen": -202.0, "logps/rejected": -182.0, "loss": 0.6924, "rewards/accuracies": 0.2874999940395355, "rewards/chosen": 0.000659942626953125, "rewards/margins": 0.0006103515625, "rewards/rejected": 4.76837158203125e-05, "step": 20 }, { "epoch": 0.0022571665036490857, "grad_norm": 2.8796384773541384, "learning_rate": 1.1278195488721804e-08, "logits/chosen": -0.9296875, "logits/rejected": -0.47265625, "logps/chosen": -200.0, "logps/rejected": -194.0, "loss": 0.6926, "rewards/accuracies": 0.32499998807907104, "rewards/chosen": 7.915496826171875e-05, "rewards/margins": 0.000469207763671875, "rewards/rejected": -0.0003910064697265625, "step": 30 }, { "epoch": 0.003009555338198781, "grad_norm": 2.772717406880292, "learning_rate": 1.5037593984962404e-08, "logits/chosen": -0.7890625, "logits/rejected": -0.482421875, "logps/chosen": -188.0, "logps/rejected": -185.0, "loss": 0.6925, "rewards/accuracies": 0.24375000596046448, "rewards/chosen": 0.000736236572265625, "rewards/margins": 0.00092315673828125, "rewards/rejected": -0.000186920166015625, "step": 40 }, { "epoch": 0.0037619441727484763, "grad_norm": 2.7523871873113572, "learning_rate": 1.8796992481203004e-08, "logits/chosen": -0.8359375, "logits/rejected": -0.5078125, "logps/chosen": -181.0, "logps/rejected": -180.0, "loss": 0.6923, "rewards/accuracies": 0.23125000298023224, "rewards/chosen": 0.0006256103515625, "rewards/margins": -0.000484466552734375, "rewards/rejected": 0.0011138916015625, "step": 50 }, { "epoch": 0.004514333007298171, "grad_norm": 2.8113480932625863, "learning_rate": 2.2556390977443608e-08, "logits/chosen": -0.92578125, "logits/rejected": -0.41015625, "logps/chosen": -185.0, "logps/rejected": -170.0, "loss": 0.6925, "rewards/accuracies": 0.26249998807907104, "rewards/chosen": 0.00034332275390625, "rewards/margins": -0.00032806396484375, "rewards/rejected": 0.00067138671875, "step": 60 }, { "epoch": 0.005266721841847867, "grad_norm": 2.360660807675169, "learning_rate": 2.6315789473684208e-08, "logits/chosen": -0.87109375, "logits/rejected": -0.333984375, "logps/chosen": -196.0, "logps/rejected": -195.0, "loss": 0.6923, "rewards/accuracies": 0.3062500059604645, "rewards/chosen": 0.001312255859375, "rewards/margins": -0.0006103515625, "rewards/rejected": 0.001922607421875, "step": 70 }, { "epoch": 0.006019110676397562, "grad_norm": 2.3167527756033786, "learning_rate": 3.007518796992481e-08, "logits/chosen": -0.86328125, "logits/rejected": -0.4453125, "logps/chosen": -181.0, "logps/rejected": -172.0, "loss": 0.6919, "rewards/accuracies": 0.29374998807907104, "rewards/chosen": 0.00150299072265625, "rewards/margins": 0.00092315673828125, "rewards/rejected": 0.000579833984375, "step": 80 }, { "epoch": 0.006771499510947258, "grad_norm": 2.2484987736457733, "learning_rate": 3.383458646616541e-08, "logits/chosen": -1.0546875, "logits/rejected": -0.5625, "logps/chosen": -194.0, "logps/rejected": -174.0, "loss": 0.6924, "rewards/accuracies": 0.25, "rewards/chosen": 0.0024261474609375, "rewards/margins": -0.00093841552734375, "rewards/rejected": 0.0033721923828125, "step": 90 }, { "epoch": 0.0075238883454969525, "grad_norm": 2.6116792495485943, "learning_rate": 3.759398496240601e-08, "logits/chosen": -0.90625, "logits/rejected": -0.427734375, "logps/chosen": -189.0, "logps/rejected": -171.0, "loss": 0.6922, "rewards/accuracies": 0.41874998807907104, "rewards/chosen": 0.004608154296875, "rewards/margins": 0.00147247314453125, "rewards/rejected": 0.0031280517578125, "step": 100 }, { "epoch": 0.008276277180046648, "grad_norm": 2.3070603153012965, "learning_rate": 4.1353383458646615e-08, "logits/chosen": -0.6953125, "logits/rejected": -0.400390625, "logps/chosen": -183.0, "logps/rejected": -176.0, "loss": 0.6922, "rewards/accuracies": 0.33125001192092896, "rewards/chosen": 0.00482177734375, "rewards/margins": 0.0010833740234375, "rewards/rejected": 0.003753662109375, "step": 110 }, { "epoch": 0.009028666014596343, "grad_norm": 2.7324446982248123, "learning_rate": 4.5112781954887216e-08, "logits/chosen": -0.9140625, "logits/rejected": -0.41796875, "logps/chosen": -182.0, "logps/rejected": -163.0, "loss": 0.6919, "rewards/accuracies": 0.4749999940395355, "rewards/chosen": 0.00787353515625, "rewards/margins": 0.0023345947265625, "rewards/rejected": 0.00555419921875, "step": 120 }, { "epoch": 0.009781054849146039, "grad_norm": 2.804110629265514, "learning_rate": 4.8872180451127816e-08, "logits/chosen": -0.8671875, "logits/rejected": -0.490234375, "logps/chosen": -161.0, "logps/rejected": -166.0, "loss": 0.6916, "rewards/accuracies": 0.512499988079071, "rewards/chosen": 0.01092529296875, "rewards/margins": 0.004180908203125, "rewards/rejected": 0.006744384765625, "step": 130 }, { "epoch": 0.010533443683695734, "grad_norm": 2.7182091326559323, "learning_rate": 5.2631578947368416e-08, "logits/chosen": -0.73828125, "logits/rejected": -0.384765625, "logps/chosen": -183.0, "logps/rejected": -169.0, "loss": 0.6919, "rewards/accuracies": 0.3812499940395355, "rewards/chosen": 0.0118408203125, "rewards/margins": 0.000469207763671875, "rewards/rejected": 0.0113525390625, "step": 140 }, { "epoch": 0.01128583251824543, "grad_norm": 2.5225619552907306, "learning_rate": 5.6390977443609016e-08, "logits/chosen": -0.89453125, "logits/rejected": -0.244140625, "logps/chosen": -197.0, "logps/rejected": -170.0, "loss": 0.6913, "rewards/accuracies": 0.5062500238418579, "rewards/chosen": 0.01556396484375, "rewards/margins": 0.0032958984375, "rewards/rejected": 0.01226806640625, "step": 150 }, { "epoch": 0.012038221352795125, "grad_norm": 2.844307012464271, "learning_rate": 6.015037593984962e-08, "logits/chosen": -0.91796875, "logits/rejected": -0.515625, "logps/chosen": -185.0, "logps/rejected": -167.0, "loss": 0.6905, "rewards/accuracies": 0.6187499761581421, "rewards/chosen": 0.0238037109375, "rewards/margins": 0.007080078125, "rewards/rejected": 0.0167236328125, "step": 160 }, { "epoch": 0.01279061018734482, "grad_norm": 2.697724626931233, "learning_rate": 6.390977443609022e-08, "logits/chosen": -0.8203125, "logits/rejected": -0.34375, "logps/chosen": -191.0, "logps/rejected": -174.0, "loss": 0.6908, "rewards/accuracies": 0.5625, "rewards/chosen": 0.031982421875, "rewards/margins": 0.005767822265625, "rewards/rejected": 0.0263671875, "step": 170 }, { "epoch": 0.013542999021894516, "grad_norm": 2.882785553271986, "learning_rate": 6.766917293233082e-08, "logits/chosen": -0.8046875, "logits/rejected": -0.439453125, "logps/chosen": -169.0, "logps/rejected": -175.0, "loss": 0.6896, "rewards/accuracies": 0.5375000238418579, "rewards/chosen": 0.0380859375, "rewards/margins": 0.007598876953125, "rewards/rejected": 0.030517578125, "step": 180 }, { "epoch": 0.01429538785644421, "grad_norm": 3.16613386711893, "learning_rate": 7.142857142857142e-08, "logits/chosen": -1.0390625, "logits/rejected": -0.54296875, "logps/chosen": -184.0, "logps/rejected": -162.0, "loss": 0.6885, "rewards/accuracies": 0.59375, "rewards/chosen": 0.046875, "rewards/margins": 0.00872802734375, "rewards/rejected": 0.0380859375, "step": 190 }, { "epoch": 0.015047776690993905, "grad_norm": 2.590830216802789, "learning_rate": 7.518796992481202e-08, "logits/chosen": -0.9921875, "logits/rejected": -0.59375, "logps/chosen": -184.0, "logps/rejected": -177.0, "loss": 0.6869, "rewards/accuracies": 0.543749988079071, "rewards/chosen": 0.058349609375, "rewards/margins": 0.01104736328125, "rewards/rejected": 0.04736328125, "step": 200 }, { "epoch": 0.0158001655255436, "grad_norm": 2.443378221010887, "learning_rate": 7.894736842105262e-08, "logits/chosen": -0.93359375, "logits/rejected": -0.578125, "logps/chosen": -177.0, "logps/rejected": -173.0, "loss": 0.687, "rewards/accuracies": 0.581250011920929, "rewards/chosen": 0.06494140625, "rewards/margins": 0.00958251953125, "rewards/rejected": 0.055419921875, "step": 210 }, { "epoch": 0.016552554360093296, "grad_norm": 2.6327635605182205, "learning_rate": 8.270676691729323e-08, "logits/chosen": -0.94921875, "logits/rejected": -0.65625, "logps/chosen": -189.0, "logps/rejected": -176.0, "loss": 0.6846, "rewards/accuracies": 0.637499988079071, "rewards/chosen": 0.08642578125, "rewards/margins": 0.018310546875, "rewards/rejected": 0.068359375, "step": 220 }, { "epoch": 0.017304943194642992, "grad_norm": 2.6945744621039935, "learning_rate": 8.646616541353382e-08, "logits/chosen": -0.92578125, "logits/rejected": -0.56640625, "logps/chosen": -168.0, "logps/rejected": -182.0, "loss": 0.6804, "rewards/accuracies": 0.6812499761581421, "rewards/chosen": 0.1064453125, "rewards/margins": 0.0308837890625, "rewards/rejected": 0.07568359375, "step": 230 }, { "epoch": 0.018057332029192685, "grad_norm": 2.559168307580934, "learning_rate": 9.022556390977443e-08, "logits/chosen": -1.0, "logits/rejected": -0.6015625, "logps/chosen": -178.0, "logps/rejected": -168.0, "loss": 0.6787, "rewards/accuracies": 0.65625, "rewards/chosen": 0.12109375, "rewards/margins": 0.0244140625, "rewards/rejected": 0.0966796875, "step": 240 }, { "epoch": 0.01880972086374238, "grad_norm": 2.5840670822230583, "learning_rate": 9.398496240601502e-08, "logits/chosen": -0.92578125, "logits/rejected": -0.4453125, "logps/chosen": -153.0, "logps/rejected": -158.0, "loss": 0.6785, "rewards/accuracies": 0.65625, "rewards/chosen": 0.1240234375, "rewards/margins": 0.0230712890625, "rewards/rejected": 0.10107421875, "step": 250 }, { "epoch": 0.019562109698292078, "grad_norm": 2.3631036306779203, "learning_rate": 9.774436090225563e-08, "logits/chosen": -0.97265625, "logits/rejected": -0.61328125, "logps/chosen": -156.0, "logps/rejected": -168.0, "loss": 0.6783, "rewards/accuracies": 0.606249988079071, "rewards/chosen": 0.126953125, "rewards/margins": 0.024658203125, "rewards/rejected": 0.1025390625, "step": 260 }, { "epoch": 0.02031449853284177, "grad_norm": 2.26775524388638, "learning_rate": 1.0150375939849622e-07, "logits/chosen": -1.0625, "logits/rejected": -0.6953125, "logps/chosen": -189.0, "logps/rejected": -170.0, "loss": 0.6757, "rewards/accuracies": 0.71875, "rewards/chosen": 0.1337890625, "rewards/margins": 0.037841796875, "rewards/rejected": 0.09619140625, "step": 270 }, { "epoch": 0.021066887367391467, "grad_norm": 2.4147320657900333, "learning_rate": 1.0526315789473683e-07, "logits/chosen": -1.015625, "logits/rejected": -0.78125, "logps/chosen": -182.0, "logps/rejected": -164.0, "loss": 0.6753, "rewards/accuracies": 0.643750011920929, "rewards/chosen": 0.1416015625, "rewards/margins": 0.03466796875, "rewards/rejected": 0.107421875, "step": 280 }, { "epoch": 0.021819276201941164, "grad_norm": 2.4961645488077804, "learning_rate": 1.0902255639097744e-07, "logits/chosen": -1.0546875, "logits/rejected": -0.69921875, "logps/chosen": -167.0, "logps/rejected": -170.0, "loss": 0.6758, "rewards/accuracies": 0.65625, "rewards/chosen": 0.142578125, "rewards/margins": 0.03662109375, "rewards/rejected": 0.10546875, "step": 290 }, { "epoch": 0.02257166503649086, "grad_norm": 2.385722549057329, "learning_rate": 1.1278195488721803e-07, "logits/chosen": -1.046875, "logits/rejected": -0.60546875, "logps/chosen": -163.0, "logps/rejected": -156.0, "loss": 0.6708, "rewards/accuracies": 0.668749988079071, "rewards/chosen": 0.1474609375, "rewards/margins": 0.041748046875, "rewards/rejected": 0.10595703125, "step": 300 }, { "epoch": 0.023324053871040553, "grad_norm": 2.318150341997613, "learning_rate": 1.1654135338345864e-07, "logits/chosen": -0.90234375, "logits/rejected": -0.578125, "logps/chosen": -166.0, "logps/rejected": -142.0, "loss": 0.6713, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.1357421875, "rewards/margins": 0.0439453125, "rewards/rejected": 0.091796875, "step": 310 }, { "epoch": 0.02407644270559025, "grad_norm": 2.309685647830638, "learning_rate": 1.2030075187969923e-07, "logits/chosen": -1.1484375, "logits/rejected": -0.6796875, "logps/chosen": -176.0, "logps/rejected": -169.0, "loss": 0.671, "rewards/accuracies": 0.6937500238418579, "rewards/chosen": 0.1435546875, "rewards/margins": 0.048828125, "rewards/rejected": 0.0947265625, "step": 320 }, { "epoch": 0.024828831540139946, "grad_norm": 2.414688671261455, "learning_rate": 1.2406015037593983e-07, "logits/chosen": -1.15625, "logits/rejected": -0.67578125, "logps/chosen": -174.0, "logps/rejected": -188.0, "loss": 0.6685, "rewards/accuracies": 0.7124999761581421, "rewards/chosen": 0.1484375, "rewards/margins": 0.059326171875, "rewards/rejected": 0.08935546875, "step": 330 }, { "epoch": 0.02558122037468964, "grad_norm": 2.3481987753669338, "learning_rate": 1.2781954887218045e-07, "logits/chosen": -1.140625, "logits/rejected": -0.68359375, "logps/chosen": -179.0, "logps/rejected": -179.0, "loss": 0.6668, "rewards/accuracies": 0.7749999761581421, "rewards/chosen": 0.1357421875, "rewards/margins": 0.0673828125, "rewards/rejected": 0.068359375, "step": 340 }, { "epoch": 0.026333609209239335, "grad_norm": 2.21683907673388, "learning_rate": 1.3157894736842104e-07, "logits/chosen": -1.1953125, "logits/rejected": -0.59765625, "logps/chosen": -187.0, "logps/rejected": -184.0, "loss": 0.6653, "rewards/accuracies": 0.737500011920929, "rewards/chosen": 0.1298828125, "rewards/margins": 0.06591796875, "rewards/rejected": 0.064453125, "step": 350 }, { "epoch": 0.02708599804378903, "grad_norm": 2.5858044800687985, "learning_rate": 1.3533834586466163e-07, "logits/chosen": -1.0625, "logits/rejected": -0.6171875, "logps/chosen": -182.0, "logps/rejected": -170.0, "loss": 0.6637, "rewards/accuracies": 0.71875, "rewards/chosen": 0.1103515625, "rewards/margins": 0.056884765625, "rewards/rejected": 0.0537109375, "step": 360 }, { "epoch": 0.027838386878338724, "grad_norm": 2.2552058830335184, "learning_rate": 1.3909774436090225e-07, "logits/chosen": -1.1015625, "logits/rejected": -0.703125, "logps/chosen": -161.0, "logps/rejected": -160.0, "loss": 0.6603, "rewards/accuracies": 0.7124999761581421, "rewards/chosen": 0.1318359375, "rewards/margins": 0.0693359375, "rewards/rejected": 0.06298828125, "step": 370 }, { "epoch": 0.02859077571288842, "grad_norm": 2.245463521776559, "learning_rate": 1.4285714285714285e-07, "logits/chosen": -1.0234375, "logits/rejected": -0.66015625, "logps/chosen": -171.0, "logps/rejected": -161.0, "loss": 0.6588, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": 0.11181640625, "rewards/margins": 0.09423828125, "rewards/rejected": 0.017333984375, "step": 380 }, { "epoch": 0.029343164547438117, "grad_norm": 2.4945067579028946, "learning_rate": 1.4661654135338344e-07, "logits/chosen": -1.203125, "logits/rejected": -0.7734375, "logps/chosen": -196.0, "logps/rejected": -176.0, "loss": 0.6586, "rewards/accuracies": 0.7562500238418579, "rewards/chosen": 0.1025390625, "rewards/margins": 0.08349609375, "rewards/rejected": 0.01904296875, "step": 390 }, { "epoch": 0.03009555338198781, "grad_norm": 2.537557036174086, "learning_rate": 1.5037593984962403e-07, "logits/chosen": -1.046875, "logits/rejected": -0.6796875, "logps/chosen": -177.0, "logps/rejected": -172.0, "loss": 0.6554, "rewards/accuracies": 0.6875, "rewards/chosen": 0.06787109375, "rewards/margins": 0.06591796875, "rewards/rejected": 0.0019989013671875, "step": 400 }, { "epoch": 0.030847942216537506, "grad_norm": 2.2909084083190083, "learning_rate": 1.5413533834586465e-07, "logits/chosen": -1.140625, "logits/rejected": -0.7890625, "logps/chosen": -186.0, "logps/rejected": -176.0, "loss": 0.6511, "rewards/accuracies": 0.737500011920929, "rewards/chosen": 0.0625, "rewards/margins": 0.08251953125, "rewards/rejected": -0.020263671875, "step": 410 }, { "epoch": 0.0316003310510872, "grad_norm": 2.7865022988016066, "learning_rate": 1.5789473684210525e-07, "logits/chosen": -1.09375, "logits/rejected": -0.7265625, "logps/chosen": -173.0, "logps/rejected": -175.0, "loss": 0.6453, "rewards/accuracies": 0.768750011920929, "rewards/chosen": 0.04931640625, "rewards/margins": 0.10302734375, "rewards/rejected": -0.053466796875, "step": 420 }, { "epoch": 0.0323527198856369, "grad_norm": 2.8095798194868906, "learning_rate": 1.6165413533834584e-07, "logits/chosen": -1.25, "logits/rejected": -0.7578125, "logps/chosen": -192.0, "logps/rejected": -183.0, "loss": 0.6413, "rewards/accuracies": 0.75, "rewards/chosen": 0.050048828125, "rewards/margins": 0.1259765625, "rewards/rejected": -0.07568359375, "step": 430 }, { "epoch": 0.03310510872018659, "grad_norm": 2.866389441302747, "learning_rate": 1.6541353383458646e-07, "logits/chosen": -1.3515625, "logits/rejected": -0.9609375, "logps/chosen": -220.0, "logps/rejected": -200.0, "loss": 0.6351, "rewards/accuracies": 0.71875, "rewards/chosen": 0.03173828125, "rewards/margins": 0.1455078125, "rewards/rejected": -0.11376953125, "step": 440 }, { "epoch": 0.033857497554736285, "grad_norm": 2.772428255917957, "learning_rate": 1.6917293233082705e-07, "logits/chosen": -1.21875, "logits/rejected": -0.8984375, "logps/chosen": -201.0, "logps/rejected": -182.0, "loss": 0.6341, "rewards/accuracies": 0.7250000238418579, "rewards/chosen": -0.00732421875, "rewards/margins": 0.10986328125, "rewards/rejected": -0.1171875, "step": 450 }, { "epoch": 0.034609886389285985, "grad_norm": 2.807349876185253, "learning_rate": 1.7293233082706765e-07, "logits/chosen": -1.3125, "logits/rejected": -0.87890625, "logps/chosen": -192.0, "logps/rejected": -208.0, "loss": 0.6199, "rewards/accuracies": 0.762499988079071, "rewards/chosen": -0.01336669921875, "rewards/margins": 0.1826171875, "rewards/rejected": -0.1962890625, "step": 460 }, { "epoch": 0.03536227522383568, "grad_norm": 2.9725127059963756, "learning_rate": 1.7669172932330824e-07, "logits/chosen": -1.2890625, "logits/rejected": -0.98828125, "logps/chosen": -195.0, "logps/rejected": -218.0, "loss": 0.6154, "rewards/accuracies": 0.7562500238418579, "rewards/chosen": -0.034912109375, "rewards/margins": 0.1865234375, "rewards/rejected": -0.2216796875, "step": 470 }, { "epoch": 0.03611466405838537, "grad_norm": 2.6858748382562365, "learning_rate": 1.8045112781954886e-07, "logits/chosen": -1.390625, "logits/rejected": -1.125, "logps/chosen": -198.0, "logps/rejected": -197.0, "loss": 0.6095, "rewards/accuracies": 0.7749999761581421, "rewards/chosen": -0.06591796875, "rewards/margins": 0.1982421875, "rewards/rejected": -0.263671875, "step": 480 }, { "epoch": 0.03686705289293507, "grad_norm": 2.7409795605816325, "learning_rate": 1.8421052631578946e-07, "logits/chosen": -1.421875, "logits/rejected": -1.1796875, "logps/chosen": -208.0, "logps/rejected": -225.0, "loss": 0.6145, "rewards/accuracies": 0.737500011920929, "rewards/chosen": -0.0849609375, "rewards/margins": 0.24609375, "rewards/rejected": -0.33203125, "step": 490 }, { "epoch": 0.03761944172748476, "grad_norm": 2.9301362950991465, "learning_rate": 1.8796992481203005e-07, "logits/chosen": -1.2890625, "logits/rejected": -0.94921875, "logps/chosen": -207.0, "logps/rejected": -204.0, "loss": 0.6032, "rewards/accuracies": 0.731249988079071, "rewards/chosen": -0.1416015625, "rewards/margins": 0.1923828125, "rewards/rejected": -0.333984375, "step": 500 }, { "epoch": 0.038371830562034456, "grad_norm": 2.9140140227932725, "learning_rate": 1.9172932330827067e-07, "logits/chosen": -1.484375, "logits/rejected": -1.2109375, "logps/chosen": -188.0, "logps/rejected": -200.0, "loss": 0.5997, "rewards/accuracies": 0.7749999761581421, "rewards/chosen": -0.1552734375, "rewards/margins": 0.263671875, "rewards/rejected": -0.41796875, "step": 510 }, { "epoch": 0.039124219396584156, "grad_norm": 2.7967520674555746, "learning_rate": 1.9548872180451126e-07, "logits/chosen": -1.484375, "logits/rejected": -1.078125, "logps/chosen": -196.0, "logps/rejected": -210.0, "loss": 0.59, "rewards/accuracies": 0.737500011920929, "rewards/chosen": -0.19140625, "rewards/margins": 0.2490234375, "rewards/rejected": -0.44140625, "step": 520 }, { "epoch": 0.03987660823113385, "grad_norm": 3.028366455797919, "learning_rate": 1.9924812030075186e-07, "logits/chosen": -1.40625, "logits/rejected": -1.1015625, "logps/chosen": -201.0, "logps/rejected": -228.0, "loss": 0.582, "rewards/accuracies": 0.7749999761581421, "rewards/chosen": -0.1943359375, "rewards/margins": 0.361328125, "rewards/rejected": -0.5546875, "step": 530 }, { "epoch": 0.04062899706568354, "grad_norm": 3.1249742515575667, "learning_rate": 2.0300751879699245e-07, "logits/chosen": -1.453125, "logits/rejected": -1.0, "logps/chosen": -206.0, "logps/rejected": -220.0, "loss": 0.5809, "rewards/accuracies": 0.75, "rewards/chosen": -0.2578125, "rewards/margins": 0.298828125, "rewards/rejected": -0.5546875, "step": 540 }, { "epoch": 0.04138138590023324, "grad_norm": 3.2105568054119704, "learning_rate": 2.0676691729323307e-07, "logits/chosen": -1.5625, "logits/rejected": -1.1015625, "logps/chosen": -204.0, "logps/rejected": -218.0, "loss": 0.5904, "rewards/accuracies": 0.737500011920929, "rewards/chosen": -0.279296875, "rewards/margins": 0.298828125, "rewards/rejected": -0.578125, "step": 550 }, { "epoch": 0.042133774734782935, "grad_norm": 3.0465561263128245, "learning_rate": 2.1052631578947366e-07, "logits/chosen": -1.625, "logits/rejected": -1.3671875, "logps/chosen": -211.0, "logps/rejected": -245.0, "loss": 0.5798, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -0.27734375, "rewards/margins": 0.333984375, "rewards/rejected": -0.609375, "step": 560 }, { "epoch": 0.042886163569332635, "grad_norm": 3.070661699082978, "learning_rate": 2.1428571428571426e-07, "logits/chosen": -1.6484375, "logits/rejected": -1.2890625, "logps/chosen": -225.0, "logps/rejected": -234.0, "loss": 0.5724, "rewards/accuracies": 0.78125, "rewards/chosen": -0.33984375, "rewards/margins": 0.326171875, "rewards/rejected": -0.66796875, "step": 570 }, { "epoch": 0.04363855240388233, "grad_norm": 3.1716497646651503, "learning_rate": 2.1804511278195488e-07, "logits/chosen": -1.625, "logits/rejected": -1.2734375, "logps/chosen": -230.0, "logps/rejected": -242.0, "loss": 0.5689, "rewards/accuracies": 0.7437499761581421, "rewards/chosen": -0.349609375, "rewards/margins": 0.384765625, "rewards/rejected": -0.734375, "step": 580 }, { "epoch": 0.04439094123843202, "grad_norm": 3.498261244361052, "learning_rate": 2.2180451127819547e-07, "logits/chosen": -1.4296875, "logits/rejected": -1.1328125, "logps/chosen": -221.0, "logps/rejected": -241.0, "loss": 0.5664, "rewards/accuracies": 0.768750011920929, "rewards/chosen": -0.416015625, "rewards/margins": 0.392578125, "rewards/rejected": -0.80859375, "step": 590 }, { "epoch": 0.04514333007298172, "grad_norm": 3.530404686169034, "learning_rate": 2.2556390977443606e-07, "logits/chosen": -1.515625, "logits/rejected": -1.390625, "logps/chosen": -246.0, "logps/rejected": -264.0, "loss": 0.5685, "rewards/accuracies": 0.75, "rewards/chosen": -0.427734375, "rewards/margins": 0.376953125, "rewards/rejected": -0.8046875, "step": 600 }, { "epoch": 0.04589571890753141, "grad_norm": 3.368233974595872, "learning_rate": 2.2932330827067666e-07, "logits/chosen": -1.4765625, "logits/rejected": -1.1796875, "logps/chosen": -203.0, "logps/rejected": -244.0, "loss": 0.5559, "rewards/accuracies": 0.737500011920929, "rewards/chosen": -0.361328125, "rewards/margins": 0.439453125, "rewards/rejected": -0.80078125, "step": 610 }, { "epoch": 0.046648107742081106, "grad_norm": 3.7736182065328245, "learning_rate": 2.3308270676691728e-07, "logits/chosen": -1.6328125, "logits/rejected": -1.28125, "logps/chosen": -214.0, "logps/rejected": -270.0, "loss": 0.5532, "rewards/accuracies": 0.7749999761581421, "rewards/chosen": -0.4296875, "rewards/margins": 0.5078125, "rewards/rejected": -0.9375, "step": 620 }, { "epoch": 0.047400496576630806, "grad_norm": 3.9934270743611746, "learning_rate": 2.3684210526315787e-07, "logits/chosen": -1.6015625, "logits/rejected": -1.1875, "logps/chosen": -244.0, "logps/rejected": -268.0, "loss": 0.5438, "rewards/accuracies": 0.7437499761581421, "rewards/chosen": -0.4921875, "rewards/margins": 0.44921875, "rewards/rejected": -0.94140625, "step": 630 }, { "epoch": 0.0481528854111805, "grad_norm": 4.456178476886664, "learning_rate": 2.4060150375939847e-07, "logits/chosen": -1.53125, "logits/rejected": -1.203125, "logps/chosen": -233.0, "logps/rejected": -294.0, "loss": 0.5376, "rewards/accuracies": 0.793749988079071, "rewards/chosen": -0.48828125, "rewards/margins": 0.6484375, "rewards/rejected": -1.140625, "step": 640 }, { "epoch": 0.04890527424573019, "grad_norm": 4.934230698562457, "learning_rate": 2.443609022556391e-07, "logits/chosen": -1.6484375, "logits/rejected": -1.265625, "logps/chosen": -234.0, "logps/rejected": -292.0, "loss": 0.5472, "rewards/accuracies": 0.768750011920929, "rewards/chosen": -0.546875, "rewards/margins": 0.546875, "rewards/rejected": -1.09375, "step": 650 }, { "epoch": 0.04965766308027989, "grad_norm": 4.548739222104471, "learning_rate": 2.4812030075187965e-07, "logits/chosen": -1.53125, "logits/rejected": -1.2890625, "logps/chosen": -214.0, "logps/rejected": -270.0, "loss": 0.5449, "rewards/accuracies": 0.762499988079071, "rewards/chosen": -0.46875, "rewards/margins": 0.53125, "rewards/rejected": -0.99609375, "step": 660 }, { "epoch": 0.050410051914829584, "grad_norm": 4.087327712324922, "learning_rate": 2.518796992481203e-07, "logits/chosen": -1.5390625, "logits/rejected": -1.265625, "logps/chosen": -249.0, "logps/rejected": -288.0, "loss": 0.5209, "rewards/accuracies": 0.75, "rewards/chosen": -0.5546875, "rewards/margins": 0.5234375, "rewards/rejected": -1.078125, "step": 670 }, { "epoch": 0.05116244074937928, "grad_norm": 4.387462590010207, "learning_rate": 2.556390977443609e-07, "logits/chosen": -1.7265625, "logits/rejected": -1.3828125, "logps/chosen": -248.0, "logps/rejected": -302.0, "loss": 0.5119, "rewards/accuracies": 0.8125, "rewards/chosen": -0.4921875, "rewards/margins": 0.70703125, "rewards/rejected": -1.203125, "step": 680 }, { "epoch": 0.05191482958392898, "grad_norm": 4.8645123103331915, "learning_rate": 2.593984962406015e-07, "logits/chosen": -1.6015625, "logits/rejected": -1.359375, "logps/chosen": -260.0, "logps/rejected": -304.0, "loss": 0.5166, "rewards/accuracies": 0.7749999761581421, "rewards/chosen": -0.60546875, "rewards/margins": 0.609375, "rewards/rejected": -1.21875, "step": 690 }, { "epoch": 0.05266721841847867, "grad_norm": 4.738167895407399, "learning_rate": 2.631578947368421e-07, "logits/chosen": -1.578125, "logits/rejected": -1.2734375, "logps/chosen": -260.0, "logps/rejected": -304.0, "loss": 0.5118, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -0.609375, "rewards/margins": 0.63671875, "rewards/rejected": -1.2421875, "step": 700 }, { "epoch": 0.05341960725302836, "grad_norm": 6.460360350580228, "learning_rate": 2.669172932330827e-07, "logits/chosen": -1.71875, "logits/rejected": -1.421875, "logps/chosen": -272.0, "logps/rejected": -340.0, "loss": 0.4999, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -0.68359375, "rewards/margins": 0.70703125, "rewards/rejected": -1.390625, "step": 710 }, { "epoch": 0.05417199608757806, "grad_norm": 4.6393895241522625, "learning_rate": 2.7067669172932327e-07, "logits/chosen": -1.609375, "logits/rejected": -1.4140625, "logps/chosen": -260.0, "logps/rejected": -332.0, "loss": 0.4851, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -0.625, "rewards/margins": 0.796875, "rewards/rejected": -1.421875, "step": 720 }, { "epoch": 0.054924384922127756, "grad_norm": 5.4127050623110105, "learning_rate": 2.744360902255639e-07, "logits/chosen": -1.578125, "logits/rejected": -1.390625, "logps/chosen": -262.0, "logps/rejected": -308.0, "loss": 0.4977, "rewards/accuracies": 0.7562500238418579, "rewards/chosen": -0.7578125, "rewards/margins": 0.7890625, "rewards/rejected": -1.546875, "step": 730 }, { "epoch": 0.05567677375667745, "grad_norm": 5.5615405759126695, "learning_rate": 2.781954887218045e-07, "logits/chosen": -1.625, "logits/rejected": -1.390625, "logps/chosen": -256.0, "logps/rejected": -316.0, "loss": 0.4943, "rewards/accuracies": 0.8187500238418579, "rewards/chosen": -0.6640625, "rewards/margins": 0.75, "rewards/rejected": -1.4140625, "step": 740 }, { "epoch": 0.05642916259122715, "grad_norm": 5.495361561462859, "learning_rate": 2.8195488721804513e-07, "logits/chosen": -1.6640625, "logits/rejected": -1.3046875, "logps/chosen": -268.0, "logps/rejected": -336.0, "loss": 0.4752, "rewards/accuracies": 0.824999988079071, "rewards/chosen": -0.76953125, "rewards/margins": 0.82421875, "rewards/rejected": -1.59375, "step": 750 }, { "epoch": 0.05718155142577684, "grad_norm": 6.619595346155631, "learning_rate": 2.857142857142857e-07, "logits/chosen": -1.5078125, "logits/rejected": -1.28125, "logps/chosen": -270.0, "logps/rejected": -344.0, "loss": 0.4791, "rewards/accuracies": 0.7749999761581421, "rewards/chosen": -0.77734375, "rewards/margins": 0.9375, "rewards/rejected": -1.71875, "step": 760 }, { "epoch": 0.057933940260326534, "grad_norm": 6.107275043053044, "learning_rate": 2.894736842105263e-07, "logits/chosen": -1.703125, "logits/rejected": -1.421875, "logps/chosen": -274.0, "logps/rejected": -334.0, "loss": 0.4961, "rewards/accuracies": 0.793749988079071, "rewards/chosen": -0.69921875, "rewards/margins": 0.890625, "rewards/rejected": -1.5859375, "step": 770 }, { "epoch": 0.058686329094876234, "grad_norm": 5.401890849863438, "learning_rate": 2.932330827067669e-07, "logits/chosen": -1.59375, "logits/rejected": -1.4296875, "logps/chosen": -253.0, "logps/rejected": -340.0, "loss": 0.4601, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -0.734375, "rewards/margins": 0.95703125, "rewards/rejected": -1.6953125, "step": 780 }, { "epoch": 0.05943871792942593, "grad_norm": 7.164268819584015, "learning_rate": 2.969924812030075e-07, "logits/chosen": -1.6171875, "logits/rejected": -1.3515625, "logps/chosen": -284.0, "logps/rejected": -334.0, "loss": 0.4652, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -0.83203125, "rewards/margins": 0.8046875, "rewards/rejected": -1.640625, "step": 790 }, { "epoch": 0.06019110676397562, "grad_norm": 5.388623493870874, "learning_rate": 3.0075187969924807e-07, "logits/chosen": -1.71875, "logits/rejected": -1.3984375, "logps/chosen": -270.0, "logps/rejected": -380.0, "loss": 0.4675, "rewards/accuracies": 0.824999988079071, "rewards/chosen": -0.7578125, "rewards/margins": 1.15625, "rewards/rejected": -1.9140625, "step": 800 }, { "epoch": 0.06094349559852532, "grad_norm": 6.718560837307663, "learning_rate": 3.0451127819548874e-07, "logits/chosen": -1.5078125, "logits/rejected": -1.2578125, "logps/chosen": -294.0, "logps/rejected": -386.0, "loss": 0.4623, "rewards/accuracies": 0.768750011920929, "rewards/chosen": -0.9375, "rewards/margins": 1.046875, "rewards/rejected": -1.984375, "step": 810 }, { "epoch": 0.06169588443307501, "grad_norm": 5.901972506936794, "learning_rate": 3.082706766917293e-07, "logits/chosen": -1.6328125, "logits/rejected": -1.34375, "logps/chosen": -268.0, "logps/rejected": -382.0, "loss": 0.4406, "rewards/accuracies": 0.8187500238418579, "rewards/chosen": -0.73828125, "rewards/margins": 1.234375, "rewards/rejected": -1.9765625, "step": 820 }, { "epoch": 0.062448273267624706, "grad_norm": 11.570823694028725, "learning_rate": 3.1203007518796993e-07, "logits/chosen": -1.65625, "logits/rejected": -1.4609375, "logps/chosen": -268.0, "logps/rejected": -374.0, "loss": 0.4769, "rewards/accuracies": 0.7875000238418579, "rewards/chosen": -0.82421875, "rewards/margins": 0.86328125, "rewards/rejected": -1.6875, "step": 830 }, { "epoch": 0.0632006621021744, "grad_norm": 7.273980307084486, "learning_rate": 3.157894736842105e-07, "logits/chosen": -1.734375, "logits/rejected": -1.359375, "logps/chosen": -282.0, "logps/rejected": -384.0, "loss": 0.46, "rewards/accuracies": 0.8187500238418579, "rewards/chosen": -0.94140625, "rewards/margins": 1.1171875, "rewards/rejected": -2.0625, "step": 840 }, { "epoch": 0.0639530509367241, "grad_norm": 6.330827049733457, "learning_rate": 3.195488721804511e-07, "logits/chosen": -1.625, "logits/rejected": -1.3984375, "logps/chosen": -280.0, "logps/rejected": -384.0, "loss": 0.4521, "rewards/accuracies": 0.8187500238418579, "rewards/chosen": -0.89453125, "rewards/margins": 1.1953125, "rewards/rejected": -2.09375, "step": 850 }, { "epoch": 0.0647054397712738, "grad_norm": 6.298588166273813, "learning_rate": 3.233082706766917e-07, "logits/chosen": -1.6015625, "logits/rejected": -1.4375, "logps/chosen": -288.0, "logps/rejected": -374.0, "loss": 0.4399, "rewards/accuracies": 0.8187500238418579, "rewards/chosen": -0.87109375, "rewards/margins": 1.0234375, "rewards/rejected": -1.8984375, "step": 860 }, { "epoch": 0.06545782860582348, "grad_norm": 6.388863885484364, "learning_rate": 3.270676691729323e-07, "logits/chosen": -1.7265625, "logits/rejected": -1.3203125, "logps/chosen": -286.0, "logps/rejected": -392.0, "loss": 0.4429, "rewards/accuracies": 0.8125, "rewards/chosen": -0.94140625, "rewards/margins": 1.296875, "rewards/rejected": -2.234375, "step": 870 }, { "epoch": 0.06621021744037318, "grad_norm": 11.45819819890946, "learning_rate": 3.308270676691729e-07, "logits/chosen": -1.6015625, "logits/rejected": -1.328125, "logps/chosen": -284.0, "logps/rejected": -378.0, "loss": 0.4259, "rewards/accuracies": 0.7875000238418579, "rewards/chosen": -1.0, "rewards/margins": 0.953125, "rewards/rejected": -1.953125, "step": 880 }, { "epoch": 0.06696260627492288, "grad_norm": 8.099749790972755, "learning_rate": 3.3458646616541354e-07, "logits/chosen": -1.6875, "logits/rejected": -1.421875, "logps/chosen": -296.0, "logps/rejected": -390.0, "loss": 0.426, "rewards/accuracies": 0.793749988079071, "rewards/chosen": -1.1015625, "rewards/margins": 1.0703125, "rewards/rejected": -2.171875, "step": 890 }, { "epoch": 0.06771499510947257, "grad_norm": 6.42104498472239, "learning_rate": 3.383458646616541e-07, "logits/chosen": -1.6484375, "logits/rejected": -1.3671875, "logps/chosen": -262.0, "logps/rejected": -362.0, "loss": 0.4407, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -0.92578125, "rewards/margins": 1.0703125, "rewards/rejected": -2.0, "step": 900 }, { "epoch": 0.06846738394402227, "grad_norm": 6.925728266310534, "learning_rate": 3.4210526315789473e-07, "logits/chosen": -1.71875, "logits/rejected": -1.359375, "logps/chosen": -274.0, "logps/rejected": -400.0, "loss": 0.4505, "rewards/accuracies": 0.762499988079071, "rewards/chosen": -1.0, "rewards/margins": 1.1328125, "rewards/rejected": -2.140625, "step": 910 }, { "epoch": 0.06921977277857197, "grad_norm": 9.21073403849389, "learning_rate": 3.458646616541353e-07, "logits/chosen": -1.5625, "logits/rejected": -1.375, "logps/chosen": -324.0, "logps/rejected": -408.0, "loss": 0.4455, "rewards/accuracies": 0.824999988079071, "rewards/chosen": -1.203125, "rewards/margins": 1.1640625, "rewards/rejected": -2.375, "step": 920 }, { "epoch": 0.06997216161312166, "grad_norm": 7.439561073630877, "learning_rate": 3.496240601503759e-07, "logits/chosen": -1.71875, "logits/rejected": -1.5234375, "logps/chosen": -286.0, "logps/rejected": -410.0, "loss": 0.4135, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -1.0078125, "rewards/margins": 1.4375, "rewards/rejected": -2.4375, "step": 930 }, { "epoch": 0.07072455044767136, "grad_norm": 7.178227106986783, "learning_rate": 3.533834586466165e-07, "logits/chosen": -1.609375, "logits/rejected": -1.5, "logps/chosen": -278.0, "logps/rejected": -416.0, "loss": 0.4268, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -0.96484375, "rewards/margins": 1.3828125, "rewards/rejected": -2.34375, "step": 940 }, { "epoch": 0.07147693928222106, "grad_norm": 7.357715076284545, "learning_rate": 3.5714285714285716e-07, "logits/chosen": -1.6015625, "logits/rejected": -1.4375, "logps/chosen": -270.0, "logps/rejected": -404.0, "loss": 0.4653, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.0, "rewards/margins": 1.21875, "rewards/rejected": -2.21875, "step": 950 }, { "epoch": 0.07222932811677074, "grad_norm": 8.195717138617148, "learning_rate": 3.609022556390977e-07, "logits/chosen": -1.6875, "logits/rejected": -1.2421875, "logps/chosen": -286.0, "logps/rejected": -398.0, "loss": 0.4323, "rewards/accuracies": 0.7875000238418579, "rewards/chosen": -0.97265625, "rewards/margins": 1.234375, "rewards/rejected": -2.203125, "step": 960 }, { "epoch": 0.07298171695132044, "grad_norm": 7.847874155271986, "learning_rate": 3.6466165413533834e-07, "logits/chosen": -1.6796875, "logits/rejected": -1.4921875, "logps/chosen": -342.0, "logps/rejected": -418.0, "loss": 0.4386, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.2890625, "rewards/margins": 1.1328125, "rewards/rejected": -2.421875, "step": 970 }, { "epoch": 0.07373410578587014, "grad_norm": 7.690804696049142, "learning_rate": 3.684210526315789e-07, "logits/chosen": -1.7265625, "logits/rejected": -1.4921875, "logps/chosen": -298.0, "logps/rejected": -428.0, "loss": 0.4248, "rewards/accuracies": 0.8125, "rewards/chosen": -1.0859375, "rewards/margins": 1.34375, "rewards/rejected": -2.4375, "step": 980 }, { "epoch": 0.07448649462041983, "grad_norm": 8.442903295984404, "learning_rate": 3.7218045112781953e-07, "logits/chosen": -1.5625, "logits/rejected": -1.4296875, "logps/chosen": -294.0, "logps/rejected": -412.0, "loss": 0.4067, "rewards/accuracies": 0.84375, "rewards/chosen": -1.203125, "rewards/margins": 1.2265625, "rewards/rejected": -2.421875, "step": 990 }, { "epoch": 0.07523888345496953, "grad_norm": 7.356713419245785, "learning_rate": 3.759398496240601e-07, "logits/chosen": -1.5, "logits/rejected": -1.234375, "logps/chosen": -312.0, "logps/rejected": -430.0, "loss": 0.4216, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -1.2578125, "rewards/margins": 1.1640625, "rewards/rejected": -2.421875, "step": 1000 }, { "epoch": 0.07599127228951923, "grad_norm": 9.065819128281419, "learning_rate": 3.796992481203007e-07, "logits/chosen": -1.578125, "logits/rejected": -1.296875, "logps/chosen": -308.0, "logps/rejected": -426.0, "loss": 0.4129, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -1.1484375, "rewards/margins": 1.3828125, "rewards/rejected": -2.53125, "step": 1010 }, { "epoch": 0.07674366112406891, "grad_norm": 8.456310735125717, "learning_rate": 3.8345864661654134e-07, "logits/chosen": -1.625, "logits/rejected": -1.4140625, "logps/chosen": -312.0, "logps/rejected": -438.0, "loss": 0.4116, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.2265625, "rewards/margins": 1.3203125, "rewards/rejected": -2.546875, "step": 1020 }, { "epoch": 0.07749604995861861, "grad_norm": 7.768877254780789, "learning_rate": 3.8721804511278196e-07, "logits/chosen": -1.65625, "logits/rejected": -1.34375, "logps/chosen": -300.0, "logps/rejected": -426.0, "loss": 0.4205, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -1.1796875, "rewards/margins": 1.4296875, "rewards/rejected": -2.609375, "step": 1030 }, { "epoch": 0.07824843879316831, "grad_norm": 8.165418827771123, "learning_rate": 3.909774436090225e-07, "logits/chosen": -1.5859375, "logits/rejected": -1.328125, "logps/chosen": -306.0, "logps/rejected": -410.0, "loss": 0.4244, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -1.203125, "rewards/margins": 1.078125, "rewards/rejected": -2.28125, "step": 1040 }, { "epoch": 0.079000827627718, "grad_norm": 7.187624416349537, "learning_rate": 3.9473684210526315e-07, "logits/chosen": -1.765625, "logits/rejected": -1.5703125, "logps/chosen": -326.0, "logps/rejected": -462.0, "loss": 0.401, "rewards/accuracies": 0.875, "rewards/chosen": -1.1953125, "rewards/margins": 1.515625, "rewards/rejected": -2.703125, "step": 1050 }, { "epoch": 0.0797532164622677, "grad_norm": 9.519855911794908, "learning_rate": 3.984962406015037e-07, "logits/chosen": -1.6875, "logits/rejected": -1.5234375, "logps/chosen": -310.0, "logps/rejected": -430.0, "loss": 0.3999, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -1.140625, "rewards/margins": 1.40625, "rewards/rejected": -2.546875, "step": 1060 }, { "epoch": 0.0805056052968174, "grad_norm": 10.281951691857858, "learning_rate": 4.0225563909774433e-07, "logits/chosen": -1.7109375, "logits/rejected": -1.59375, "logps/chosen": -324.0, "logps/rejected": -428.0, "loss": 0.3941, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": -1.21875, "rewards/margins": 1.2734375, "rewards/rejected": -2.484375, "step": 1070 }, { "epoch": 0.08125799413136708, "grad_norm": 8.690791395476843, "learning_rate": 4.060150375939849e-07, "logits/chosen": -1.734375, "logits/rejected": -1.515625, "logps/chosen": -338.0, "logps/rejected": -474.0, "loss": 0.4242, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -1.328125, "rewards/margins": 1.4296875, "rewards/rejected": -2.75, "step": 1080 }, { "epoch": 0.08201038296591678, "grad_norm": 8.778671269804748, "learning_rate": 4.0977443609022557e-07, "logits/chosen": -1.65625, "logits/rejected": -1.5, "logps/chosen": -344.0, "logps/rejected": -472.0, "loss": 0.414, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -1.375, "rewards/margins": 1.4296875, "rewards/rejected": -2.8125, "step": 1090 }, { "epoch": 0.08276277180046648, "grad_norm": 8.280716906667923, "learning_rate": 4.1353383458646614e-07, "logits/chosen": -1.4921875, "logits/rejected": -1.3046875, "logps/chosen": -328.0, "logps/rejected": -450.0, "loss": 0.3887, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -1.2890625, "rewards/margins": 1.4921875, "rewards/rejected": -2.78125, "step": 1100 }, { "epoch": 0.08351516063501617, "grad_norm": 10.150546556340007, "learning_rate": 4.1729323308270676e-07, "logits/chosen": -1.6015625, "logits/rejected": -1.421875, "logps/chosen": -322.0, "logps/rejected": -476.0, "loss": 0.365, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": -1.328125, "rewards/margins": 1.8515625, "rewards/rejected": -3.1875, "step": 1110 }, { "epoch": 0.08426754946956587, "grad_norm": 11.947834970608445, "learning_rate": 4.2105263157894733e-07, "logits/chosen": -1.6484375, "logits/rejected": -1.359375, "logps/chosen": -348.0, "logps/rejected": -462.0, "loss": 0.4156, "rewards/accuracies": 0.78125, "rewards/chosen": -1.5390625, "rewards/margins": 1.3046875, "rewards/rejected": -2.84375, "step": 1120 }, { "epoch": 0.08501993830411557, "grad_norm": 12.749012075468157, "learning_rate": 4.2481203007518795e-07, "logits/chosen": -1.5859375, "logits/rejected": -1.390625, "logps/chosen": -312.0, "logps/rejected": -474.0, "loss": 0.3935, "rewards/accuracies": 0.824999988079071, "rewards/chosen": -1.1796875, "rewards/margins": 1.7265625, "rewards/rejected": -2.90625, "step": 1130 }, { "epoch": 0.08577232713866527, "grad_norm": 7.946978365468776, "learning_rate": 4.285714285714285e-07, "logits/chosen": -1.65625, "logits/rejected": -1.3671875, "logps/chosen": -328.0, "logps/rejected": -472.0, "loss": 0.3878, "rewards/accuracies": 0.8187500238418579, "rewards/chosen": -1.4296875, "rewards/margins": 1.4375, "rewards/rejected": -2.875, "step": 1140 }, { "epoch": 0.08652471597321496, "grad_norm": 8.067449328185853, "learning_rate": 4.3233082706766913e-07, "logits/chosen": -1.6015625, "logits/rejected": -1.2421875, "logps/chosen": -310.0, "logps/rejected": -438.0, "loss": 0.382, "rewards/accuracies": 0.84375, "rewards/chosen": -1.296875, "rewards/margins": 1.359375, "rewards/rejected": -2.65625, "step": 1150 }, { "epoch": 0.08727710480776465, "grad_norm": 7.765280758347942, "learning_rate": 4.3609022556390975e-07, "logits/chosen": -1.6640625, "logits/rejected": -1.515625, "logps/chosen": -344.0, "logps/rejected": -480.0, "loss": 0.3992, "rewards/accuracies": 0.8125, "rewards/chosen": -1.4609375, "rewards/margins": 1.484375, "rewards/rejected": -2.9375, "step": 1160 }, { "epoch": 0.08802949364231435, "grad_norm": 9.429865311190007, "learning_rate": 4.398496240601504e-07, "logits/chosen": -1.59375, "logits/rejected": -1.4140625, "logps/chosen": -328.0, "logps/rejected": -490.0, "loss": 0.4012, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -1.515625, "rewards/margins": 1.75, "rewards/rejected": -3.265625, "step": 1170 }, { "epoch": 0.08878188247686404, "grad_norm": 9.158884062158586, "learning_rate": 4.4360902255639094e-07, "logits/chosen": -1.4609375, "logits/rejected": -1.296875, "logps/chosen": -326.0, "logps/rejected": -430.0, "loss": 0.3994, "rewards/accuracies": 0.7562500238418579, "rewards/chosen": -1.4296875, "rewards/margins": 1.140625, "rewards/rejected": -2.578125, "step": 1180 }, { "epoch": 0.08953427131141374, "grad_norm": 10.572705153614718, "learning_rate": 4.4736842105263156e-07, "logits/chosen": -1.6171875, "logits/rejected": -1.5, "logps/chosen": -322.0, "logps/rejected": -436.0, "loss": 0.3782, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -1.2421875, "rewards/margins": 1.4921875, "rewards/rejected": -2.734375, "step": 1190 }, { "epoch": 0.09028666014596344, "grad_norm": 14.321432019819634, "learning_rate": 4.5112781954887213e-07, "logits/chosen": -1.7265625, "logits/rejected": -1.546875, "logps/chosen": -330.0, "logps/rejected": -490.0, "loss": 0.3917, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -1.3828125, "rewards/margins": 1.5546875, "rewards/rejected": -2.9375, "step": 1200 }, { "epoch": 0.09103904898051313, "grad_norm": 9.331488901948692, "learning_rate": 4.5488721804511275e-07, "logits/chosen": -1.6953125, "logits/rejected": -1.2578125, "logps/chosen": -324.0, "logps/rejected": -536.0, "loss": 0.3832, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -1.3671875, "rewards/margins": 2.171875, "rewards/rejected": -3.53125, "step": 1210 }, { "epoch": 0.09179143781506283, "grad_norm": 15.095275311973246, "learning_rate": 4.586466165413533e-07, "logits/chosen": -1.59375, "logits/rejected": -1.25, "logps/chosen": -310.0, "logps/rejected": -458.0, "loss": 0.3773, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -1.296875, "rewards/margins": 1.6640625, "rewards/rejected": -2.953125, "step": 1220 }, { "epoch": 0.09254382664961253, "grad_norm": 10.82580886221011, "learning_rate": 4.62406015037594e-07, "logits/chosen": -1.6171875, "logits/rejected": -1.4765625, "logps/chosen": -324.0, "logps/rejected": -482.0, "loss": 0.4019, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -1.265625, "rewards/margins": 1.6796875, "rewards/rejected": -2.9375, "step": 1230 }, { "epoch": 0.09329621548416221, "grad_norm": 12.259304024090845, "learning_rate": 4.6616541353383456e-07, "logits/chosen": -1.7421875, "logits/rejected": -1.5625, "logps/chosen": -338.0, "logps/rejected": -468.0, "loss": 0.4065, "rewards/accuracies": 0.7875000238418579, "rewards/chosen": -1.5234375, "rewards/margins": 1.46875, "rewards/rejected": -3.0, "step": 1240 }, { "epoch": 0.09404860431871191, "grad_norm": 10.119466735079934, "learning_rate": 4.699248120300752e-07, "logits/chosen": -1.734375, "logits/rejected": -1.65625, "logps/chosen": -350.0, "logps/rejected": -434.0, "loss": 0.3866, "rewards/accuracies": 0.8125, "rewards/chosen": -1.375, "rewards/margins": 1.1875, "rewards/rejected": -2.5625, "step": 1250 }, { "epoch": 0.09480099315326161, "grad_norm": 10.314247098039594, "learning_rate": 4.7368421052631574e-07, "logits/chosen": -1.796875, "logits/rejected": -1.640625, "logps/chosen": -344.0, "logps/rejected": -496.0, "loss": 0.3997, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -1.578125, "rewards/margins": 1.6875, "rewards/rejected": -3.265625, "step": 1260 }, { "epoch": 0.0955533819878113, "grad_norm": 12.216469828402198, "learning_rate": 4.774436090225564e-07, "logits/chosen": -1.71875, "logits/rejected": -1.5390625, "logps/chosen": -336.0, "logps/rejected": -472.0, "loss": 0.3609, "rewards/accuracies": 0.8187500238418579, "rewards/chosen": -1.515625, "rewards/margins": 1.4453125, "rewards/rejected": -2.96875, "step": 1270 }, { "epoch": 0.096305770822361, "grad_norm": 11.089907603442242, "learning_rate": 4.812030075187969e-07, "logits/chosen": -1.6328125, "logits/rejected": -1.5625, "logps/chosen": -330.0, "logps/rejected": -484.0, "loss": 0.3825, "rewards/accuracies": 0.84375, "rewards/chosen": -1.4140625, "rewards/margins": 1.640625, "rewards/rejected": -3.046875, "step": 1280 }, { "epoch": 0.0970581596569107, "grad_norm": 9.377379394587553, "learning_rate": 4.849624060150376e-07, "logits/chosen": -1.6171875, "logits/rejected": -1.4453125, "logps/chosen": -336.0, "logps/rejected": -474.0, "loss": 0.3713, "rewards/accuracies": 0.84375, "rewards/chosen": -1.453125, "rewards/margins": 1.6640625, "rewards/rejected": -3.109375, "step": 1290 }, { "epoch": 0.09781054849146038, "grad_norm": 9.729201601097698, "learning_rate": 4.887218045112782e-07, "logits/chosen": -1.75, "logits/rejected": -1.6328125, "logps/chosen": -358.0, "logps/rejected": -496.0, "loss": 0.3908, "rewards/accuracies": 0.793749988079071, "rewards/chosen": -1.4765625, "rewards/margins": 1.6640625, "rewards/rejected": -3.140625, "step": 1300 }, { "epoch": 0.09856293732601008, "grad_norm": 13.60068925946147, "learning_rate": 4.924812030075187e-07, "logits/chosen": -1.671875, "logits/rejected": -1.4609375, "logps/chosen": -350.0, "logps/rejected": -476.0, "loss": 0.3647, "rewards/accuracies": 0.824999988079071, "rewards/chosen": -1.453125, "rewards/margins": 1.5703125, "rewards/rejected": -3.03125, "step": 1310 }, { "epoch": 0.09931532616055978, "grad_norm": 9.636174000460331, "learning_rate": 4.962406015037593e-07, "logits/chosen": -1.75, "logits/rejected": -1.5703125, "logps/chosen": -354.0, "logps/rejected": -490.0, "loss": 0.3768, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -1.6953125, "rewards/margins": 1.609375, "rewards/rejected": -3.296875, "step": 1320 }, { "epoch": 0.10006771499510947, "grad_norm": 9.943050722563664, "learning_rate": 5e-07, "logits/chosen": -1.6875, "logits/rejected": -1.5078125, "logps/chosen": -320.0, "logps/rejected": -490.0, "loss": 0.3651, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -1.390625, "rewards/margins": 1.71875, "rewards/rejected": -3.109375, "step": 1330 }, { "epoch": 0.10082010382965917, "grad_norm": 9.504729651291504, "learning_rate": 4.999991376679495e-07, "logits/chosen": -1.59375, "logits/rejected": -1.53125, "logps/chosen": -330.0, "logps/rejected": -508.0, "loss": 0.3499, "rewards/accuracies": 0.84375, "rewards/chosen": -1.3828125, "rewards/margins": 1.875, "rewards/rejected": -3.25, "step": 1340 }, { "epoch": 0.10157249266420887, "grad_norm": 11.321252061101601, "learning_rate": 4.999965506777466e-07, "logits/chosen": -1.7421875, "logits/rejected": -1.5546875, "logps/chosen": -350.0, "logps/rejected": -604.0, "loss": 0.3684, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -1.5703125, "rewards/margins": 2.515625, "rewards/rejected": -4.09375, "step": 1350 }, { "epoch": 0.10232488149875855, "grad_norm": 10.825796197738248, "learning_rate": 4.999922390472384e-07, "logits/chosen": -1.71875, "logits/rejected": -1.5703125, "logps/chosen": -352.0, "logps/rejected": -512.0, "loss": 0.366, "rewards/accuracies": 0.84375, "rewards/chosen": -1.578125, "rewards/margins": 1.734375, "rewards/rejected": -3.3125, "step": 1360 }, { "epoch": 0.10307727033330825, "grad_norm": 8.718633228945263, "learning_rate": 4.999862028061692e-07, "logits/chosen": -1.71875, "logits/rejected": -1.4765625, "logps/chosen": -336.0, "logps/rejected": -474.0, "loss": 0.3738, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -1.4609375, "rewards/margins": 1.515625, "rewards/rejected": -2.96875, "step": 1370 }, { "epoch": 0.10382965916785795, "grad_norm": 9.367978341570046, "learning_rate": 4.99978441996181e-07, "logits/chosen": -1.6953125, "logits/rejected": -1.421875, "logps/chosen": -334.0, "logps/rejected": -464.0, "loss": 0.3805, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -1.3515625, "rewards/margins": 1.484375, "rewards/rejected": -2.84375, "step": 1380 }, { "epoch": 0.10458204800240764, "grad_norm": 9.7883115379069, "learning_rate": 4.999689566708128e-07, "logits/chosen": -1.8125, "logits/rejected": -1.6328125, "logps/chosen": -350.0, "logps/rejected": -496.0, "loss": 0.3652, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -1.5625, "rewards/margins": 1.671875, "rewards/rejected": -3.234375, "step": 1390 }, { "epoch": 0.10533443683695734, "grad_norm": 11.91915529543043, "learning_rate": 4.999577468955008e-07, "logits/chosen": -1.734375, "logits/rejected": -1.5859375, "logps/chosen": -356.0, "logps/rejected": -516.0, "loss": 0.3909, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -1.6875, "rewards/margins": 1.7578125, "rewards/rejected": -3.453125, "step": 1400 }, { "epoch": 0.10608682567150704, "grad_norm": 9.69306913255843, "learning_rate": 4.999448127475773e-07, "logits/chosen": -1.8203125, "logits/rejected": -1.4765625, "logps/chosen": -322.0, "logps/rejected": -516.0, "loss": 0.3524, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -1.3984375, "rewards/margins": 2.078125, "rewards/rejected": -3.46875, "step": 1410 }, { "epoch": 0.10683921450605673, "grad_norm": 11.340012291236317, "learning_rate": 4.999301543162706e-07, "logits/chosen": -1.6953125, "logits/rejected": -1.578125, "logps/chosen": -344.0, "logps/rejected": -532.0, "loss": 0.3465, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -1.53125, "rewards/margins": 1.8125, "rewards/rejected": -3.34375, "step": 1420 }, { "epoch": 0.10759160334060643, "grad_norm": 11.120878941326632, "learning_rate": 4.999137717027041e-07, "logits/chosen": -1.6953125, "logits/rejected": -1.5, "logps/chosen": -348.0, "logps/rejected": -520.0, "loss": 0.3648, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -1.7265625, "rewards/margins": 1.765625, "rewards/rejected": -3.484375, "step": 1430 }, { "epoch": 0.10834399217515613, "grad_norm": 19.737768722008916, "learning_rate": 4.998956650198959e-07, "logits/chosen": -1.6796875, "logits/rejected": -1.59375, "logps/chosen": -320.0, "logps/rejected": -466.0, "loss": 0.356, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -1.40625, "rewards/margins": 1.53125, "rewards/rejected": -2.9375, "step": 1440 }, { "epoch": 0.10909638100970581, "grad_norm": 15.00872847702333, "learning_rate": 4.998758343927576e-07, "logits/chosen": -1.7265625, "logits/rejected": -1.6484375, "logps/chosen": -378.0, "logps/rejected": -524.0, "loss": 0.3515, "rewards/accuracies": 0.793749988079071, "rewards/chosen": -1.8125, "rewards/margins": 1.6328125, "rewards/rejected": -3.4375, "step": 1450 }, { "epoch": 0.10984876984425551, "grad_norm": 7.673477876279292, "learning_rate": 4.998542799580941e-07, "logits/chosen": -1.7265625, "logits/rejected": -1.484375, "logps/chosen": -354.0, "logps/rejected": -492.0, "loss": 0.3643, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -1.6171875, "rewards/margins": 1.6484375, "rewards/rejected": -3.265625, "step": 1460 }, { "epoch": 0.11060115867880521, "grad_norm": 19.58708769357009, "learning_rate": 4.998310018646021e-07, "logits/chosen": -1.6875, "logits/rejected": -1.640625, "logps/chosen": -334.0, "logps/rejected": -446.0, "loss": 0.382, "rewards/accuracies": 0.8187500238418579, "rewards/chosen": -1.4375, "rewards/margins": 1.3671875, "rewards/rejected": -2.796875, "step": 1470 }, { "epoch": 0.1113535475133549, "grad_norm": 9.057199107122731, "learning_rate": 4.998060002728689e-07, "logits/chosen": -1.765625, "logits/rejected": -1.6484375, "logps/chosen": -384.0, "logps/rejected": -544.0, "loss": 0.3446, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": -1.78125, "rewards/margins": 1.828125, "rewards/rejected": -3.609375, "step": 1480 }, { "epoch": 0.1121059363479046, "grad_norm": 9.136587896355477, "learning_rate": 4.99779275355372e-07, "logits/chosen": -1.828125, "logits/rejected": -1.625, "logps/chosen": -378.0, "logps/rejected": -536.0, "loss": 0.3729, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -1.71875, "rewards/margins": 1.609375, "rewards/rejected": -3.328125, "step": 1490 }, { "epoch": 0.1128583251824543, "grad_norm": 13.165762942765546, "learning_rate": 4.997508272964775e-07, "logits/chosen": -1.75, "logits/rejected": -1.484375, "logps/chosen": -344.0, "logps/rejected": -536.0, "loss": 0.3415, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -1.4921875, "rewards/margins": 2.046875, "rewards/rejected": -3.53125, "step": 1500 }, { "epoch": 0.11361071401700398, "grad_norm": 8.898203621221239, "learning_rate": 4.997206562924387e-07, "logits/chosen": -1.8203125, "logits/rejected": -1.59375, "logps/chosen": -392.0, "logps/rejected": -556.0, "loss": 0.3522, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -1.90625, "rewards/margins": 1.8515625, "rewards/rejected": -3.765625, "step": 1510 }, { "epoch": 0.11436310285155368, "grad_norm": 9.085548813720974, "learning_rate": 4.99688762551395e-07, "logits/chosen": -1.703125, "logits/rejected": -1.515625, "logps/chosen": -350.0, "logps/rejected": -470.0, "loss": 0.3583, "rewards/accuracies": 0.8187500238418579, "rewards/chosen": -1.6484375, "rewards/margins": 1.484375, "rewards/rejected": -3.125, "step": 1520 }, { "epoch": 0.11511549168610338, "grad_norm": 10.780024391996617, "learning_rate": 4.996551462933705e-07, "logits/chosen": -1.75, "logits/rejected": -1.375, "logps/chosen": -372.0, "logps/rejected": -536.0, "loss": 0.3595, "rewards/accuracies": 0.84375, "rewards/chosen": -1.765625, "rewards/margins": 1.828125, "rewards/rejected": -3.59375, "step": 1530 }, { "epoch": 0.11586788052065307, "grad_norm": 9.318904950174627, "learning_rate": 4.996198077502719e-07, "logits/chosen": -1.65625, "logits/rejected": -1.578125, "logps/chosen": -318.0, "logps/rejected": -478.0, "loss": 0.349, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -1.328125, "rewards/margins": 1.8203125, "rewards/rejected": -3.140625, "step": 1540 }, { "epoch": 0.11662026935520277, "grad_norm": 11.890428313495384, "learning_rate": 4.99582747165888e-07, "logits/chosen": -1.8515625, "logits/rejected": -1.6796875, "logps/chosen": -376.0, "logps/rejected": -596.0, "loss": 0.3529, "rewards/accuracies": 0.84375, "rewards/chosen": -1.8671875, "rewards/margins": 1.9765625, "rewards/rejected": -3.84375, "step": 1550 }, { "epoch": 0.11737265818975247, "grad_norm": 9.301832109310098, "learning_rate": 4.995439647958869e-07, "logits/chosen": -1.9375, "logits/rejected": -1.7578125, "logps/chosen": -350.0, "logps/rejected": -512.0, "loss": 0.3486, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -1.5546875, "rewards/margins": 1.765625, "rewards/rejected": -3.328125, "step": 1560 }, { "epoch": 0.11812504702430215, "grad_norm": 9.933482316327112, "learning_rate": 4.995034609078148e-07, "logits/chosen": -1.84375, "logits/rejected": -1.7109375, "logps/chosen": -352.0, "logps/rejected": -540.0, "loss": 0.3494, "rewards/accuracies": 0.84375, "rewards/chosen": -1.703125, "rewards/margins": 1.8984375, "rewards/rejected": -3.59375, "step": 1570 }, { "epoch": 0.11887743585885185, "grad_norm": 10.522564254534228, "learning_rate": 4.994612357810942e-07, "logits/chosen": -2.015625, "logits/rejected": -1.78125, "logps/chosen": -438.0, "logps/rejected": -584.0, "loss": 0.3357, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -2.171875, "rewards/margins": 1.890625, "rewards/rejected": -4.0625, "step": 1580 }, { "epoch": 0.11962982469340155, "grad_norm": 16.739348218533863, "learning_rate": 4.994172897070217e-07, "logits/chosen": -1.734375, "logits/rejected": -1.4765625, "logps/chosen": -366.0, "logps/rejected": -528.0, "loss": 0.3578, "rewards/accuracies": 0.84375, "rewards/chosen": -1.6796875, "rewards/margins": 1.75, "rewards/rejected": -3.4375, "step": 1590 }, { "epoch": 0.12038221352795124, "grad_norm": 8.771366624270362, "learning_rate": 4.99371622988766e-07, "logits/chosen": -1.8984375, "logits/rejected": -1.7265625, "logps/chosen": -356.0, "logps/rejected": -544.0, "loss": 0.3417, "rewards/accuracies": 0.824999988079071, "rewards/chosen": -1.78125, "rewards/margins": 1.8515625, "rewards/rejected": -3.625, "step": 1600 }, { "epoch": 0.12113460236250094, "grad_norm": 9.888835611485346, "learning_rate": 4.993242359413664e-07, "logits/chosen": -1.9296875, "logits/rejected": -1.75, "logps/chosen": -368.0, "logps/rejected": -572.0, "loss": 0.3563, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -1.8046875, "rewards/margins": 2.03125, "rewards/rejected": -3.84375, "step": 1610 }, { "epoch": 0.12188699119705064, "grad_norm": 10.183044772499086, "learning_rate": 4.992751288917297e-07, "logits/chosen": -1.8984375, "logits/rejected": -1.7890625, "logps/chosen": -350.0, "logps/rejected": -532.0, "loss": 0.3377, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -1.625, "rewards/margins": 1.9765625, "rewards/rejected": -3.59375, "step": 1620 }, { "epoch": 0.12263938003160033, "grad_norm": 9.44769650505056, "learning_rate": 4.992243021786285e-07, "logits/chosen": -1.875, "logits/rejected": -1.7890625, "logps/chosen": -376.0, "logps/rejected": -536.0, "loss": 0.3258, "rewards/accuracies": 0.8125, "rewards/chosen": -1.96875, "rewards/margins": 1.6171875, "rewards/rejected": -3.59375, "step": 1630 }, { "epoch": 0.12339176886615003, "grad_norm": 8.841099026467015, "learning_rate": 4.99171756152699e-07, "logits/chosen": -1.8671875, "logits/rejected": -1.6796875, "logps/chosen": -372.0, "logps/rejected": -576.0, "loss": 0.3376, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -1.59375, "rewards/margins": 2.265625, "rewards/rejected": -3.859375, "step": 1640 }, { "epoch": 0.12414415770069973, "grad_norm": 10.552216445310533, "learning_rate": 4.991174911764381e-07, "logits/chosen": -1.8203125, "logits/rejected": -1.609375, "logps/chosen": -382.0, "logps/rejected": -556.0, "loss": 0.3321, "rewards/accuracies": 0.875, "rewards/chosen": -1.8828125, "rewards/margins": 1.7421875, "rewards/rejected": -3.625, "step": 1650 }, { "epoch": 0.12489654653524941, "grad_norm": 10.190507893880179, "learning_rate": 4.990615076242011e-07, "logits/chosen": -2.03125, "logits/rejected": -1.8046875, "logps/chosen": -398.0, "logps/rejected": -644.0, "loss": 0.3435, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -1.859375, "rewards/margins": 2.515625, "rewards/rejected": -4.375, "step": 1660 }, { "epoch": 0.12564893536979913, "grad_norm": 9.874328523547966, "learning_rate": 4.990038058821995e-07, "logits/chosen": -1.859375, "logits/rejected": -1.8125, "logps/chosen": -360.0, "logps/rejected": -528.0, "loss": 0.3356, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": -1.75, "rewards/margins": 1.7734375, "rewards/rejected": -3.515625, "step": 1670 }, { "epoch": 0.1264013242043488, "grad_norm": 9.773695577810862, "learning_rate": 4.989443863484976e-07, "logits/chosen": -1.9375, "logits/rejected": -1.875, "logps/chosen": -366.0, "logps/rejected": -524.0, "loss": 0.3394, "rewards/accuracies": 0.8187500238418579, "rewards/chosen": -1.8046875, "rewards/margins": 1.75, "rewards/rejected": -3.546875, "step": 1680 }, { "epoch": 0.1271537130388985, "grad_norm": 11.668116001436987, "learning_rate": 4.988832494330106e-07, "logits/chosen": -1.84375, "logits/rejected": -1.7578125, "logps/chosen": -388.0, "logps/rejected": -576.0, "loss": 0.3598, "rewards/accuracies": 0.8187500238418579, "rewards/chosen": -1.953125, "rewards/margins": 1.765625, "rewards/rejected": -3.71875, "step": 1690 }, { "epoch": 0.1279061018734482, "grad_norm": 10.851636396005233, "learning_rate": 4.988203955575006e-07, "logits/chosen": -1.921875, "logits/rejected": -1.8125, "logps/chosen": -372.0, "logps/rejected": -572.0, "loss": 0.3213, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -1.75, "rewards/margins": 2.0625, "rewards/rejected": -3.8125, "step": 1700 }, { "epoch": 0.1286584907079979, "grad_norm": 11.17187154058831, "learning_rate": 4.987558251555755e-07, "logits/chosen": -1.8515625, "logits/rejected": -1.890625, "logps/chosen": -372.0, "logps/rejected": -556.0, "loss": 0.3187, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -1.796875, "rewards/margins": 1.953125, "rewards/rejected": -3.75, "step": 1710 }, { "epoch": 0.1294108795425476, "grad_norm": 7.705081401915257, "learning_rate": 4.98689538672684e-07, "logits/chosen": -1.9296875, "logits/rejected": -1.65625, "logps/chosen": -382.0, "logps/rejected": -580.0, "loss": 0.3207, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -1.875, "rewards/margins": 2.28125, "rewards/rejected": -4.15625, "step": 1720 }, { "epoch": 0.1301632683770973, "grad_norm": 13.997421163967692, "learning_rate": 4.986215365661137e-07, "logits/chosen": -1.8671875, "logits/rejected": -1.8125, "logps/chosen": -400.0, "logps/rejected": -600.0, "loss": 0.3256, "rewards/accuracies": 0.824999988079071, "rewards/chosen": -2.09375, "rewards/margins": 2.015625, "rewards/rejected": -4.09375, "step": 1730 }, { "epoch": 0.13091565721164697, "grad_norm": 11.512452495012095, "learning_rate": 4.985518193049879e-07, "logits/chosen": -1.9140625, "logits/rejected": -1.8203125, "logps/chosen": -402.0, "logps/rejected": -576.0, "loss": 0.3523, "rewards/accuracies": 0.84375, "rewards/chosen": -1.8828125, "rewards/margins": 2.0, "rewards/rejected": -3.875, "step": 1740 }, { "epoch": 0.13166804604619667, "grad_norm": 10.281211014855446, "learning_rate": 4.984803873702619e-07, "logits/chosen": -1.8515625, "logits/rejected": -1.78125, "logps/chosen": -366.0, "logps/rejected": -536.0, "loss": 0.3569, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -1.734375, "rewards/margins": 1.796875, "rewards/rejected": -3.53125, "step": 1750 }, { "epoch": 0.13242043488074637, "grad_norm": 10.983213211180036, "learning_rate": 4.984072412547202e-07, "logits/chosen": -1.875, "logits/rejected": -1.8515625, "logps/chosen": -370.0, "logps/rejected": -564.0, "loss": 0.3407, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -1.9296875, "rewards/margins": 1.9765625, "rewards/rejected": -3.90625, "step": 1760 }, { "epoch": 0.13317282371529607, "grad_norm": 11.617051268358038, "learning_rate": 4.983323814629727e-07, "logits/chosen": -1.953125, "logits/rejected": -1.78125, "logps/chosen": -394.0, "logps/rejected": -564.0, "loss": 0.3585, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -2.03125, "rewards/margins": 1.8046875, "rewards/rejected": -3.828125, "step": 1770 }, { "epoch": 0.13392521254984577, "grad_norm": 14.978908016353657, "learning_rate": 4.982558085114515e-07, "logits/chosen": -1.90625, "logits/rejected": -1.8671875, "logps/chosen": -398.0, "logps/rejected": -564.0, "loss": 0.3535, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.15625, "rewards/margins": 1.8359375, "rewards/rejected": -3.984375, "step": 1780 }, { "epoch": 0.13467760138439547, "grad_norm": 8.919207621155557, "learning_rate": 4.981775229284068e-07, "logits/chosen": -1.8828125, "logits/rejected": -1.765625, "logps/chosen": -392.0, "logps/rejected": -560.0, "loss": 0.3501, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": -1.984375, "rewards/margins": 1.828125, "rewards/rejected": -3.8125, "step": 1790 }, { "epoch": 0.13542999021894514, "grad_norm": 9.626125132890424, "learning_rate": 4.98097525253904e-07, "logits/chosen": -1.859375, "logits/rejected": -1.734375, "logps/chosen": -368.0, "logps/rejected": -516.0, "loss": 0.3585, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -1.71875, "rewards/margins": 1.625, "rewards/rejected": -3.34375, "step": 1800 }, { "epoch": 0.13618237905349484, "grad_norm": 9.493860732364704, "learning_rate": 4.980158160398198e-07, "logits/chosen": -1.96875, "logits/rejected": -1.734375, "logps/chosen": -374.0, "logps/rejected": -556.0, "loss": 0.327, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -1.8828125, "rewards/margins": 2.015625, "rewards/rejected": -3.890625, "step": 1810 }, { "epoch": 0.13693476788804454, "grad_norm": 15.810638317433718, "learning_rate": 4.979323958498378e-07, "logits/chosen": -1.96875, "logits/rejected": -1.8125, "logps/chosen": -404.0, "logps/rejected": -584.0, "loss": 0.3315, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -2.09375, "rewards/margins": 2.203125, "rewards/rejected": -4.28125, "step": 1820 }, { "epoch": 0.13768715672259424, "grad_norm": 10.801350446777317, "learning_rate": 4.978472652594453e-07, "logits/chosen": -1.9140625, "logits/rejected": -1.7890625, "logps/chosen": -370.0, "logps/rejected": -584.0, "loss": 0.3377, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -1.921875, "rewards/margins": 2.21875, "rewards/rejected": -4.125, "step": 1830 }, { "epoch": 0.13843954555714394, "grad_norm": 10.034921670700433, "learning_rate": 4.977604248559289e-07, "logits/chosen": -1.9609375, "logits/rejected": -1.8125, "logps/chosen": -376.0, "logps/rejected": -580.0, "loss": 0.3241, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -1.984375, "rewards/margins": 2.078125, "rewards/rejected": -4.0625, "step": 1840 }, { "epoch": 0.13919193439169364, "grad_norm": 11.326375865101697, "learning_rate": 4.976718752383709e-07, "logits/chosen": -2.03125, "logits/rejected": -1.953125, "logps/chosen": -398.0, "logps/rejected": -604.0, "loss": 0.3332, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.15625, "rewards/margins": 2.109375, "rewards/rejected": -4.28125, "step": 1850 }, { "epoch": 0.1399443232262433, "grad_norm": 9.957695070457628, "learning_rate": 4.975816170176445e-07, "logits/chosen": -1.9453125, "logits/rejected": -1.8359375, "logps/chosen": -398.0, "logps/rejected": -596.0, "loss": 0.3211, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -1.9609375, "rewards/margins": 2.171875, "rewards/rejected": -4.125, "step": 1860 }, { "epoch": 0.140696712060793, "grad_norm": 10.628606355174536, "learning_rate": 4.974896508164101e-07, "logits/chosen": -2.03125, "logits/rejected": -1.8828125, "logps/chosen": -388.0, "logps/rejected": -564.0, "loss": 0.337, "rewards/accuracies": 0.824999988079071, "rewards/chosen": -2.0625, "rewards/margins": 1.8984375, "rewards/rejected": -3.96875, "step": 1870 }, { "epoch": 0.1414491008953427, "grad_norm": 8.344548178109614, "learning_rate": 4.973959772691112e-07, "logits/chosen": -1.890625, "logits/rejected": -1.71875, "logps/chosen": -376.0, "logps/rejected": -624.0, "loss": 0.3377, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -1.9375, "rewards/margins": 2.375, "rewards/rejected": -4.3125, "step": 1880 }, { "epoch": 0.1422014897298924, "grad_norm": 11.2770423636356, "learning_rate": 4.973005970219692e-07, "logits/chosen": -2.046875, "logits/rejected": -1.875, "logps/chosen": -428.0, "logps/rejected": -604.0, "loss": 0.3156, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.296875, "rewards/margins": 2.09375, "rewards/rejected": -4.375, "step": 1890 }, { "epoch": 0.1429538785644421, "grad_norm": 17.6581231943882, "learning_rate": 4.972035107329796e-07, "logits/chosen": -2.046875, "logits/rejected": -1.859375, "logps/chosen": -442.0, "logps/rejected": -644.0, "loss": 0.3082, "rewards/accuracies": 0.875, "rewards/chosen": -2.421875, "rewards/margins": 2.28125, "rewards/rejected": -4.6875, "step": 1900 }, { "epoch": 0.1437062673989918, "grad_norm": 10.887275343960397, "learning_rate": 4.971047190719076e-07, "logits/chosen": -1.84375, "logits/rejected": -1.65625, "logps/chosen": -376.0, "logps/rejected": -628.0, "loss": 0.3013, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -2.0625, "rewards/margins": 2.203125, "rewards/rejected": -4.28125, "step": 1910 }, { "epoch": 0.14445865623354148, "grad_norm": 25.328609129924764, "learning_rate": 4.970042227202828e-07, "logits/chosen": -2.015625, "logits/rejected": -1.859375, "logps/chosen": -414.0, "logps/rejected": -644.0, "loss": 0.3042, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": -2.375, "rewards/margins": 2.296875, "rewards/rejected": -4.6875, "step": 1920 }, { "epoch": 0.14521104506809118, "grad_norm": 11.151478872436398, "learning_rate": 4.969020223713948e-07, "logits/chosen": -1.953125, "logits/rejected": -1.6875, "logps/chosen": -410.0, "logps/rejected": -624.0, "loss": 0.3146, "rewards/accuracies": 0.875, "rewards/chosen": -2.1875, "rewards/margins": 2.296875, "rewards/rejected": -4.46875, "step": 1930 }, { "epoch": 0.14596343390264088, "grad_norm": 14.69037417305483, "learning_rate": 4.967981187302889e-07, "logits/chosen": -2.171875, "logits/rejected": -1.9375, "logps/chosen": -448.0, "logps/rejected": -680.0, "loss": 0.3239, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.53125, "rewards/margins": 2.421875, "rewards/rejected": -4.96875, "step": 1940 }, { "epoch": 0.14671582273719058, "grad_norm": 10.469012625466451, "learning_rate": 4.966925125137607e-07, "logits/chosen": -1.96875, "logits/rejected": -1.6875, "logps/chosen": -394.0, "logps/rejected": -596.0, "loss": 0.3028, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.078125, "rewards/margins": 2.171875, "rewards/rejected": -4.25, "step": 1950 }, { "epoch": 0.14746821157174028, "grad_norm": 10.586567120161504, "learning_rate": 4.965852044503512e-07, "logits/chosen": -1.828125, "logits/rejected": -1.6875, "logps/chosen": -398.0, "logps/rejected": -624.0, "loss": 0.33, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.046875, "rewards/margins": 2.125, "rewards/rejected": -4.1875, "step": 1960 }, { "epoch": 0.14822060040628998, "grad_norm": 10.716253759810703, "learning_rate": 4.964761952803416e-07, "logits/chosen": -1.9140625, "logits/rejected": -1.8359375, "logps/chosen": -416.0, "logps/rejected": -600.0, "loss": 0.3344, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": -2.359375, "rewards/margins": 1.859375, "rewards/rejected": -4.21875, "step": 1970 }, { "epoch": 0.14897298924083965, "grad_norm": 11.53550871318456, "learning_rate": 4.963654857557488e-07, "logits/chosen": -1.8828125, "logits/rejected": -1.828125, "logps/chosen": -416.0, "logps/rejected": -596.0, "loss": 0.3278, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -2.265625, "rewards/margins": 2.0, "rewards/rejected": -4.28125, "step": 1980 }, { "epoch": 0.14972537807538935, "grad_norm": 7.824848270680874, "learning_rate": 4.962530766403199e-07, "logits/chosen": -2.015625, "logits/rejected": -1.9140625, "logps/chosen": -394.0, "logps/rejected": -600.0, "loss": 0.3048, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -2.140625, "rewards/margins": 2.09375, "rewards/rejected": -4.25, "step": 1990 }, { "epoch": 0.15047776690993905, "grad_norm": 10.54357470913026, "learning_rate": 4.961389687095267e-07, "logits/chosen": -1.9921875, "logits/rejected": -1.8984375, "logps/chosen": -402.0, "logps/rejected": -576.0, "loss": 0.3149, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": -2.109375, "rewards/margins": 1.9140625, "rewards/rejected": -4.03125, "step": 2000 }, { "epoch": 0.15123015574448875, "grad_norm": 11.50470119906008, "learning_rate": 4.960231627505606e-07, "logits/chosen": -1.9609375, "logits/rejected": -1.765625, "logps/chosen": -410.0, "logps/rejected": -616.0, "loss": 0.3096, "rewards/accuracies": 0.875, "rewards/chosen": -2.109375, "rewards/margins": 2.296875, "rewards/rejected": -4.40625, "step": 2010 }, { "epoch": 0.15198254457903845, "grad_norm": 10.24435969209101, "learning_rate": 4.959056595623271e-07, "logits/chosen": -1.921875, "logits/rejected": -1.8125, "logps/chosen": -438.0, "logps/rejected": -632.0, "loss": 0.3171, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -2.359375, "rewards/margins": 2.15625, "rewards/rejected": -4.5, "step": 2020 }, { "epoch": 0.15273493341358815, "grad_norm": 9.725090982745549, "learning_rate": 4.957864599554404e-07, "logits/chosen": -1.984375, "logits/rejected": -1.609375, "logps/chosen": -368.0, "logps/rejected": -564.0, "loss": 0.3474, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -1.8125, "rewards/margins": 2.015625, "rewards/rejected": -3.828125, "step": 2030 }, { "epoch": 0.15348732224813783, "grad_norm": 12.138046757461087, "learning_rate": 4.956655647522176e-07, "logits/chosen": -1.9765625, "logits/rejected": -1.8359375, "logps/chosen": -446.0, "logps/rejected": -648.0, "loss": 0.2982, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.4375, "rewards/margins": 2.28125, "rewards/rejected": -4.71875, "step": 2040 }, { "epoch": 0.15423971108268753, "grad_norm": 10.157442606730516, "learning_rate": 4.95542974786673e-07, "logits/chosen": -2.03125, "logits/rejected": -1.8125, "logps/chosen": -444.0, "logps/rejected": -656.0, "loss": 0.3035, "rewards/accuracies": 0.824999988079071, "rewards/chosen": -2.5, "rewards/margins": 2.28125, "rewards/rejected": -4.78125, "step": 2050 }, { "epoch": 0.15499209991723722, "grad_norm": 11.277965313033228, "learning_rate": 4.954186909045129e-07, "logits/chosen": -2.109375, "logits/rejected": -2.046875, "logps/chosen": -410.0, "logps/rejected": -608.0, "loss": 0.3277, "rewards/accuracies": 0.875, "rewards/chosen": -2.234375, "rewards/margins": 2.140625, "rewards/rejected": -4.375, "step": 2060 }, { "epoch": 0.15574448875178692, "grad_norm": 8.761479030747685, "learning_rate": 4.95292713963129e-07, "logits/chosen": -2.125, "logits/rejected": -1.9765625, "logps/chosen": -398.0, "logps/rejected": -580.0, "loss": 0.3266, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.171875, "rewards/margins": 1.953125, "rewards/rejected": -4.125, "step": 2070 }, { "epoch": 0.15649687758633662, "grad_norm": 9.629447968813118, "learning_rate": 4.951650448315929e-07, "logits/chosen": -2.015625, "logits/rejected": -1.9375, "logps/chosen": -414.0, "logps/rejected": -588.0, "loss": 0.3076, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.21875, "rewards/margins": 1.921875, "rewards/rejected": -4.15625, "step": 2080 }, { "epoch": 0.15724926642088632, "grad_norm": 8.848523398423048, "learning_rate": 4.950356843906501e-07, "logits/chosen": -2.140625, "logits/rejected": -2.0625, "logps/chosen": -432.0, "logps/rejected": -636.0, "loss": 0.3522, "rewards/accuracies": 0.8187500238418579, "rewards/chosen": -2.5, "rewards/margins": 2.0625, "rewards/rejected": -4.5625, "step": 2090 }, { "epoch": 0.158001655255436, "grad_norm": 14.714867202226637, "learning_rate": 4.949046335327138e-07, "logits/chosen": -2.046875, "logits/rejected": -2.0, "logps/chosen": -406.0, "logps/rejected": -608.0, "loss": 0.2989, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.09375, "rewards/margins": 2.15625, "rewards/rejected": -4.25, "step": 2100 }, { "epoch": 0.1587540440899857, "grad_norm": 13.38924937184168, "learning_rate": 4.94771893161859e-07, "logits/chosen": -2.046875, "logits/rejected": -1.796875, "logps/chosen": -426.0, "logps/rejected": -640.0, "loss": 0.3255, "rewards/accuracies": 0.8125, "rewards/chosen": -2.3125, "rewards/margins": 2.328125, "rewards/rejected": -4.65625, "step": 2110 }, { "epoch": 0.1595064329245354, "grad_norm": 9.794286572212386, "learning_rate": 4.946374641938157e-07, "logits/chosen": -1.9921875, "logits/rejected": -1.9765625, "logps/chosen": -424.0, "logps/rejected": -604.0, "loss": 0.3338, "rewards/accuracies": 0.8125, "rewards/chosen": -2.390625, "rewards/margins": 1.90625, "rewards/rejected": -4.3125, "step": 2120 }, { "epoch": 0.1602588217590851, "grad_norm": 11.143311358657257, "learning_rate": 4.945013475559632e-07, "logits/chosen": -2.046875, "logits/rejected": -1.9921875, "logps/chosen": -442.0, "logps/rejected": -608.0, "loss": 0.3177, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.375, "rewards/margins": 1.8984375, "rewards/rejected": -4.28125, "step": 2130 }, { "epoch": 0.1610112105936348, "grad_norm": 12.209812257424028, "learning_rate": 4.943635441873235e-07, "logits/chosen": -1.984375, "logits/rejected": -1.921875, "logps/chosen": -412.0, "logps/rejected": -624.0, "loss": 0.3155, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -2.265625, "rewards/margins": 2.203125, "rewards/rejected": -4.46875, "step": 2140 }, { "epoch": 0.1617635994281845, "grad_norm": 12.412606003098777, "learning_rate": 4.942240550385547e-07, "logits/chosen": -2.0625, "logits/rejected": -1.9765625, "logps/chosen": -402.0, "logps/rejected": -620.0, "loss": 0.2921, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.15625, "rewards/margins": 2.25, "rewards/rejected": -4.40625, "step": 2150 }, { "epoch": 0.16251598826273417, "grad_norm": 13.49359353435683, "learning_rate": 4.940828810719444e-07, "logits/chosen": -2.125, "logits/rejected": -2.03125, "logps/chosen": -412.0, "logps/rejected": -628.0, "loss": 0.3083, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.21875, "rewards/margins": 2.296875, "rewards/rejected": -4.53125, "step": 2160 }, { "epoch": 0.16326837709728387, "grad_norm": 10.453383748162928, "learning_rate": 4.939400232614033e-07, "logits/chosen": -1.96875, "logits/rejected": -1.96875, "logps/chosen": -416.0, "logps/rejected": -628.0, "loss": 0.3272, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.28125, "rewards/margins": 2.171875, "rewards/rejected": -4.4375, "step": 2170 }, { "epoch": 0.16402076593183357, "grad_norm": 12.796032641245228, "learning_rate": 4.937954825924585e-07, "logits/chosen": -2.015625, "logits/rejected": -1.96875, "logps/chosen": -412.0, "logps/rejected": -576.0, "loss": 0.2926, "rewards/accuracies": 0.875, "rewards/chosen": -2.296875, "rewards/margins": 1.859375, "rewards/rejected": -4.15625, "step": 2180 }, { "epoch": 0.16477315476638327, "grad_norm": 11.223098791600563, "learning_rate": 4.936492600622464e-07, "logits/chosen": -2.09375, "logits/rejected": -1.984375, "logps/chosen": -458.0, "logps/rejected": -664.0, "loss": 0.3446, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.5, "rewards/margins": 2.40625, "rewards/rejected": -4.90625, "step": 2190 }, { "epoch": 0.16552554360093297, "grad_norm": 10.662678517802988, "learning_rate": 4.935013566795058e-07, "logits/chosen": -2.015625, "logits/rejected": -1.890625, "logps/chosen": -408.0, "logps/rejected": -636.0, "loss": 0.3007, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.296875, "rewards/margins": 2.25, "rewards/rejected": -4.5625, "step": 2200 }, { "epoch": 0.16627793243548267, "grad_norm": 15.148796237888766, "learning_rate": 4.933517734645714e-07, "logits/chosen": -2.046875, "logits/rejected": -1.921875, "logps/chosen": -394.0, "logps/rejected": -680.0, "loss": 0.3017, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.265625, "rewards/margins": 2.71875, "rewards/rejected": -4.96875, "step": 2210 }, { "epoch": 0.16703032127003234, "grad_norm": 9.903147484851852, "learning_rate": 4.932005114493665e-07, "logits/chosen": -2.15625, "logits/rejected": -1.9140625, "logps/chosen": -430.0, "logps/rejected": -684.0, "loss": 0.2949, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.40625, "rewards/margins": 2.640625, "rewards/rejected": -5.03125, "step": 2220 }, { "epoch": 0.16778271010458204, "grad_norm": 13.342073838731102, "learning_rate": 4.930475716773956e-07, "logits/chosen": -2.03125, "logits/rejected": -2.015625, "logps/chosen": -418.0, "logps/rejected": -624.0, "loss": 0.3135, "rewards/accuracies": 0.84375, "rewards/chosen": -2.265625, "rewards/margins": 2.140625, "rewards/rejected": -4.40625, "step": 2230 }, { "epoch": 0.16853509893913174, "grad_norm": 10.430245292813721, "learning_rate": 4.928929552037378e-07, "logits/chosen": -2.078125, "logits/rejected": -1.984375, "logps/chosen": -394.0, "logps/rejected": -588.0, "loss": 0.3136, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.09375, "rewards/margins": 2.15625, "rewards/rejected": -4.25, "step": 2240 }, { "epoch": 0.16928748777368144, "grad_norm": 11.31249119717281, "learning_rate": 4.927366630950389e-07, "logits/chosen": -2.109375, "logits/rejected": -2.109375, "logps/chosen": -406.0, "logps/rejected": -608.0, "loss": 0.2944, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.1875, "rewards/margins": 2.21875, "rewards/rejected": -4.40625, "step": 2250 }, { "epoch": 0.17003987660823114, "grad_norm": 16.817697120625688, "learning_rate": 4.925786964295046e-07, "logits/chosen": -2.015625, "logits/rejected": -1.9296875, "logps/chosen": -448.0, "logps/rejected": -644.0, "loss": 0.3202, "rewards/accuracies": 0.84375, "rewards/chosen": -2.5625, "rewards/margins": 2.171875, "rewards/rejected": -4.75, "step": 2260 }, { "epoch": 0.17079226544278084, "grad_norm": 10.730746729206086, "learning_rate": 4.924190562968926e-07, "logits/chosen": -2.078125, "logits/rejected": -1.9765625, "logps/chosen": -430.0, "logps/rejected": -588.0, "loss": 0.2986, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.3125, "rewards/margins": 1.890625, "rewards/rejected": -4.21875, "step": 2270 }, { "epoch": 0.17154465427733054, "grad_norm": 9.1044148553965, "learning_rate": 4.922577437985052e-07, "logits/chosen": -2.03125, "logits/rejected": -1.8671875, "logps/chosen": -422.0, "logps/rejected": -620.0, "loss": 0.2718, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.375, "rewards/margins": 2.046875, "rewards/rejected": -4.40625, "step": 2280 }, { "epoch": 0.1722970431118802, "grad_norm": 13.449211687764839, "learning_rate": 4.920947600471821e-07, "logits/chosen": -2.140625, "logits/rejected": -2.015625, "logps/chosen": -420.0, "logps/rejected": -632.0, "loss": 0.2992, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -2.328125, "rewards/margins": 2.25, "rewards/rejected": -4.59375, "step": 2290 }, { "epoch": 0.1730494319464299, "grad_norm": 10.539948794467563, "learning_rate": 4.919301061672919e-07, "logits/chosen": -2.140625, "logits/rejected": -1.9140625, "logps/chosen": -426.0, "logps/rejected": -648.0, "loss": 0.2914, "rewards/accuracies": 0.875, "rewards/chosen": -2.421875, "rewards/margins": 2.3125, "rewards/rejected": -4.75, "step": 2300 }, { "epoch": 0.1738018207809796, "grad_norm": 10.856539661997529, "learning_rate": 4.917637832947256e-07, "logits/chosen": -2.109375, "logits/rejected": -2.03125, "logps/chosen": -448.0, "logps/rejected": -660.0, "loss": 0.3229, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -2.4375, "rewards/margins": 2.328125, "rewards/rejected": -4.75, "step": 2310 }, { "epoch": 0.1745542096155293, "grad_norm": 10.32209719190504, "learning_rate": 4.915957925768871e-07, "logits/chosen": -2.078125, "logits/rejected": -2.046875, "logps/chosen": -408.0, "logps/rejected": -620.0, "loss": 0.3064, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.34375, "rewards/margins": 2.15625, "rewards/rejected": -4.5, "step": 2320 }, { "epoch": 0.175306598450079, "grad_norm": 12.448870553490586, "learning_rate": 4.914261351726868e-07, "logits/chosen": -2.046875, "logits/rejected": -1.953125, "logps/chosen": -420.0, "logps/rejected": -636.0, "loss": 0.3192, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.140625, "rewards/margins": 2.328125, "rewards/rejected": -4.46875, "step": 2330 }, { "epoch": 0.1760589872846287, "grad_norm": 9.393207196959352, "learning_rate": 4.91254812252533e-07, "logits/chosen": -2.046875, "logits/rejected": -1.765625, "logps/chosen": -402.0, "logps/rejected": -648.0, "loss": 0.2993, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.171875, "rewards/margins": 2.375, "rewards/rejected": -4.5625, "step": 2340 }, { "epoch": 0.17681137611917838, "grad_norm": 10.403142456009094, "learning_rate": 4.910818249983235e-07, "logits/chosen": -2.171875, "logits/rejected": -2.171875, "logps/chosen": -422.0, "logps/rejected": -632.0, "loss": 0.3022, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.375, "rewards/margins": 2.109375, "rewards/rejected": -4.46875, "step": 2350 }, { "epoch": 0.17756376495372808, "grad_norm": 10.22724105010258, "learning_rate": 4.909071746034379e-07, "logits/chosen": -2.1875, "logits/rejected": -2.046875, "logps/chosen": -436.0, "logps/rejected": -620.0, "loss": 0.3054, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -2.328125, "rewards/margins": 2.015625, "rewards/rejected": -4.34375, "step": 2360 }, { "epoch": 0.17831615378827778, "grad_norm": 10.260718824983782, "learning_rate": 4.907308622727293e-07, "logits/chosen": -2.078125, "logits/rejected": -1.921875, "logps/chosen": -388.0, "logps/rejected": -596.0, "loss": 0.3054, "rewards/accuracies": 0.875, "rewards/chosen": -2.109375, "rewards/margins": 2.1875, "rewards/rejected": -4.3125, "step": 2370 }, { "epoch": 0.17906854262282748, "grad_norm": 7.834160477741724, "learning_rate": 4.90552889222516e-07, "logits/chosen": -2.0, "logits/rejected": -1.953125, "logps/chosen": -408.0, "logps/rejected": -644.0, "loss": 0.2858, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.09375, "rewards/margins": 2.3125, "rewards/rejected": -4.40625, "step": 2380 }, { "epoch": 0.17982093145737718, "grad_norm": 13.415561954847117, "learning_rate": 4.903732566805727e-07, "logits/chosen": -2.0625, "logits/rejected": -2.015625, "logps/chosen": -456.0, "logps/rejected": -704.0, "loss": 0.3074, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.640625, "rewards/margins": 2.578125, "rewards/rejected": -5.21875, "step": 2390 }, { "epoch": 0.18057332029192688, "grad_norm": 11.176785126737313, "learning_rate": 4.901919658861228e-07, "logits/chosen": -2.3125, "logits/rejected": -2.25, "logps/chosen": -426.0, "logps/rejected": -612.0, "loss": 0.3084, "rewards/accuracies": 0.875, "rewards/chosen": -2.328125, "rewards/margins": 2.09375, "rewards/rejected": -4.4375, "step": 2400 }, { "epoch": 0.18132570912647655, "grad_norm": 12.417985416221901, "learning_rate": 4.900090180898292e-07, "logits/chosen": -2.1875, "logits/rejected": -2.09375, "logps/chosen": -440.0, "logps/rejected": -664.0, "loss": 0.2949, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.453125, "rewards/margins": 2.390625, "rewards/rejected": -4.84375, "step": 2410 }, { "epoch": 0.18207809796102625, "grad_norm": 8.189343431918507, "learning_rate": 4.898244145537857e-07, "logits/chosen": -2.234375, "logits/rejected": -2.09375, "logps/chosen": -422.0, "logps/rejected": -628.0, "loss": 0.3067, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.359375, "rewards/margins": 2.125, "rewards/rejected": -4.5, "step": 2420 }, { "epoch": 0.18283048679557595, "grad_norm": 8.786936206411413, "learning_rate": 4.896381565515087e-07, "logits/chosen": -2.125, "logits/rejected": -2.21875, "logps/chosen": -416.0, "logps/rejected": -600.0, "loss": 0.2736, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.25, "rewards/margins": 2.1875, "rewards/rejected": -4.4375, "step": 2430 }, { "epoch": 0.18358287563012565, "grad_norm": 11.627270504073307, "learning_rate": 4.894502453679284e-07, "logits/chosen": -2.15625, "logits/rejected": -2.09375, "logps/chosen": -404.0, "logps/rejected": -632.0, "loss": 0.3039, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.140625, "rewards/margins": 2.421875, "rewards/rejected": -4.5625, "step": 2440 }, { "epoch": 0.18433526446467535, "grad_norm": 10.007116906392051, "learning_rate": 4.892606822993793e-07, "logits/chosen": -2.203125, "logits/rejected": -2.15625, "logps/chosen": -394.0, "logps/rejected": -640.0, "loss": 0.3083, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.09375, "rewards/margins": 2.4375, "rewards/rejected": -4.53125, "step": 2450 }, { "epoch": 0.18508765329922505, "grad_norm": 15.663020670216987, "learning_rate": 4.890694686535918e-07, "logits/chosen": -2.328125, "logits/rejected": -2.34375, "logps/chosen": -470.0, "logps/rejected": -696.0, "loss": 0.2909, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.59375, "rewards/margins": 2.484375, "rewards/rejected": -5.09375, "step": 2460 }, { "epoch": 0.18584004213377472, "grad_norm": 12.775012399400252, "learning_rate": 4.888766057496833e-07, "logits/chosen": -2.15625, "logits/rejected": -2.046875, "logps/chosen": -456.0, "logps/rejected": -704.0, "loss": 0.3073, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -2.765625, "rewards/margins": 2.484375, "rewards/rejected": -5.25, "step": 2470 }, { "epoch": 0.18659243096832442, "grad_norm": 11.397271504950481, "learning_rate": 4.886820949181486e-07, "logits/chosen": -2.015625, "logits/rejected": -1.9765625, "logps/chosen": -394.0, "logps/rejected": -652.0, "loss": 0.2919, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.15625, "rewards/margins": 2.625, "rewards/rejected": -4.78125, "step": 2480 }, { "epoch": 0.18734481980287412, "grad_norm": 9.374141876259515, "learning_rate": 4.884859375008512e-07, "logits/chosen": -2.171875, "logits/rejected": -2.0, "logps/chosen": -422.0, "logps/rejected": -676.0, "loss": 0.335, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.1875, "rewards/margins": 2.59375, "rewards/rejected": -4.78125, "step": 2490 }, { "epoch": 0.18809720863742382, "grad_norm": 7.876725843631922, "learning_rate": 4.882881348510136e-07, "logits/chosen": -2.265625, "logits/rejected": -2.359375, "logps/chosen": -420.0, "logps/rejected": -692.0, "loss": 0.2789, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.25, "rewards/margins": 2.90625, "rewards/rejected": -5.15625, "step": 2500 }, { "epoch": 0.18884959747197352, "grad_norm": 9.641857799600695, "learning_rate": 4.880886883332083e-07, "logits/chosen": -2.234375, "logits/rejected": -2.078125, "logps/chosen": -426.0, "logps/rejected": -700.0, "loss": 0.2684, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -2.296875, "rewards/margins": 2.78125, "rewards/rejected": -5.0625, "step": 2510 }, { "epoch": 0.18960198630652322, "grad_norm": 11.09475561943636, "learning_rate": 4.878875993233486e-07, "logits/chosen": -2.15625, "logits/rejected": -2.125, "logps/chosen": -426.0, "logps/rejected": -660.0, "loss": 0.283, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -2.5, "rewards/margins": 2.3125, "rewards/rejected": -4.8125, "step": 2520 }, { "epoch": 0.1903543751410729, "grad_norm": 11.53193990848018, "learning_rate": 4.876848692086782e-07, "logits/chosen": -2.3125, "logits/rejected": -2.265625, "logps/chosen": -454.0, "logps/rejected": -708.0, "loss": 0.285, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.53125, "rewards/margins": 2.578125, "rewards/rejected": -5.125, "step": 2530 }, { "epoch": 0.1911067639756226, "grad_norm": 11.148511135082407, "learning_rate": 4.874804993877625e-07, "logits/chosen": -2.21875, "logits/rejected": -2.21875, "logps/chosen": -420.0, "logps/rejected": -628.0, "loss": 0.3012, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": -2.40625, "rewards/margins": 2.171875, "rewards/rejected": -4.5625, "step": 2540 }, { "epoch": 0.1918591528101723, "grad_norm": 8.438537062032745, "learning_rate": 4.872744912704788e-07, "logits/chosen": -2.203125, "logits/rejected": -2.15625, "logps/chosen": -398.0, "logps/rejected": -640.0, "loss": 0.2761, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.109375, "rewards/margins": 2.65625, "rewards/rejected": -4.75, "step": 2550 }, { "epoch": 0.192611541644722, "grad_norm": 11.392921531192046, "learning_rate": 4.870668462780062e-07, "logits/chosen": -2.296875, "logits/rejected": -2.171875, "logps/chosen": -436.0, "logps/rejected": -636.0, "loss": 0.2985, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.53125, "rewards/margins": 2.140625, "rewards/rejected": -4.65625, "step": 2560 }, { "epoch": 0.1933639304792717, "grad_norm": 13.756679764207105, "learning_rate": 4.868575658428163e-07, "logits/chosen": -2.265625, "logits/rejected": -2.234375, "logps/chosen": -418.0, "logps/rejected": -616.0, "loss": 0.3041, "rewards/accuracies": 0.875, "rewards/chosen": -2.359375, "rewards/margins": 2.1875, "rewards/rejected": -4.53125, "step": 2570 }, { "epoch": 0.1941163193138214, "grad_norm": 16.159686204914145, "learning_rate": 4.866466514086628e-07, "logits/chosen": -2.0625, "logits/rejected": -1.96875, "logps/chosen": -444.0, "logps/rejected": -692.0, "loss": 0.2807, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.375, "rewards/margins": 2.8125, "rewards/rejected": -5.1875, "step": 2580 }, { "epoch": 0.19486870814837107, "grad_norm": 11.562055008439973, "learning_rate": 4.864341044305719e-07, "logits/chosen": -2.25, "logits/rejected": -2.203125, "logps/chosen": -468.0, "logps/rejected": -640.0, "loss": 0.2892, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.625, "rewards/margins": 2.140625, "rewards/rejected": -4.78125, "step": 2590 }, { "epoch": 0.19562109698292077, "grad_norm": 12.701497373393279, "learning_rate": 4.862199263748323e-07, "logits/chosen": -2.21875, "logits/rejected": -2.25, "logps/chosen": -456.0, "logps/rejected": -664.0, "loss": 0.2873, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -2.6875, "rewards/margins": 2.328125, "rewards/rejected": -5.03125, "step": 2600 }, { "epoch": 0.19637348581747047, "grad_norm": 10.006201775427607, "learning_rate": 4.860041187189846e-07, "logits/chosen": -2.265625, "logits/rejected": -2.28125, "logps/chosen": -418.0, "logps/rejected": -664.0, "loss": 0.3033, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.359375, "rewards/margins": 2.53125, "rewards/rejected": -4.90625, "step": 2610 }, { "epoch": 0.19712587465202017, "grad_norm": 13.101551278926136, "learning_rate": 4.857866829518118e-07, "logits/chosen": -2.15625, "logits/rejected": -2.1875, "logps/chosen": -422.0, "logps/rejected": -696.0, "loss": 0.2801, "rewards/accuracies": 0.875, "rewards/chosen": -2.453125, "rewards/margins": 2.53125, "rewards/rejected": -5.0, "step": 2620 }, { "epoch": 0.19787826348656987, "grad_norm": 9.180399155015817, "learning_rate": 4.855676205733287e-07, "logits/chosen": -2.203125, "logits/rejected": -2.109375, "logps/chosen": -448.0, "logps/rejected": -664.0, "loss": 0.3175, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -2.546875, "rewards/margins": 2.25, "rewards/rejected": -4.78125, "step": 2630 }, { "epoch": 0.19863065232111957, "grad_norm": 14.43296995453312, "learning_rate": 4.853469330947712e-07, "logits/chosen": -2.3125, "logits/rejected": -2.390625, "logps/chosen": -462.0, "logps/rejected": -696.0, "loss": 0.2829, "rewards/accuracies": 0.875, "rewards/chosen": -2.546875, "rewards/margins": 2.640625, "rewards/rejected": -5.1875, "step": 2640 }, { "epoch": 0.19938304115566924, "grad_norm": 21.4831645187365, "learning_rate": 4.851246220385862e-07, "logits/chosen": -2.296875, "logits/rejected": -2.296875, "logps/chosen": -428.0, "logps/rejected": -636.0, "loss": 0.2975, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.34375, "rewards/margins": 2.296875, "rewards/rejected": -4.65625, "step": 2650 }, { "epoch": 0.20013542999021894, "grad_norm": 8.420398319955915, "learning_rate": 4.849006889384217e-07, "logits/chosen": -2.359375, "logits/rejected": -2.34375, "logps/chosen": -432.0, "logps/rejected": -604.0, "loss": 0.2968, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.28125, "rewards/margins": 2.015625, "rewards/rejected": -4.3125, "step": 2660 }, { "epoch": 0.20088781882476864, "grad_norm": 11.500993598468789, "learning_rate": 4.84675135339115e-07, "logits/chosen": -2.21875, "logits/rejected": -2.15625, "logps/chosen": -418.0, "logps/rejected": -660.0, "loss": 0.3046, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.328125, "rewards/margins": 2.546875, "rewards/rejected": -4.875, "step": 2670 }, { "epoch": 0.20164020765931834, "grad_norm": 10.83990111365577, "learning_rate": 4.84447962796683e-07, "logits/chosen": -2.125, "logits/rejected": -1.96875, "logps/chosen": -476.0, "logps/rejected": -700.0, "loss": 0.2929, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.703125, "rewards/margins": 2.453125, "rewards/rejected": -5.15625, "step": 2680 }, { "epoch": 0.20239259649386804, "grad_norm": 11.641647036167495, "learning_rate": 4.842191728783107e-07, "logits/chosen": -2.265625, "logits/rejected": -2.21875, "logps/chosen": -440.0, "logps/rejected": -632.0, "loss": 0.3104, "rewards/accuracies": 0.8125, "rewards/chosen": -2.5, "rewards/margins": 2.015625, "rewards/rejected": -4.53125, "step": 2690 }, { "epoch": 0.20314498532841774, "grad_norm": 12.500070915002162, "learning_rate": 4.839887671623414e-07, "logits/chosen": -2.296875, "logits/rejected": -2.171875, "logps/chosen": -412.0, "logps/rejected": -636.0, "loss": 0.2774, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.28125, "rewards/margins": 2.34375, "rewards/rejected": -4.625, "step": 2700 }, { "epoch": 0.2038973741629674, "grad_norm": 20.752808629895043, "learning_rate": 4.837567472382651e-07, "logits/chosen": -2.234375, "logits/rejected": -2.1875, "logps/chosen": -450.0, "logps/rejected": -692.0, "loss": 0.3063, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.71875, "rewards/margins": 2.4375, "rewards/rejected": -5.15625, "step": 2710 }, { "epoch": 0.2046497629975171, "grad_norm": 14.263196343647966, "learning_rate": 4.835231147067072e-07, "logits/chosen": -2.203125, "logits/rejected": -2.28125, "logps/chosen": -438.0, "logps/rejected": -676.0, "loss": 0.3013, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": -2.5625, "rewards/margins": 2.375, "rewards/rejected": -4.9375, "step": 2720 }, { "epoch": 0.2054021518320668, "grad_norm": 9.648562025851524, "learning_rate": 4.832878711794185e-07, "logits/chosen": -2.28125, "logits/rejected": -2.0625, "logps/chosen": -414.0, "logps/rejected": -708.0, "loss": 0.2811, "rewards/accuracies": 0.9375, "rewards/chosen": -2.265625, "rewards/margins": 2.953125, "rewards/rejected": -5.21875, "step": 2730 }, { "epoch": 0.2061545406666165, "grad_norm": 8.015454708270681, "learning_rate": 4.830510182792632e-07, "logits/chosen": -2.28125, "logits/rejected": -2.421875, "logps/chosen": -454.0, "logps/rejected": -720.0, "loss": 0.2612, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -2.6875, "rewards/margins": 2.765625, "rewards/rejected": -5.46875, "step": 2740 }, { "epoch": 0.2069069295011662, "grad_norm": 12.142727881552306, "learning_rate": 4.82812557640208e-07, "logits/chosen": -2.34375, "logits/rejected": -2.515625, "logps/chosen": -496.0, "logps/rejected": -672.0, "loss": 0.2953, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.8125, "rewards/margins": 2.359375, "rewards/rejected": -5.1875, "step": 2750 }, { "epoch": 0.2076593183357159, "grad_norm": 13.706013122202746, "learning_rate": 4.82572490907311e-07, "logits/chosen": -2.21875, "logits/rejected": -2.171875, "logps/chosen": -458.0, "logps/rejected": -660.0, "loss": 0.2929, "rewards/accuracies": 0.875, "rewards/chosen": -2.5625, "rewards/margins": 2.375, "rewards/rejected": -4.9375, "step": 2760 }, { "epoch": 0.20841170717026558, "grad_norm": 11.552871250845598, "learning_rate": 4.8233081973671e-07, "logits/chosen": -2.25, "logits/rejected": -2.046875, "logps/chosen": -472.0, "logps/rejected": -708.0, "loss": 0.2767, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.953125, "rewards/margins": 2.359375, "rewards/rejected": -5.3125, "step": 2770 }, { "epoch": 0.20916409600481528, "grad_norm": 8.579041512050576, "learning_rate": 4.820875457956115e-07, "logits/chosen": -2.328125, "logits/rejected": -2.421875, "logps/chosen": -450.0, "logps/rejected": -692.0, "loss": 0.2865, "rewards/accuracies": 0.875, "rewards/chosen": -2.640625, "rewards/margins": 2.515625, "rewards/rejected": -5.15625, "step": 2780 }, { "epoch": 0.20991648483936498, "grad_norm": 13.462953949626367, "learning_rate": 4.818426707622788e-07, "logits/chosen": -2.46875, "logits/rejected": -2.375, "logps/chosen": -482.0, "logps/rejected": -700.0, "loss": 0.2753, "rewards/accuracies": 0.875, "rewards/chosen": -2.859375, "rewards/margins": 2.375, "rewards/rejected": -5.25, "step": 2790 }, { "epoch": 0.21066887367391468, "grad_norm": 8.987669086174309, "learning_rate": 4.815961963260207e-07, "logits/chosen": -2.390625, "logits/rejected": -2.390625, "logps/chosen": -460.0, "logps/rejected": -676.0, "loss": 0.2733, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.671875, "rewards/margins": 2.375, "rewards/rejected": -5.0625, "step": 2800 }, { "epoch": 0.21142126250846438, "grad_norm": 11.493474460458286, "learning_rate": 4.813481241871794e-07, "logits/chosen": -2.234375, "logits/rejected": -2.21875, "logps/chosen": -448.0, "logps/rejected": -708.0, "loss": 0.2764, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.640625, "rewards/margins": 2.53125, "rewards/rejected": -5.15625, "step": 2810 }, { "epoch": 0.21217365134301408, "grad_norm": 11.998904961235786, "learning_rate": 4.810984560571195e-07, "logits/chosen": -2.3125, "logits/rejected": -2.21875, "logps/chosen": -428.0, "logps/rejected": -704.0, "loss": 0.2871, "rewards/accuracies": 0.84375, "rewards/chosen": -2.53125, "rewards/margins": 2.984375, "rewards/rejected": -5.53125, "step": 2820 }, { "epoch": 0.21292604017756375, "grad_norm": 13.365861076454605, "learning_rate": 4.808471936582156e-07, "logits/chosen": -2.328125, "logits/rejected": -2.4375, "logps/chosen": -456.0, "logps/rejected": -700.0, "loss": 0.2758, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.71875, "rewards/margins": 2.53125, "rewards/rejected": -5.25, "step": 2830 }, { "epoch": 0.21367842901211345, "grad_norm": 13.422858209585815, "learning_rate": 4.805943387238409e-07, "logits/chosen": -2.265625, "logits/rejected": -2.296875, "logps/chosen": -440.0, "logps/rejected": -668.0, "loss": 0.2723, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.515625, "rewards/margins": 2.5, "rewards/rejected": -5.0, "step": 2840 }, { "epoch": 0.21443081784666315, "grad_norm": 13.990760185540251, "learning_rate": 4.803398929983543e-07, "logits/chosen": -2.4375, "logits/rejected": -2.328125, "logps/chosen": -458.0, "logps/rejected": -712.0, "loss": 0.3043, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.78125, "rewards/margins": 2.625, "rewards/rejected": -5.40625, "step": 2850 }, { "epoch": 0.21518320668121285, "grad_norm": 11.473182251991883, "learning_rate": 4.800838582370898e-07, "logits/chosen": -2.375, "logits/rejected": -2.25, "logps/chosen": -460.0, "logps/rejected": -660.0, "loss": 0.3064, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -2.78125, "rewards/margins": 2.09375, "rewards/rejected": -4.875, "step": 2860 }, { "epoch": 0.21593559551576255, "grad_norm": 13.2757599483512, "learning_rate": 4.79826236206343e-07, "logits/chosen": -2.28125, "logits/rejected": -2.09375, "logps/chosen": -456.0, "logps/rejected": -684.0, "loss": 0.2779, "rewards/accuracies": 0.90625, "rewards/chosen": -2.671875, "rewards/margins": 2.375, "rewards/rejected": -5.03125, "step": 2870 }, { "epoch": 0.21668798435031225, "grad_norm": 15.796447374015552, "learning_rate": 4.795670286833599e-07, "logits/chosen": -2.3125, "logits/rejected": -2.28125, "logps/chosen": -464.0, "logps/rejected": -704.0, "loss": 0.2746, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -2.609375, "rewards/margins": 2.5625, "rewards/rejected": -5.15625, "step": 2880 }, { "epoch": 0.21744037318486192, "grad_norm": 11.483080155833736, "learning_rate": 4.79306237456324e-07, "logits/chosen": -2.21875, "logits/rejected": -2.265625, "logps/chosen": -440.0, "logps/rejected": -668.0, "loss": 0.2755, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.5625, "rewards/margins": 2.296875, "rewards/rejected": -4.875, "step": 2890 }, { "epoch": 0.21819276201941162, "grad_norm": 13.041849145948497, "learning_rate": 4.790438643243447e-07, "logits/chosen": -2.328125, "logits/rejected": -2.40625, "logps/chosen": -452.0, "logps/rejected": -680.0, "loss": 0.293, "rewards/accuracies": 0.9375, "rewards/chosen": -2.421875, "rewards/margins": 2.578125, "rewards/rejected": -5.0, "step": 2900 }, { "epoch": 0.21894515085396132, "grad_norm": 11.178900042891765, "learning_rate": 4.787799110974436e-07, "logits/chosen": -2.28125, "logits/rejected": -2.25, "logps/chosen": -454.0, "logps/rejected": -676.0, "loss": 0.2731, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.703125, "rewards/margins": 2.421875, "rewards/rejected": -5.125, "step": 2910 }, { "epoch": 0.21969753968851102, "grad_norm": 12.460911893104443, "learning_rate": 4.785143795965437e-07, "logits/chosen": -2.390625, "logits/rejected": -2.390625, "logps/chosen": -464.0, "logps/rejected": -688.0, "loss": 0.2731, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.75, "rewards/margins": 2.3125, "rewards/rejected": -5.0625, "step": 2920 }, { "epoch": 0.22044992852306072, "grad_norm": 10.836551206091377, "learning_rate": 4.782472716534554e-07, "logits/chosen": -2.28125, "logits/rejected": -2.171875, "logps/chosen": -504.0, "logps/rejected": -764.0, "loss": 0.2631, "rewards/accuracies": 0.90625, "rewards/chosen": -2.984375, "rewards/margins": 2.671875, "rewards/rejected": -5.65625, "step": 2930 }, { "epoch": 0.22120231735761042, "grad_norm": 15.348018846892368, "learning_rate": 4.779785891108647e-07, "logits/chosen": -2.375, "logits/rejected": -2.21875, "logps/chosen": -468.0, "logps/rejected": -660.0, "loss": 0.2947, "rewards/accuracies": 0.875, "rewards/chosen": -2.71875, "rewards/margins": 2.125, "rewards/rejected": -4.84375, "step": 2940 }, { "epoch": 0.2219547061921601, "grad_norm": 10.060239033279466, "learning_rate": 4.777083338223202e-07, "logits/chosen": -2.3125, "logits/rejected": -2.3125, "logps/chosen": -430.0, "logps/rejected": -652.0, "loss": 0.2713, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -2.546875, "rewards/margins": 2.265625, "rewards/rejected": -4.8125, "step": 2950 }, { "epoch": 0.2227070950267098, "grad_norm": 11.578423536903305, "learning_rate": 4.774365076522202e-07, "logits/chosen": -2.46875, "logits/rejected": -2.421875, "logps/chosen": -500.0, "logps/rejected": -696.0, "loss": 0.2899, "rewards/accuracies": 0.875, "rewards/chosen": -2.84375, "rewards/margins": 2.375, "rewards/rejected": -5.25, "step": 2960 }, { "epoch": 0.2234594838612595, "grad_norm": 9.493796241422485, "learning_rate": 4.771631124758e-07, "logits/chosen": -2.375, "logits/rejected": -2.1875, "logps/chosen": -436.0, "logps/rejected": -692.0, "loss": 0.2694, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.609375, "rewards/margins": 2.53125, "rewards/rejected": -5.125, "step": 2970 }, { "epoch": 0.2242118726958092, "grad_norm": 10.641881748395264, "learning_rate": 4.76888150179119e-07, "logits/chosen": -2.375, "logits/rejected": -2.375, "logps/chosen": -452.0, "logps/rejected": -744.0, "loss": 0.2698, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.609375, "rewards/margins": 3.046875, "rewards/rejected": -5.65625, "step": 2980 }, { "epoch": 0.2249642615303589, "grad_norm": 16.153028668045003, "learning_rate": 4.7661162265904773e-07, "logits/chosen": -2.453125, "logits/rejected": -2.46875, "logps/chosen": -468.0, "logps/rejected": -744.0, "loss": 0.2797, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.6875, "rewards/margins": 2.984375, "rewards/rejected": -5.65625, "step": 2990 }, { "epoch": 0.2257166503649086, "grad_norm": 10.983619498175136, "learning_rate": 4.7633353182325443e-07, "logits/chosen": -2.3125, "logits/rejected": -2.25, "logps/chosen": -440.0, "logps/rejected": -740.0, "loss": 0.2553, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.640625, "rewards/margins": 2.859375, "rewards/rejected": -5.5, "step": 3000 }, { "epoch": 0.22646903919945827, "grad_norm": 15.749882633269605, "learning_rate": 4.760538795901923e-07, "logits/chosen": -2.265625, "logits/rejected": -2.28125, "logps/chosen": -480.0, "logps/rejected": -712.0, "loss": 0.2895, "rewards/accuracies": 0.824999988079071, "rewards/chosen": -2.90625, "rewards/margins": 2.484375, "rewards/rejected": -5.40625, "step": 3010 }, { "epoch": 0.22722142803400797, "grad_norm": 11.021582532956606, "learning_rate": 4.757726678890859e-07, "logits/chosen": -2.359375, "logits/rejected": -2.375, "logps/chosen": -466.0, "logps/rejected": -704.0, "loss": 0.2792, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.71875, "rewards/margins": 2.65625, "rewards/rejected": -5.375, "step": 3020 }, { "epoch": 0.22797381686855767, "grad_norm": 9.324191443362047, "learning_rate": 4.754898986599182e-07, "logits/chosen": -2.25, "logits/rejected": -2.28125, "logps/chosen": -486.0, "logps/rejected": -692.0, "loss": 0.2915, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.015625, "rewards/margins": 2.21875, "rewards/rejected": -5.25, "step": 3030 }, { "epoch": 0.22872620570310737, "grad_norm": 11.823152534766754, "learning_rate": 4.75205573853417e-07, "logits/chosen": -2.265625, "logits/rejected": -2.09375, "logps/chosen": -452.0, "logps/rejected": -656.0, "loss": 0.291, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.65625, "rewards/margins": 2.296875, "rewards/rejected": -4.9375, "step": 3040 }, { "epoch": 0.22947859453765707, "grad_norm": 10.103536270787842, "learning_rate": 4.749196954310414e-07, "logits/chosen": -2.328125, "logits/rejected": -2.34375, "logps/chosen": -458.0, "logps/rejected": -700.0, "loss": 0.2633, "rewards/accuracies": 0.90625, "rewards/chosen": -2.671875, "rewards/margins": 2.5625, "rewards/rejected": -5.21875, "step": 3050 }, { "epoch": 0.23023098337220677, "grad_norm": 11.573356925341455, "learning_rate": 4.746322653649683e-07, "logits/chosen": -2.546875, "logits/rejected": -2.453125, "logps/chosen": -508.0, "logps/rejected": -776.0, "loss": 0.286, "rewards/accuracies": 0.84375, "rewards/chosen": -3.25, "rewards/margins": 2.71875, "rewards/rejected": -5.96875, "step": 3060 }, { "epoch": 0.23098337220675647, "grad_norm": 9.629528461377197, "learning_rate": 4.7434328563807913e-07, "logits/chosen": -2.40625, "logits/rejected": -2.421875, "logps/chosen": -432.0, "logps/rejected": -732.0, "loss": 0.2762, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.515625, "rewards/margins": 2.859375, "rewards/rejected": -5.375, "step": 3070 }, { "epoch": 0.23173576104130614, "grad_norm": 12.328781666243955, "learning_rate": 4.7405275824394574e-07, "logits/chosen": -2.375, "logits/rejected": -2.40625, "logps/chosen": -456.0, "logps/rejected": -648.0, "loss": 0.2746, "rewards/accuracies": 0.824999988079071, "rewards/chosen": -2.625, "rewards/margins": 2.140625, "rewards/rejected": -4.75, "step": 3080 }, { "epoch": 0.23248814987585584, "grad_norm": 15.210003142651068, "learning_rate": 4.737606851868167e-07, "logits/chosen": -2.484375, "logits/rejected": -2.5625, "logps/chosen": -510.0, "logps/rejected": -752.0, "loss": 0.2733, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -3.03125, "rewards/margins": 2.890625, "rewards/rejected": -5.90625, "step": 3090 }, { "epoch": 0.23324053871040554, "grad_norm": 8.755069030351935, "learning_rate": 4.734670684816037e-07, "logits/chosen": -2.34375, "logits/rejected": -2.296875, "logps/chosen": -488.0, "logps/rejected": -760.0, "loss": 0.2546, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.703125, "rewards/margins": 2.84375, "rewards/rejected": -5.5625, "step": 3100 }, { "epoch": 0.23399292754495524, "grad_norm": 10.283129702374858, "learning_rate": 4.7317191015386744e-07, "logits/chosen": -2.546875, "logits/rejected": -2.59375, "logps/chosen": -432.0, "logps/rejected": -684.0, "loss": 0.2647, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.640625, "rewards/margins": 2.5625, "rewards/rejected": -5.1875, "step": 3110 }, { "epoch": 0.23474531637950494, "grad_norm": 11.564474160364048, "learning_rate": 4.7287521223980395e-07, "logits/chosen": -2.265625, "logits/rejected": -2.40625, "logps/chosen": -456.0, "logps/rejected": -704.0, "loss": 0.2815, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.515625, "rewards/margins": 2.703125, "rewards/rejected": -5.21875, "step": 3120 }, { "epoch": 0.23549770521405464, "grad_norm": 8.416698175306761, "learning_rate": 4.725769767862301e-07, "logits/chosen": -2.421875, "logits/rejected": -2.46875, "logps/chosen": -450.0, "logps/rejected": -660.0, "loss": 0.2563, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.625, "rewards/margins": 2.28125, "rewards/rejected": -4.90625, "step": 3130 }, { "epoch": 0.2362500940486043, "grad_norm": 10.06567697369509, "learning_rate": 4.7227720585056986e-07, "logits/chosen": -2.578125, "logits/rejected": -2.609375, "logps/chosen": -490.0, "logps/rejected": -752.0, "loss": 0.264, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.90625, "rewards/margins": 3.0625, "rewards/rejected": -5.96875, "step": 3140 }, { "epoch": 0.237002482883154, "grad_norm": 9.29105026928847, "learning_rate": 4.7197590150083986e-07, "logits/chosen": -2.546875, "logits/rejected": -2.53125, "logps/chosen": -484.0, "logps/rejected": -780.0, "loss": 0.2677, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.84375, "rewards/margins": 3.078125, "rewards/rejected": -5.90625, "step": 3150 }, { "epoch": 0.2377548717177037, "grad_norm": 13.318380945168878, "learning_rate": 4.716730658156354e-07, "logits/chosen": -2.484375, "logits/rejected": -2.53125, "logps/chosen": -464.0, "logps/rejected": -744.0, "loss": 0.2685, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.78125, "rewards/margins": 2.90625, "rewards/rejected": -5.6875, "step": 3160 }, { "epoch": 0.2385072605522534, "grad_norm": 9.870130688358575, "learning_rate": 4.7136870088411564e-07, "logits/chosen": -2.515625, "logits/rejected": -2.59375, "logps/chosen": -468.0, "logps/rejected": -724.0, "loss": 0.2655, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.78125, "rewards/margins": 2.796875, "rewards/rejected": -5.59375, "step": 3170 }, { "epoch": 0.2392596493868031, "grad_norm": 10.982883590086931, "learning_rate": 4.710628088059898e-07, "logits/chosen": -2.546875, "logits/rejected": -2.484375, "logps/chosen": -452.0, "logps/rejected": -708.0, "loss": 0.2604, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.65625, "rewards/margins": 2.546875, "rewards/rejected": -5.1875, "step": 3180 }, { "epoch": 0.2400120382213528, "grad_norm": 11.579071033337119, "learning_rate": 4.707553916915022e-07, "logits/chosen": -2.53125, "logits/rejected": -2.609375, "logps/chosen": -470.0, "logps/rejected": -700.0, "loss": 0.2889, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.953125, "rewards/margins": 2.453125, "rewards/rejected": -5.40625, "step": 3190 }, { "epoch": 0.24076442705590248, "grad_norm": 10.91128867087661, "learning_rate": 4.704464516614178e-07, "logits/chosen": -2.5, "logits/rejected": -2.34375, "logps/chosen": -462.0, "logps/rejected": -716.0, "loss": 0.2702, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.765625, "rewards/margins": 2.578125, "rewards/rejected": -5.34375, "step": 3200 }, { "epoch": 0.24151681589045218, "grad_norm": 14.455706838813876, "learning_rate": 4.7013599084700787e-07, "logits/chosen": -2.421875, "logits/rejected": -2.359375, "logps/chosen": -486.0, "logps/rejected": -704.0, "loss": 0.293, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.96875, "rewards/margins": 2.40625, "rewards/rejected": -5.375, "step": 3210 }, { "epoch": 0.24226920472500188, "grad_norm": 12.606738237341512, "learning_rate": 4.698240113900348e-07, "logits/chosen": -2.421875, "logits/rejected": -2.34375, "logps/chosen": -428.0, "logps/rejected": -712.0, "loss": 0.2755, "rewards/accuracies": 0.90625, "rewards/chosen": -2.546875, "rewards/margins": 2.828125, "rewards/rejected": -5.375, "step": 3220 }, { "epoch": 0.24302159355955158, "grad_norm": 11.859056176810007, "learning_rate": 4.6951051544273765e-07, "logits/chosen": -2.34375, "logits/rejected": -2.1875, "logps/chosen": -442.0, "logps/rejected": -724.0, "loss": 0.2585, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.46875, "rewards/margins": 2.796875, "rewards/rejected": -5.28125, "step": 3230 }, { "epoch": 0.24377398239410128, "grad_norm": 10.814100109298046, "learning_rate": 4.6919550516781723e-07, "logits/chosen": -2.53125, "logits/rejected": -2.46875, "logps/chosen": -444.0, "logps/rejected": -700.0, "loss": 0.2683, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.609375, "rewards/margins": 2.640625, "rewards/rejected": -5.25, "step": 3240 }, { "epoch": 0.24452637122865098, "grad_norm": 12.098951611806221, "learning_rate": 4.688789827384213e-07, "logits/chosen": -2.390625, "logits/rejected": -2.46875, "logps/chosen": -482.0, "logps/rejected": -688.0, "loss": 0.2909, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.84375, "rewards/margins": 2.25, "rewards/rejected": -5.09375, "step": 3250 }, { "epoch": 0.24527876006320065, "grad_norm": 12.363507547937484, "learning_rate": 4.6856095033812927e-07, "logits/chosen": -2.34375, "logits/rejected": -2.296875, "logps/chosen": -424.0, "logps/rejected": -656.0, "loss": 0.2608, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.34375, "rewards/margins": 2.4375, "rewards/rejected": -4.78125, "step": 3260 }, { "epoch": 0.24603114889775035, "grad_norm": 11.153976884017242, "learning_rate": 4.682414101609374e-07, "logits/chosen": -2.3125, "logits/rejected": -2.390625, "logps/chosen": -452.0, "logps/rejected": -700.0, "loss": 0.273, "rewards/accuracies": 0.875, "rewards/chosen": -2.6875, "rewards/margins": 2.5, "rewards/rejected": -5.1875, "step": 3270 }, { "epoch": 0.24678353773230005, "grad_norm": 11.019325656369531, "learning_rate": 4.679203644112436e-07, "logits/chosen": -2.484375, "logits/rejected": -2.5, "logps/chosen": -466.0, "logps/rejected": -736.0, "loss": 0.2597, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.78125, "rewards/margins": 2.71875, "rewards/rejected": -5.5, "step": 3280 }, { "epoch": 0.24753592656684975, "grad_norm": 9.822991361576156, "learning_rate": 4.6759781530383214e-07, "logits/chosen": -2.328125, "logits/rejected": -2.5, "logps/chosen": -464.0, "logps/rejected": -720.0, "loss": 0.2826, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.859375, "rewards/margins": 2.703125, "rewards/rejected": -5.5625, "step": 3290 }, { "epoch": 0.24828831540139945, "grad_norm": 10.856602457553329, "learning_rate": 4.672737650638586e-07, "logits/chosen": -2.5, "logits/rejected": -2.46875, "logps/chosen": -476.0, "logps/rejected": -684.0, "loss": 0.277, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.828125, "rewards/margins": 2.375, "rewards/rejected": -5.21875, "step": 3300 }, { "epoch": 0.24904070423594915, "grad_norm": 12.070368316452733, "learning_rate": 4.669482159268341e-07, "logits/chosen": -2.421875, "logits/rejected": -2.4375, "logps/chosen": -432.0, "logps/rejected": -696.0, "loss": 0.2702, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.546875, "rewards/margins": 2.75, "rewards/rejected": -5.28125, "step": 3310 }, { "epoch": 0.24979309307049882, "grad_norm": 10.884960189063547, "learning_rate": 4.666211701386103e-07, "logits/chosen": -2.40625, "logits/rejected": -2.46875, "logps/chosen": -456.0, "logps/rejected": -688.0, "loss": 0.2681, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.71875, "rewards/margins": 2.609375, "rewards/rejected": -5.34375, "step": 3320 }, { "epoch": 0.25054548190504855, "grad_norm": 9.425055782792782, "learning_rate": 4.662926299553638e-07, "logits/chosen": -2.546875, "logits/rejected": -2.53125, "logps/chosen": -462.0, "logps/rejected": -716.0, "loss": 0.2536, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.6875, "rewards/margins": 2.75, "rewards/rejected": -5.4375, "step": 3330 }, { "epoch": 0.25129787073959825, "grad_norm": 11.577079386440818, "learning_rate": 4.6596259764358037e-07, "logits/chosen": -2.359375, "logits/rejected": -2.328125, "logps/chosen": -460.0, "logps/rejected": -776.0, "loss": 0.2487, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.71875, "rewards/margins": 3.15625, "rewards/rejected": -5.875, "step": 3340 }, { "epoch": 0.2520502595741479, "grad_norm": 12.032227039622406, "learning_rate": 4.6563107548003967e-07, "logits/chosen": -2.484375, "logits/rejected": -2.515625, "logps/chosen": -444.0, "logps/rejected": -720.0, "loss": 0.2795, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.65625, "rewards/margins": 2.84375, "rewards/rejected": -5.5, "step": 3350 }, { "epoch": 0.2528026484086976, "grad_norm": 10.938923875572582, "learning_rate": 4.6529806575179895e-07, "logits/chosen": -2.453125, "logits/rejected": -2.390625, "logps/chosen": -424.0, "logps/rejected": -644.0, "loss": 0.2797, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.5, "rewards/margins": 2.3125, "rewards/rejected": -4.8125, "step": 3360 }, { "epoch": 0.2535550372432473, "grad_norm": 12.712846874830957, "learning_rate": 4.6496357075617816e-07, "logits/chosen": -2.375, "logits/rejected": -2.34375, "logps/chosen": -464.0, "logps/rejected": -744.0, "loss": 0.2543, "rewards/accuracies": 0.90625, "rewards/chosen": -2.5625, "rewards/margins": 3.0, "rewards/rejected": -5.5625, "step": 3370 }, { "epoch": 0.254307426077797, "grad_norm": 9.357206348086631, "learning_rate": 4.646275928007431e-07, "logits/chosen": -2.578125, "logits/rejected": -2.53125, "logps/chosen": -480.0, "logps/rejected": -724.0, "loss": 0.2536, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.953125, "rewards/margins": 2.5625, "rewards/rejected": -5.53125, "step": 3380 }, { "epoch": 0.2550598149123467, "grad_norm": 11.776081446315107, "learning_rate": 4.642901342032905e-07, "logits/chosen": -2.46875, "logits/rejected": -2.5, "logps/chosen": -476.0, "logps/rejected": -744.0, "loss": 0.2794, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.921875, "rewards/margins": 2.84375, "rewards/rejected": -5.75, "step": 3390 }, { "epoch": 0.2558122037468964, "grad_norm": 13.682212636715727, "learning_rate": 4.639511972918311e-07, "logits/chosen": -2.5625, "logits/rejected": -2.59375, "logps/chosen": -468.0, "logps/rejected": -708.0, "loss": 0.2655, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.9375, "rewards/margins": 2.484375, "rewards/rejected": -5.40625, "step": 3400 }, { "epoch": 0.2565645925814461, "grad_norm": 13.142888486637387, "learning_rate": 4.636107844045742e-07, "logits/chosen": -2.34375, "logits/rejected": -2.359375, "logps/chosen": -444.0, "logps/rejected": -728.0, "loss": 0.2618, "rewards/accuracies": 0.90625, "rewards/chosen": -2.59375, "rewards/margins": 2.8125, "rewards/rejected": -5.40625, "step": 3410 }, { "epoch": 0.2573169814159958, "grad_norm": 13.866071513298566, "learning_rate": 4.632688978899114e-07, "logits/chosen": -2.484375, "logits/rejected": -2.3125, "logps/chosen": -458.0, "logps/rejected": -732.0, "loss": 0.2595, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.6875, "rewards/margins": 2.703125, "rewards/rejected": -5.40625, "step": 3420 }, { "epoch": 0.2580693702505455, "grad_norm": 12.268000874109546, "learning_rate": 4.629255401064004e-07, "logits/chosen": -2.453125, "logits/rejected": -2.484375, "logps/chosen": -502.0, "logps/rejected": -736.0, "loss": 0.257, "rewards/accuracies": 0.875, "rewards/chosen": -3.140625, "rewards/margins": 2.5, "rewards/rejected": -5.625, "step": 3430 }, { "epoch": 0.2588217590850952, "grad_norm": 10.59004333565291, "learning_rate": 4.625807134227483e-07, "logits/chosen": -2.40625, "logits/rejected": -2.546875, "logps/chosen": -460.0, "logps/rejected": -700.0, "loss": 0.2747, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.734375, "rewards/margins": 2.5625, "rewards/rejected": -5.3125, "step": 3440 }, { "epoch": 0.2595741479196449, "grad_norm": 8.545763333464617, "learning_rate": 4.622344202177961e-07, "logits/chosen": -2.390625, "logits/rejected": -2.25, "logps/chosen": -456.0, "logps/rejected": -724.0, "loss": 0.2531, "rewards/accuracies": 0.875, "rewards/chosen": -2.625, "rewards/margins": 2.640625, "rewards/rejected": -5.28125, "step": 3450 }, { "epoch": 0.2603265367541946, "grad_norm": 17.47211772247205, "learning_rate": 4.6188666288050163e-07, "logits/chosen": -2.53125, "logits/rejected": -2.578125, "logps/chosen": -472.0, "logps/rejected": -752.0, "loss": 0.2801, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.796875, "rewards/margins": 3.0, "rewards/rejected": -5.8125, "step": 3460 }, { "epoch": 0.26107892558874424, "grad_norm": 8.554992710182484, "learning_rate": 4.615374438099232e-07, "logits/chosen": -2.453125, "logits/rejected": -2.546875, "logps/chosen": -476.0, "logps/rejected": -688.0, "loss": 0.2786, "rewards/accuracies": 0.84375, "rewards/chosen": -2.890625, "rewards/margins": 2.234375, "rewards/rejected": -5.125, "step": 3470 }, { "epoch": 0.26183131442329394, "grad_norm": 13.184478264125092, "learning_rate": 4.611867654152033e-07, "logits/chosen": -2.390625, "logits/rejected": -2.453125, "logps/chosen": -476.0, "logps/rejected": -724.0, "loss": 0.2613, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.9375, "rewards/margins": 2.5, "rewards/rejected": -5.4375, "step": 3480 }, { "epoch": 0.26258370325784364, "grad_norm": 14.19073981876287, "learning_rate": 4.608346301155516e-07, "logits/chosen": -2.4375, "logits/rejected": -2.328125, "logps/chosen": -486.0, "logps/rejected": -764.0, "loss": 0.2639, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.84375, "rewards/margins": 2.921875, "rewards/rejected": -5.75, "step": 3490 }, { "epoch": 0.26333609209239334, "grad_norm": 9.183184296599434, "learning_rate": 4.604810403402285e-07, "logits/chosen": -2.546875, "logits/rejected": -2.484375, "logps/chosen": -466.0, "logps/rejected": -704.0, "loss": 0.2878, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -2.90625, "rewards/margins": 2.578125, "rewards/rejected": -5.46875, "step": 3500 }, { "epoch": 0.26408848092694304, "grad_norm": 11.772301309520701, "learning_rate": 4.601259985285284e-07, "logits/chosen": -2.453125, "logits/rejected": -2.34375, "logps/chosen": -492.0, "logps/rejected": -736.0, "loss": 0.2511, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -2.953125, "rewards/margins": 2.671875, "rewards/rejected": -5.625, "step": 3510 }, { "epoch": 0.26484086976149274, "grad_norm": 9.531109177760952, "learning_rate": 4.5976950712976286e-07, "logits/chosen": -2.390625, "logits/rejected": -2.484375, "logps/chosen": -472.0, "logps/rejected": -740.0, "loss": 0.2676, "rewards/accuracies": 0.875, "rewards/chosen": -2.890625, "rewards/margins": 2.75, "rewards/rejected": -5.625, "step": 3520 }, { "epoch": 0.26559325859604244, "grad_norm": 10.45054679863606, "learning_rate": 4.5941156860324345e-07, "logits/chosen": -2.3125, "logits/rejected": -2.1875, "logps/chosen": -492.0, "logps/rejected": -780.0, "loss": 0.2747, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.875, "rewards/margins": 3.171875, "rewards/rejected": -6.0625, "step": 3530 }, { "epoch": 0.26634564743059214, "grad_norm": 10.597729463137316, "learning_rate": 4.590521854182651e-07, "logits/chosen": -2.328125, "logits/rejected": -2.484375, "logps/chosen": -492.0, "logps/rejected": -760.0, "loss": 0.2525, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.78125, "rewards/margins": 2.953125, "rewards/rejected": -5.71875, "step": 3540 }, { "epoch": 0.26709803626514184, "grad_norm": 11.220941405018289, "learning_rate": 4.5869136005408893e-07, "logits/chosen": -2.46875, "logits/rejected": -2.546875, "logps/chosen": -506.0, "logps/rejected": -780.0, "loss": 0.2787, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.796875, "rewards/margins": 3.203125, "rewards/rejected": -6.0, "step": 3550 }, { "epoch": 0.26785042509969154, "grad_norm": 10.499251191182202, "learning_rate": 4.5832909499992514e-07, "logits/chosen": -2.484375, "logits/rejected": -2.421875, "logps/chosen": -456.0, "logps/rejected": -708.0, "loss": 0.2792, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.734375, "rewards/margins": 2.65625, "rewards/rejected": -5.375, "step": 3560 }, { "epoch": 0.26860281393424124, "grad_norm": 10.676669184766462, "learning_rate": 4.579653927549159e-07, "logits/chosen": -2.40625, "logits/rejected": -2.46875, "logps/chosen": -480.0, "logps/rejected": -752.0, "loss": 0.2609, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.90625, "rewards/margins": 2.953125, "rewards/rejected": -5.875, "step": 3570 }, { "epoch": 0.26935520276879094, "grad_norm": 12.367898130153655, "learning_rate": 4.57600255828118e-07, "logits/chosen": -2.484375, "logits/rejected": -2.34375, "logps/chosen": -480.0, "logps/rejected": -808.0, "loss": 0.2866, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.953125, "rewards/margins": 3.234375, "rewards/rejected": -6.1875, "step": 3580 }, { "epoch": 0.2701075916033406, "grad_norm": 10.950643760834097, "learning_rate": 4.572336867384856e-07, "logits/chosen": -2.3125, "logits/rejected": -2.296875, "logps/chosen": -500.0, "logps/rejected": -732.0, "loss": 0.2648, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -3.0, "rewards/margins": 2.609375, "rewards/rejected": -5.59375, "step": 3590 }, { "epoch": 0.2708599804378903, "grad_norm": 11.102842359026281, "learning_rate": 4.5686568801485306e-07, "logits/chosen": -2.484375, "logits/rejected": -2.453125, "logps/chosen": -458.0, "logps/rejected": -696.0, "loss": 0.2382, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.765625, "rewards/margins": 2.5625, "rewards/rejected": -5.34375, "step": 3600 }, { "epoch": 0.27161236927244, "grad_norm": 19.242216390554898, "learning_rate": 4.5649626219591685e-07, "logits/chosen": -2.453125, "logits/rejected": -2.265625, "logps/chosen": -480.0, "logps/rejected": -768.0, "loss": 0.2854, "rewards/accuracies": 0.90625, "rewards/chosen": -2.96875, "rewards/margins": 2.953125, "rewards/rejected": -5.90625, "step": 3610 }, { "epoch": 0.2723647581069897, "grad_norm": 12.964030095763036, "learning_rate": 4.5612541183021905e-07, "logits/chosen": -2.453125, "logits/rejected": -2.25, "logps/chosen": -466.0, "logps/rejected": -760.0, "loss": 0.2566, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.875, "rewards/margins": 3.03125, "rewards/rejected": -5.90625, "step": 3620 }, { "epoch": 0.2731171469415394, "grad_norm": 12.301798911047383, "learning_rate": 4.5575313947612875e-07, "logits/chosen": -2.453125, "logits/rejected": -2.515625, "logps/chosen": -486.0, "logps/rejected": -788.0, "loss": 0.2723, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.0625, "rewards/margins": 3.0625, "rewards/rejected": -6.125, "step": 3630 }, { "epoch": 0.2738695357760891, "grad_norm": 12.468836714781107, "learning_rate": 4.553794477018251e-07, "logits/chosen": -2.4375, "logits/rejected": -2.3125, "logps/chosen": -478.0, "logps/rejected": -708.0, "loss": 0.2721, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.828125, "rewards/margins": 2.5, "rewards/rejected": -5.3125, "step": 3640 }, { "epoch": 0.2746219246106388, "grad_norm": 10.505853223732606, "learning_rate": 4.550043390852791e-07, "logits/chosen": -2.421875, "logits/rejected": -2.359375, "logps/chosen": -456.0, "logps/rejected": -740.0, "loss": 0.2472, "rewards/accuracies": 0.9375, "rewards/chosen": -2.734375, "rewards/margins": 2.84375, "rewards/rejected": -5.59375, "step": 3650 }, { "epoch": 0.2753743134451885, "grad_norm": 12.356299213718698, "learning_rate": 4.546278162142364e-07, "logits/chosen": -2.59375, "logits/rejected": -2.71875, "logps/chosen": -478.0, "logps/rejected": -740.0, "loss": 0.2522, "rewards/accuracies": 0.90625, "rewards/chosen": -2.96875, "rewards/margins": 2.875, "rewards/rejected": -5.84375, "step": 3660 }, { "epoch": 0.2761267022797382, "grad_norm": 11.183385248605012, "learning_rate": 4.542498816861988e-07, "logits/chosen": -2.5625, "logits/rejected": -2.453125, "logps/chosen": -472.0, "logps/rejected": -732.0, "loss": 0.2481, "rewards/accuracies": 0.875, "rewards/chosen": -2.875, "rewards/margins": 2.796875, "rewards/rejected": -5.6875, "step": 3670 }, { "epoch": 0.2768790911142879, "grad_norm": 11.161802779990282, "learning_rate": 4.538705381084067e-07, "logits/chosen": -2.40625, "logits/rejected": -2.515625, "logps/chosen": -472.0, "logps/rejected": -696.0, "loss": 0.2778, "rewards/accuracies": 0.875, "rewards/chosen": -2.9375, "rewards/margins": 2.46875, "rewards/rejected": -5.40625, "step": 3680 }, { "epoch": 0.2776314799488376, "grad_norm": 14.823136481766522, "learning_rate": 4.5348978809782123e-07, "logits/chosen": -2.5, "logits/rejected": -2.3125, "logps/chosen": -466.0, "logps/rejected": -696.0, "loss": 0.274, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.734375, "rewards/margins": 2.5, "rewards/rejected": -5.25, "step": 3690 }, { "epoch": 0.2783838687833873, "grad_norm": 11.219920882878569, "learning_rate": 4.531076342811059e-07, "logits/chosen": -2.5, "logits/rejected": -2.359375, "logps/chosen": -480.0, "logps/rejected": -760.0, "loss": 0.2258, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.796875, "rewards/margins": 2.953125, "rewards/rejected": -5.75, "step": 3700 }, { "epoch": 0.2791362576179369, "grad_norm": 14.53718252912113, "learning_rate": 4.5272407929460846e-07, "logits/chosen": -2.546875, "logits/rejected": -2.34375, "logps/chosen": -502.0, "logps/rejected": -744.0, "loss": 0.2686, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.0625, "rewards/margins": 2.78125, "rewards/rejected": -5.84375, "step": 3710 }, { "epoch": 0.2798886464524866, "grad_norm": 9.547790627909988, "learning_rate": 4.5233912578434297e-07, "logits/chosen": -2.453125, "logits/rejected": -2.453125, "logps/chosen": -520.0, "logps/rejected": -800.0, "loss": 0.2584, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -3.265625, "rewards/margins": 2.71875, "rewards/rejected": -5.96875, "step": 3720 }, { "epoch": 0.2806410352870363, "grad_norm": 11.288375368877425, "learning_rate": 4.5195277640597165e-07, "logits/chosen": -2.515625, "logits/rejected": -2.46875, "logps/chosen": -492.0, "logps/rejected": -768.0, "loss": 0.2678, "rewards/accuracies": 0.84375, "rewards/chosen": -3.015625, "rewards/margins": 2.78125, "rewards/rejected": -5.8125, "step": 3730 }, { "epoch": 0.281393424121586, "grad_norm": 12.551442620089833, "learning_rate": 4.5156503382478583e-07, "logits/chosen": -2.453125, "logits/rejected": -2.296875, "logps/chosen": -496.0, "logps/rejected": -736.0, "loss": 0.2645, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.953125, "rewards/margins": 2.734375, "rewards/rejected": -5.6875, "step": 3740 }, { "epoch": 0.2821458129561357, "grad_norm": 11.78941233805565, "learning_rate": 4.511759007156886e-07, "logits/chosen": -2.546875, "logits/rejected": -2.546875, "logps/chosen": -478.0, "logps/rejected": -720.0, "loss": 0.2486, "rewards/accuracies": 0.84375, "rewards/chosen": -2.921875, "rewards/margins": 2.546875, "rewards/rejected": -5.46875, "step": 3750 }, { "epoch": 0.2828982017906854, "grad_norm": 10.826852781877946, "learning_rate": 4.507853797631753e-07, "logits/chosen": -2.546875, "logits/rejected": -2.578125, "logps/chosen": -494.0, "logps/rejected": -736.0, "loss": 0.253, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": -2.984375, "rewards/margins": 2.59375, "rewards/rejected": -5.5625, "step": 3760 }, { "epoch": 0.2836505906252351, "grad_norm": 10.535066020245763, "learning_rate": 4.503934736613161e-07, "logits/chosen": -2.578125, "logits/rejected": -2.703125, "logps/chosen": -498.0, "logps/rejected": -760.0, "loss": 0.2432, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.03125, "rewards/margins": 2.875, "rewards/rejected": -5.90625, "step": 3770 }, { "epoch": 0.2844029794597848, "grad_norm": 9.690016837244414, "learning_rate": 4.500001851137363e-07, "logits/chosen": -2.5625, "logits/rejected": -2.609375, "logps/chosen": -444.0, "logps/rejected": -724.0, "loss": 0.2575, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.703125, "rewards/margins": 2.9375, "rewards/rejected": -5.65625, "step": 3780 }, { "epoch": 0.2851553682943345, "grad_norm": 9.631674051688996, "learning_rate": 4.496055168335986e-07, "logits/chosen": -2.53125, "logits/rejected": -2.546875, "logps/chosen": -458.0, "logps/rejected": -736.0, "loss": 0.2692, "rewards/accuracies": 0.875, "rewards/chosen": -2.75, "rewards/margins": 2.625, "rewards/rejected": -5.375, "step": 3790 }, { "epoch": 0.2859077571288842, "grad_norm": 13.406201921247867, "learning_rate": 4.4920947154358384e-07, "logits/chosen": -2.640625, "logits/rejected": -2.625, "logps/chosen": -496.0, "logps/rejected": -748.0, "loss": 0.2574, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.921875, "rewards/margins": 2.65625, "rewards/rejected": -5.5625, "step": 3800 }, { "epoch": 0.2866601459634339, "grad_norm": 11.331052590475895, "learning_rate": 4.4881205197587236e-07, "logits/chosen": -2.546875, "logits/rejected": -2.59375, "logps/chosen": -486.0, "logps/rejected": -712.0, "loss": 0.2453, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.90625, "rewards/margins": 2.53125, "rewards/rejected": -5.4375, "step": 3810 }, { "epoch": 0.2874125347979836, "grad_norm": 10.132242619101351, "learning_rate": 4.484132608721252e-07, "logits/chosen": -2.515625, "logits/rejected": -2.578125, "logps/chosen": -466.0, "logps/rejected": -740.0, "loss": 0.2574, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.9375, "rewards/margins": 2.734375, "rewards/rejected": -5.65625, "step": 3820 }, { "epoch": 0.2881649236325333, "grad_norm": 15.540753087958967, "learning_rate": 4.480131009834651e-07, "logits/chosen": -2.5625, "logits/rejected": -2.328125, "logps/chosen": -512.0, "logps/rejected": -820.0, "loss": 0.263, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.328125, "rewards/margins": 3.140625, "rewards/rejected": -6.46875, "step": 3830 }, { "epoch": 0.28891731246708297, "grad_norm": 9.760758487501617, "learning_rate": 4.476115750704578e-07, "logits/chosen": -2.484375, "logits/rejected": -2.46875, "logps/chosen": -520.0, "logps/rejected": -784.0, "loss": 0.2763, "rewards/accuracies": 0.90625, "rewards/chosen": -3.265625, "rewards/margins": 2.90625, "rewards/rejected": -6.15625, "step": 3840 }, { "epoch": 0.28966970130163266, "grad_norm": 10.575353249977534, "learning_rate": 4.472086859030926e-07, "logits/chosen": -2.515625, "logits/rejected": -2.453125, "logps/chosen": -468.0, "logps/rejected": -752.0, "loss": 0.2363, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.96875, "rewards/margins": 2.796875, "rewards/rejected": -5.75, "step": 3850 }, { "epoch": 0.29042209013618236, "grad_norm": 8.914185499166935, "learning_rate": 4.468044362607633e-07, "logits/chosen": -2.390625, "logits/rejected": -2.34375, "logps/chosen": -476.0, "logps/rejected": -768.0, "loss": 0.2393, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -2.859375, "rewards/margins": 2.90625, "rewards/rejected": -5.75, "step": 3860 }, { "epoch": 0.29117447897073206, "grad_norm": 12.410632221867862, "learning_rate": 4.4639882893224924e-07, "logits/chosen": -2.484375, "logits/rejected": -2.375, "logps/chosen": -508.0, "logps/rejected": -744.0, "loss": 0.2567, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -3.109375, "rewards/margins": 2.640625, "rewards/rejected": -5.75, "step": 3870 }, { "epoch": 0.29192686780528176, "grad_norm": 12.857195530916082, "learning_rate": 4.4599186671569623e-07, "logits/chosen": -2.5, "logits/rejected": -2.546875, "logps/chosen": -498.0, "logps/rejected": -744.0, "loss": 0.2492, "rewards/accuracies": 0.875, "rewards/chosen": -3.046875, "rewards/margins": 2.609375, "rewards/rejected": -5.65625, "step": 3880 }, { "epoch": 0.29267925663983146, "grad_norm": 11.275475142899934, "learning_rate": 4.4558355241859655e-07, "logits/chosen": -2.65625, "logits/rejected": -2.8125, "logps/chosen": -458.0, "logps/rejected": -740.0, "loss": 0.2629, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.890625, "rewards/margins": 2.90625, "rewards/rejected": -5.8125, "step": 3890 }, { "epoch": 0.29343164547438116, "grad_norm": 12.85112307067444, "learning_rate": 4.451738888577705e-07, "logits/chosen": -2.484375, "logits/rejected": -2.484375, "logps/chosen": -480.0, "logps/rejected": -720.0, "loss": 0.2612, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.890625, "rewards/margins": 2.578125, "rewards/rejected": -5.46875, "step": 3900 }, { "epoch": 0.29418403430893086, "grad_norm": 11.022345608373648, "learning_rate": 4.44762878859346e-07, "logits/chosen": -2.4375, "logits/rejected": -2.609375, "logps/chosen": -488.0, "logps/rejected": -700.0, "loss": 0.2583, "rewards/accuracies": 0.824999988079071, "rewards/chosen": -2.890625, "rewards/margins": 2.375, "rewards/rejected": -5.28125, "step": 3910 }, { "epoch": 0.29493642314348056, "grad_norm": 8.498042065013243, "learning_rate": 4.443505252587399e-07, "logits/chosen": -2.5625, "logits/rejected": -2.703125, "logps/chosen": -480.0, "logps/rejected": -764.0, "loss": 0.2584, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.90625, "rewards/margins": 3.0625, "rewards/rejected": -5.96875, "step": 3920 }, { "epoch": 0.29568881197803026, "grad_norm": 11.222999141565719, "learning_rate": 4.43936830900638e-07, "logits/chosen": -2.421875, "logits/rejected": -2.34375, "logps/chosen": -424.0, "logps/rejected": -728.0, "loss": 0.2478, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.640625, "rewards/margins": 2.96875, "rewards/rejected": -5.59375, "step": 3930 }, { "epoch": 0.29644120081257996, "grad_norm": 14.912732530622156, "learning_rate": 4.4352179863897556e-07, "logits/chosen": -2.46875, "logits/rejected": -2.515625, "logps/chosen": -484.0, "logps/rejected": -720.0, "loss": 0.271, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.953125, "rewards/margins": 2.5625, "rewards/rejected": -5.53125, "step": 3940 }, { "epoch": 0.29719358964712966, "grad_norm": 10.03857079231368, "learning_rate": 4.4310543133691746e-07, "logits/chosen": -2.296875, "logits/rejected": -2.421875, "logps/chosen": -500.0, "logps/rejected": -760.0, "loss": 0.2616, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.875, "rewards/margins": 2.84375, "rewards/rejected": -5.71875, "step": 3950 }, { "epoch": 0.2979459784816793, "grad_norm": 12.756271269442097, "learning_rate": 4.426877318668387e-07, "logits/chosen": -2.6875, "logits/rejected": -2.578125, "logps/chosen": -462.0, "logps/rejected": -748.0, "loss": 0.2419, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.828125, "rewards/margins": 2.921875, "rewards/rejected": -5.75, "step": 3960 }, { "epoch": 0.298698367316229, "grad_norm": 12.229554576456081, "learning_rate": 4.422687031103045e-07, "logits/chosen": -2.53125, "logits/rejected": -2.609375, "logps/chosen": -512.0, "logps/rejected": -768.0, "loss": 0.2888, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -3.078125, "rewards/margins": 2.921875, "rewards/rejected": -6.0, "step": 3970 }, { "epoch": 0.2994507561507787, "grad_norm": 11.672851332396752, "learning_rate": 4.4184834795805016e-07, "logits/chosen": -2.515625, "logits/rejected": -2.46875, "logps/chosen": -430.0, "logps/rejected": -716.0, "loss": 0.2436, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.59375, "rewards/margins": 2.90625, "rewards/rejected": -5.5, "step": 3980 }, { "epoch": 0.3002031449853284, "grad_norm": 11.964930793025617, "learning_rate": 4.414266693099615e-07, "logits/chosen": -2.578125, "logits/rejected": -2.546875, "logps/chosen": -490.0, "logps/rejected": -748.0, "loss": 0.2632, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.828125, "rewards/margins": 2.84375, "rewards/rejected": -5.6875, "step": 3990 }, { "epoch": 0.3009555338198781, "grad_norm": 10.780406539767858, "learning_rate": 4.4100367007505455e-07, "logits/chosen": -2.609375, "logits/rejected": -2.578125, "logps/chosen": -488.0, "logps/rejected": -792.0, "loss": 0.2488, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.03125, "rewards/margins": 3.0625, "rewards/rejected": -6.09375, "step": 4000 }, { "epoch": 0.3017079226544278, "grad_norm": 13.178389561584726, "learning_rate": 4.405793531714558e-07, "logits/chosen": -2.484375, "logits/rejected": -2.625, "logps/chosen": -456.0, "logps/rejected": -748.0, "loss": 0.2534, "rewards/accuracies": 0.90625, "rewards/chosen": -2.71875, "rewards/margins": 3.046875, "rewards/rejected": -5.75, "step": 4010 }, { "epoch": 0.3024603114889775, "grad_norm": 9.846354146786112, "learning_rate": 4.401537215263816e-07, "logits/chosen": -2.515625, "logits/rejected": -2.453125, "logps/chosen": -510.0, "logps/rejected": -752.0, "loss": 0.2434, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.109375, "rewards/margins": 2.65625, "rewards/rejected": -5.75, "step": 4020 }, { "epoch": 0.3032127003235272, "grad_norm": 10.634767101300294, "learning_rate": 4.3972677807611866e-07, "logits/chosen": -2.46875, "logits/rejected": -2.375, "logps/chosen": -454.0, "logps/rejected": -764.0, "loss": 0.2588, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.59375, "rewards/margins": 3.0, "rewards/rejected": -5.59375, "step": 4030 }, { "epoch": 0.3039650891580769, "grad_norm": 11.969763150579288, "learning_rate": 4.39298525766003e-07, "logits/chosen": -2.625, "logits/rejected": -2.59375, "logps/chosen": -472.0, "logps/rejected": -728.0, "loss": 0.2394, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.765625, "rewards/margins": 2.828125, "rewards/rejected": -5.59375, "step": 4040 }, { "epoch": 0.3047174779926266, "grad_norm": 13.662430666895762, "learning_rate": 4.388689675504002e-07, "logits/chosen": -2.5625, "logits/rejected": -2.625, "logps/chosen": -506.0, "logps/rejected": -768.0, "loss": 0.2601, "rewards/accuracies": 0.84375, "rewards/chosen": -2.96875, "rewards/margins": 2.828125, "rewards/rejected": -5.78125, "step": 4050 }, { "epoch": 0.3054698668271763, "grad_norm": 10.374289106825854, "learning_rate": 4.3843810639268475e-07, "logits/chosen": -2.609375, "logits/rejected": -2.546875, "logps/chosen": -470.0, "logps/rejected": -756.0, "loss": 0.2631, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.75, "rewards/margins": 3.15625, "rewards/rejected": -5.90625, "step": 4060 }, { "epoch": 0.306222255661726, "grad_norm": 8.031050170061544, "learning_rate": 4.3800594526521983e-07, "logits/chosen": -2.40625, "logits/rejected": -2.484375, "logps/chosen": -430.0, "logps/rejected": -680.0, "loss": 0.2503, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.546875, "rewards/margins": 2.59375, "rewards/rejected": -5.15625, "step": 4070 }, { "epoch": 0.30697464449627565, "grad_norm": 12.112511125269888, "learning_rate": 4.375724871493365e-07, "logits/chosen": -2.515625, "logits/rejected": -2.453125, "logps/chosen": -436.0, "logps/rejected": -700.0, "loss": 0.259, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.46875, "rewards/margins": 2.859375, "rewards/rejected": -5.34375, "step": 4080 }, { "epoch": 0.30772703333082535, "grad_norm": 9.644656096795064, "learning_rate": 4.371377350353134e-07, "logits/chosen": -2.3125, "logits/rejected": -2.4375, "logps/chosen": -444.0, "logps/rejected": -712.0, "loss": 0.2587, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.625, "rewards/margins": 2.828125, "rewards/rejected": -5.4375, "step": 4090 }, { "epoch": 0.30847942216537505, "grad_norm": 12.410770425440592, "learning_rate": 4.3670169192235586e-07, "logits/chosen": -2.5, "logits/rejected": -2.484375, "logps/chosen": -472.0, "logps/rejected": -708.0, "loss": 0.2687, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.9375, "rewards/margins": 2.484375, "rewards/rejected": -5.40625, "step": 4100 }, { "epoch": 0.30923181099992475, "grad_norm": 10.197784428002407, "learning_rate": 4.362643608185757e-07, "logits/chosen": -2.40625, "logits/rejected": -2.46875, "logps/chosen": -488.0, "logps/rejected": -740.0, "loss": 0.264, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.78125, "rewards/margins": 2.6875, "rewards/rejected": -5.46875, "step": 4110 }, { "epoch": 0.30998419983447445, "grad_norm": 16.812024332129702, "learning_rate": 4.358257447409698e-07, "logits/chosen": -2.59375, "logits/rejected": -2.53125, "logps/chosen": -470.0, "logps/rejected": -768.0, "loss": 0.2458, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.859375, "rewards/margins": 3.0, "rewards/rejected": -5.875, "step": 4120 }, { "epoch": 0.31073658866902415, "grad_norm": 8.164681258059472, "learning_rate": 4.353858467153998e-07, "logits/chosen": -2.484375, "logits/rejected": -2.34375, "logps/chosen": -476.0, "logps/rejected": -732.0, "loss": 0.2698, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.953125, "rewards/margins": 2.6875, "rewards/rejected": -5.65625, "step": 4130 }, { "epoch": 0.31148897750357385, "grad_norm": 12.726703171919148, "learning_rate": 4.349446697765711e-07, "logits/chosen": -2.546875, "logits/rejected": -2.5625, "logps/chosen": -464.0, "logps/rejected": -712.0, "loss": 0.2656, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.75, "rewards/margins": 2.71875, "rewards/rejected": -5.46875, "step": 4140 }, { "epoch": 0.31224136633812355, "grad_norm": 11.213959456331763, "learning_rate": 4.345022169680117e-07, "logits/chosen": -2.609375, "logits/rejected": -2.484375, "logps/chosen": -446.0, "logps/rejected": -740.0, "loss": 0.2432, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.71875, "rewards/margins": 2.96875, "rewards/rejected": -5.6875, "step": 4150 }, { "epoch": 0.31299375517267325, "grad_norm": 10.438058563368365, "learning_rate": 4.3405849134205164e-07, "logits/chosen": -2.625, "logits/rejected": -2.609375, "logps/chosen": -464.0, "logps/rejected": -736.0, "loss": 0.257, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.8125, "rewards/margins": 2.734375, "rewards/rejected": -5.53125, "step": 4160 }, { "epoch": 0.31374614400722295, "grad_norm": 13.199417761020497, "learning_rate": 4.3361349595980136e-07, "logits/chosen": -2.5625, "logits/rejected": -2.453125, "logps/chosen": -472.0, "logps/rejected": -752.0, "loss": 0.2303, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.734375, "rewards/margins": 2.90625, "rewards/rejected": -5.65625, "step": 4170 }, { "epoch": 0.31449853284177265, "grad_norm": 13.982432755106686, "learning_rate": 4.331672338911313e-07, "logits/chosen": -2.53125, "logits/rejected": -2.46875, "logps/chosen": -478.0, "logps/rejected": -776.0, "loss": 0.2369, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.921875, "rewards/margins": 3.078125, "rewards/rejected": -6.0, "step": 4180 }, { "epoch": 0.31525092167632235, "grad_norm": 16.20255717389604, "learning_rate": 4.3271970821465e-07, "logits/chosen": -2.421875, "logits/rejected": -2.640625, "logps/chosen": -490.0, "logps/rejected": -780.0, "loss": 0.2436, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.953125, "rewards/margins": 2.96875, "rewards/rejected": -5.90625, "step": 4190 }, { "epoch": 0.316003310510872, "grad_norm": 13.660473912452064, "learning_rate": 4.3227092201768346e-07, "logits/chosen": -2.515625, "logits/rejected": -2.453125, "logps/chosen": -480.0, "logps/rejected": -740.0, "loss": 0.2555, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -3.015625, "rewards/margins": 2.71875, "rewards/rejected": -5.75, "step": 4200 }, { "epoch": 0.3167556993454217, "grad_norm": 11.234258427267267, "learning_rate": 4.3182087839625326e-07, "logits/chosen": -2.453125, "logits/rejected": -2.328125, "logps/chosen": -464.0, "logps/rejected": -736.0, "loss": 0.2518, "rewards/accuracies": 0.90625, "rewards/chosen": -2.796875, "rewards/margins": 2.765625, "rewards/rejected": -5.5625, "step": 4210 }, { "epoch": 0.3175080881799714, "grad_norm": 9.990306994309964, "learning_rate": 4.313695804550559e-07, "logits/chosen": -2.5, "logits/rejected": -2.421875, "logps/chosen": -464.0, "logps/rejected": -744.0, "loss": 0.2434, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.765625, "rewards/margins": 2.71875, "rewards/rejected": -5.46875, "step": 4220 }, { "epoch": 0.3182604770145211, "grad_norm": 9.406816667503909, "learning_rate": 4.309170313074408e-07, "logits/chosen": -2.53125, "logits/rejected": -2.453125, "logps/chosen": -486.0, "logps/rejected": -768.0, "loss": 0.2777, "rewards/accuracies": 0.90625, "rewards/chosen": -3.015625, "rewards/margins": 2.703125, "rewards/rejected": -5.71875, "step": 4230 }, { "epoch": 0.3190128658490708, "grad_norm": 10.832802247234659, "learning_rate": 4.304632340753889e-07, "logits/chosen": -2.421875, "logits/rejected": -2.375, "logps/chosen": -450.0, "logps/rejected": -744.0, "loss": 0.2615, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.671875, "rewards/margins": 3.03125, "rewards/rejected": -5.6875, "step": 4240 }, { "epoch": 0.3197652546836205, "grad_norm": 12.060575186405842, "learning_rate": 4.3000819188949145e-07, "logits/chosen": -2.46875, "logits/rejected": -2.59375, "logps/chosen": -466.0, "logps/rejected": -688.0, "loss": 0.2557, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -2.90625, "rewards/margins": 2.375, "rewards/rejected": -5.28125, "step": 4250 }, { "epoch": 0.3205176435181702, "grad_norm": 14.46898376617159, "learning_rate": 4.2955190788892827e-07, "logits/chosen": -2.59375, "logits/rejected": -2.65625, "logps/chosen": -488.0, "logps/rejected": -744.0, "loss": 0.2543, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.0, "rewards/margins": 2.71875, "rewards/rejected": -5.71875, "step": 4260 }, { "epoch": 0.3212700323527199, "grad_norm": 12.532680960498416, "learning_rate": 4.2909438522144566e-07, "logits/chosen": -2.46875, "logits/rejected": -2.375, "logps/chosen": -462.0, "logps/rejected": -764.0, "loss": 0.2654, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.796875, "rewards/margins": 3.015625, "rewards/rejected": -5.8125, "step": 4270 }, { "epoch": 0.3220224211872696, "grad_norm": 11.217782708368768, "learning_rate": 4.2863562704333545e-07, "logits/chosen": -2.515625, "logits/rejected": -2.5, "logps/chosen": -464.0, "logps/rejected": -732.0, "loss": 0.228, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -2.703125, "rewards/margins": 2.828125, "rewards/rejected": -5.53125, "step": 4280 }, { "epoch": 0.3227748100218193, "grad_norm": 10.727863986833247, "learning_rate": 4.2817563651941263e-07, "logits/chosen": -2.59375, "logits/rejected": -2.578125, "logps/chosen": -478.0, "logps/rejected": -744.0, "loss": 0.2418, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -2.96875, "rewards/margins": 2.796875, "rewards/rejected": -5.78125, "step": 4290 }, { "epoch": 0.323527198856369, "grad_norm": 10.995592719786089, "learning_rate": 4.277144168229939e-07, "logits/chosen": -2.546875, "logits/rejected": -2.359375, "logps/chosen": -452.0, "logps/rejected": -728.0, "loss": 0.2614, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.6875, "rewards/margins": 2.75, "rewards/rejected": -5.4375, "step": 4300 }, { "epoch": 0.3242795876909187, "grad_norm": 9.088522897788746, "learning_rate": 4.272519711358753e-07, "logits/chosen": -2.546875, "logits/rejected": -2.5625, "logps/chosen": -436.0, "logps/rejected": -704.0, "loss": 0.219, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.546875, "rewards/margins": 2.78125, "rewards/rejected": -5.3125, "step": 4310 }, { "epoch": 0.32503197652546834, "grad_norm": 12.163573920920753, "learning_rate": 4.2678830264831077e-07, "logits/chosen": -2.53125, "logits/rejected": -2.5625, "logps/chosen": -470.0, "logps/rejected": -756.0, "loss": 0.2343, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -2.703125, "rewards/margins": 3.0, "rewards/rejected": -5.71875, "step": 4320 }, { "epoch": 0.32578436536001804, "grad_norm": 12.38048376999724, "learning_rate": 4.2632341455899e-07, "logits/chosen": -2.625, "logits/rejected": -2.5625, "logps/chosen": -480.0, "logps/rejected": -752.0, "loss": 0.2407, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": -3.015625, "rewards/margins": 2.84375, "rewards/rejected": -5.875, "step": 4330 }, { "epoch": 0.32653675419456774, "grad_norm": 9.081663390239243, "learning_rate": 4.25857310075016e-07, "logits/chosen": -2.6875, "logits/rejected": -2.671875, "logps/chosen": -466.0, "logps/rejected": -748.0, "loss": 0.2553, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.765625, "rewards/margins": 2.984375, "rewards/rejected": -5.75, "step": 4340 }, { "epoch": 0.32728914302911744, "grad_norm": 14.471799360031962, "learning_rate": 4.2538999241188374e-07, "logits/chosen": -2.5625, "logits/rejected": -2.4375, "logps/chosen": -478.0, "logps/rejected": -728.0, "loss": 0.2339, "rewards/accuracies": 0.875, "rewards/chosen": -2.96875, "rewards/margins": 2.796875, "rewards/rejected": -5.78125, "step": 4350 }, { "epoch": 0.32804153186366714, "grad_norm": 12.009972841651225, "learning_rate": 4.2492146479345693e-07, "logits/chosen": -2.5, "logits/rejected": -2.328125, "logps/chosen": -486.0, "logps/rejected": -796.0, "loss": 0.2454, "rewards/accuracies": 0.90625, "rewards/chosen": -3.046875, "rewards/margins": 3.15625, "rewards/rejected": -6.21875, "step": 4360 }, { "epoch": 0.32879392069821683, "grad_norm": 12.431757533541964, "learning_rate": 4.244517304519467e-07, "logits/chosen": -2.546875, "logits/rejected": -2.421875, "logps/chosen": -444.0, "logps/rejected": -788.0, "loss": 0.2532, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.734375, "rewards/margins": 3.265625, "rewards/rejected": -6.0, "step": 4370 }, { "epoch": 0.32954630953276653, "grad_norm": 11.085611383835223, "learning_rate": 4.2398079262788893e-07, "logits/chosen": -2.59375, "logits/rejected": -2.578125, "logps/chosen": -452.0, "logps/rejected": -716.0, "loss": 0.2427, "rewards/accuracies": 0.90625, "rewards/chosen": -2.71875, "rewards/margins": 2.71875, "rewards/rejected": -5.4375, "step": 4380 }, { "epoch": 0.33029869836731623, "grad_norm": 12.15918940977203, "learning_rate": 4.2350865457012184e-07, "logits/chosen": -2.609375, "logits/rejected": -2.453125, "logps/chosen": -456.0, "logps/rejected": -732.0, "loss": 0.2392, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.65625, "rewards/margins": 2.921875, "rewards/rejected": -5.59375, "step": 4390 }, { "epoch": 0.33105108720186593, "grad_norm": 11.857781044355193, "learning_rate": 4.2303531953576366e-07, "logits/chosen": -2.65625, "logits/rejected": -2.5625, "logps/chosen": -478.0, "logps/rejected": -776.0, "loss": 0.2547, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.796875, "rewards/margins": 3.09375, "rewards/rejected": -5.90625, "step": 4400 }, { "epoch": 0.33180347603641563, "grad_norm": 15.067235228350018, "learning_rate": 4.2256079079019015e-07, "logits/chosen": -2.640625, "logits/rejected": -2.75, "logps/chosen": -460.0, "logps/rejected": -756.0, "loss": 0.2701, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.859375, "rewards/margins": 2.984375, "rewards/rejected": -5.84375, "step": 4410 }, { "epoch": 0.33255586487096533, "grad_norm": 10.482790501659238, "learning_rate": 4.220850716070121e-07, "logits/chosen": -2.609375, "logits/rejected": -2.671875, "logps/chosen": -468.0, "logps/rejected": -728.0, "loss": 0.2398, "rewards/accuracies": 0.90625, "rewards/chosen": -2.796875, "rewards/margins": 2.84375, "rewards/rejected": -5.625, "step": 4420 }, { "epoch": 0.33330825370551503, "grad_norm": 11.740905064320092, "learning_rate": 4.2160816526805267e-07, "logits/chosen": -2.625, "logits/rejected": -2.453125, "logps/chosen": -470.0, "logps/rejected": -788.0, "loss": 0.2497, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.921875, "rewards/margins": 3.21875, "rewards/rejected": -6.15625, "step": 4430 }, { "epoch": 0.3340606425400647, "grad_norm": 11.655346483663704, "learning_rate": 4.211300750633248e-07, "logits/chosen": -2.5625, "logits/rejected": -2.53125, "logps/chosen": -496.0, "logps/rejected": -796.0, "loss": 0.2379, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.125, "rewards/margins": 3.03125, "rewards/rejected": -6.15625, "step": 4440 }, { "epoch": 0.3348130313746144, "grad_norm": 12.091830856232859, "learning_rate": 4.2065080429100865e-07, "logits/chosen": -2.734375, "logits/rejected": -2.546875, "logps/chosen": -510.0, "logps/rejected": -800.0, "loss": 0.2415, "rewards/accuracies": 0.90625, "rewards/chosen": -3.1875, "rewards/margins": 2.9375, "rewards/rejected": -6.125, "step": 4450 }, { "epoch": 0.3355654202091641, "grad_norm": 11.980457592825926, "learning_rate": 4.2017035625742846e-07, "logits/chosen": -2.703125, "logits/rejected": -2.578125, "logps/chosen": -498.0, "logps/rejected": -804.0, "loss": 0.2302, "rewards/accuracies": 0.90625, "rewards/chosen": -3.15625, "rewards/margins": 3.09375, "rewards/rejected": -6.25, "step": 4460 }, { "epoch": 0.3363178090437138, "grad_norm": 9.604090623367547, "learning_rate": 4.196887342770302e-07, "logits/chosen": -2.609375, "logits/rejected": -2.671875, "logps/chosen": -510.0, "logps/rejected": -820.0, "loss": 0.2239, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.0625, "rewards/margins": 3.390625, "rewards/rejected": -6.46875, "step": 4470 }, { "epoch": 0.3370701978782635, "grad_norm": 11.129790685834623, "learning_rate": 4.1920594167235845e-07, "logits/chosen": -2.546875, "logits/rejected": -2.484375, "logps/chosen": -498.0, "logps/rejected": -780.0, "loss": 0.2573, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -2.984375, "rewards/margins": 3.078125, "rewards/rejected": -6.0625, "step": 4480 }, { "epoch": 0.3378225867128132, "grad_norm": 10.645049325493066, "learning_rate": 4.187219817740336e-07, "logits/chosen": -2.640625, "logits/rejected": -2.5, "logps/chosen": -476.0, "logps/rejected": -764.0, "loss": 0.2473, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.84375, "rewards/margins": 2.828125, "rewards/rejected": -5.6875, "step": 4490 }, { "epoch": 0.3385749755473629, "grad_norm": 9.558681796260734, "learning_rate": 4.182368579207285e-07, "logits/chosen": -2.484375, "logits/rejected": -2.234375, "logps/chosen": -520.0, "logps/rejected": -788.0, "loss": 0.2593, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.203125, "rewards/margins": 2.84375, "rewards/rejected": -6.0625, "step": 4500 }, { "epoch": 0.3393273643819126, "grad_norm": 12.720628082724422, "learning_rate": 4.177505734591461e-07, "logits/chosen": -2.65625, "logits/rejected": -2.65625, "logps/chosen": -476.0, "logps/rejected": -768.0, "loss": 0.2497, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.78125, "rewards/margins": 3.09375, "rewards/rejected": -5.875, "step": 4510 }, { "epoch": 0.3400797532164623, "grad_norm": 12.767644671779443, "learning_rate": 4.172631317439956e-07, "logits/chosen": -2.453125, "logits/rejected": -2.40625, "logps/chosen": -482.0, "logps/rejected": -772.0, "loss": 0.2525, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.796875, "rewards/margins": 2.921875, "rewards/rejected": -5.71875, "step": 4520 }, { "epoch": 0.340832142051012, "grad_norm": 13.22903559188859, "learning_rate": 4.167745361379702e-07, "logits/chosen": -2.53125, "logits/rejected": -2.546875, "logps/chosen": -488.0, "logps/rejected": -780.0, "loss": 0.2358, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.046875, "rewards/margins": 3.0, "rewards/rejected": -6.0625, "step": 4530 }, { "epoch": 0.3415845308855617, "grad_norm": 11.837789088467614, "learning_rate": 4.162847900117229e-07, "logits/chosen": -2.515625, "logits/rejected": -2.234375, "logps/chosen": -482.0, "logps/rejected": -776.0, "loss": 0.2473, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.96875, "rewards/margins": 2.84375, "rewards/rejected": -5.8125, "step": 4540 }, { "epoch": 0.3423369197201114, "grad_norm": 11.540938053218126, "learning_rate": 4.1579389674384394e-07, "logits/chosen": -2.46875, "logits/rejected": -2.46875, "logps/chosen": -488.0, "logps/rejected": -708.0, "loss": 0.2529, "rewards/accuracies": 0.875, "rewards/chosen": -3.0, "rewards/margins": 2.453125, "rewards/rejected": -5.46875, "step": 4550 }, { "epoch": 0.3430893085546611, "grad_norm": 10.390478627743907, "learning_rate": 4.153018597208374e-07, "logits/chosen": -2.625, "logits/rejected": -2.609375, "logps/chosen": -454.0, "logps/rejected": -732.0, "loss": 0.2382, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.75, "rewards/margins": 2.875, "rewards/rejected": -5.625, "step": 4560 }, { "epoch": 0.3438416973892107, "grad_norm": 12.85986134953924, "learning_rate": 4.1480868233709765e-07, "logits/chosen": -2.65625, "logits/rejected": -2.578125, "logps/chosen": -496.0, "logps/rejected": -792.0, "loss": 0.2387, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -2.890625, "rewards/margins": 3.234375, "rewards/rejected": -6.125, "step": 4570 }, { "epoch": 0.3445940862237604, "grad_norm": 16.069181225786572, "learning_rate": 4.1431436799488606e-07, "logits/chosen": -2.65625, "logits/rejected": -2.5625, "logps/chosen": -506.0, "logps/rejected": -796.0, "loss": 0.2367, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.171875, "rewards/margins": 3.046875, "rewards/rejected": -6.21875, "step": 4580 }, { "epoch": 0.3453464750583101, "grad_norm": 9.24591239596984, "learning_rate": 4.1381892010430717e-07, "logits/chosen": -2.65625, "logits/rejected": -2.390625, "logps/chosen": -498.0, "logps/rejected": -780.0, "loss": 0.2532, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.09375, "rewards/margins": 2.96875, "rewards/rejected": -6.0625, "step": 4590 }, { "epoch": 0.3460988638928598, "grad_norm": 13.11247446769123, "learning_rate": 4.13322342083286e-07, "logits/chosen": -2.421875, "logits/rejected": -2.453125, "logps/chosen": -512.0, "logps/rejected": -772.0, "loss": 0.2562, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.953125, "rewards/margins": 2.90625, "rewards/rejected": -5.84375, "step": 4600 }, { "epoch": 0.3468512527274095, "grad_norm": 11.684064266692445, "learning_rate": 4.1282463735754356e-07, "logits/chosen": -2.46875, "logits/rejected": -2.5625, "logps/chosen": -472.0, "logps/rejected": -724.0, "loss": 0.2679, "rewards/accuracies": 0.875, "rewards/chosen": -2.828125, "rewards/margins": 2.5625, "rewards/rejected": -5.375, "step": 4610 }, { "epoch": 0.3476036415619592, "grad_norm": 15.602275687557375, "learning_rate": 4.123258093605739e-07, "logits/chosen": -2.484375, "logits/rejected": -2.4375, "logps/chosen": -494.0, "logps/rejected": -764.0, "loss": 0.253, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.90625, "rewards/margins": 2.984375, "rewards/rejected": -5.90625, "step": 4620 }, { "epoch": 0.3483560303965089, "grad_norm": 13.917628181771066, "learning_rate": 4.118258615336199e-07, "logits/chosen": -2.65625, "logits/rejected": -2.546875, "logps/chosen": -496.0, "logps/rejected": -780.0, "loss": 0.2381, "rewards/accuracies": 0.90625, "rewards/chosen": -3.0, "rewards/margins": 2.96875, "rewards/rejected": -5.96875, "step": 4630 }, { "epoch": 0.3491084192310586, "grad_norm": 12.23407641371924, "learning_rate": 4.1132479732564964e-07, "logits/chosen": -2.46875, "logits/rejected": -2.46875, "logps/chosen": -512.0, "logps/rejected": -852.0, "loss": 0.2228, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.15625, "rewards/margins": 3.296875, "rewards/rejected": -6.46875, "step": 4640 }, { "epoch": 0.3498608080656083, "grad_norm": 13.650963200025805, "learning_rate": 4.1082262019333315e-07, "logits/chosen": -2.546875, "logits/rejected": -2.28125, "logps/chosen": -472.0, "logps/rejected": -816.0, "loss": 0.2409, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -2.875, "rewards/margins": 3.34375, "rewards/rejected": -6.21875, "step": 4650 }, { "epoch": 0.350613196900158, "grad_norm": 10.18766256888111, "learning_rate": 4.103193336010179e-07, "logits/chosen": -2.484375, "logits/rejected": -2.53125, "logps/chosen": -500.0, "logps/rejected": -768.0, "loss": 0.2267, "rewards/accuracies": 0.875, "rewards/chosen": -2.921875, "rewards/margins": 3.125, "rewards/rejected": -6.0625, "step": 4660 }, { "epoch": 0.3513655857347077, "grad_norm": 12.17101244693378, "learning_rate": 4.098149410207051e-07, "logits/chosen": -2.390625, "logits/rejected": -2.296875, "logps/chosen": -480.0, "logps/rejected": -768.0, "loss": 0.2634, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.0, "rewards/margins": 2.8125, "rewards/rejected": -5.8125, "step": 4670 }, { "epoch": 0.3521179745692574, "grad_norm": 11.620826660095515, "learning_rate": 4.09309445932026e-07, "logits/chosen": -2.5, "logits/rejected": -2.296875, "logps/chosen": -474.0, "logps/rejected": -708.0, "loss": 0.2657, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.96875, "rewards/margins": 2.40625, "rewards/rejected": -5.34375, "step": 4680 }, { "epoch": 0.35287036340380706, "grad_norm": 11.197340386538853, "learning_rate": 4.088028518222173e-07, "logits/chosen": -2.5, "logits/rejected": -2.4375, "logps/chosen": -462.0, "logps/rejected": -716.0, "loss": 0.2381, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.734375, "rewards/margins": 2.828125, "rewards/rejected": -5.5625, "step": 4690 }, { "epoch": 0.35362275223835676, "grad_norm": 9.80619974138951, "learning_rate": 4.0829516218609785e-07, "logits/chosen": -2.5625, "logits/rejected": -2.40625, "logps/chosen": -504.0, "logps/rejected": -824.0, "loss": 0.238, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.0, "rewards/margins": 3.40625, "rewards/rejected": -6.40625, "step": 4700 }, { "epoch": 0.35437514107290646, "grad_norm": 9.03111941093957, "learning_rate": 4.077863805260439e-07, "logits/chosen": -2.5625, "logits/rejected": -2.65625, "logps/chosen": -512.0, "logps/rejected": -776.0, "loss": 0.2405, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.015625, "rewards/margins": 3.015625, "rewards/rejected": -6.03125, "step": 4710 }, { "epoch": 0.35512752990745616, "grad_norm": 9.76333092882564, "learning_rate": 4.0727651035196545e-07, "logits/chosen": -2.609375, "logits/rejected": -2.28125, "logps/chosen": -494.0, "logps/rejected": -884.0, "loss": 0.2426, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.0625, "rewards/margins": 3.71875, "rewards/rejected": -6.78125, "step": 4720 }, { "epoch": 0.35587991874200586, "grad_norm": 13.978467994206847, "learning_rate": 4.0676555518128157e-07, "logits/chosen": -2.484375, "logits/rejected": -2.390625, "logps/chosen": -524.0, "logps/rejected": -760.0, "loss": 0.2737, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -3.328125, "rewards/margins": 2.546875, "rewards/rejected": -5.875, "step": 4730 }, { "epoch": 0.35663230757655556, "grad_norm": 13.4576925201583, "learning_rate": 4.062535185388964e-07, "logits/chosen": -2.453125, "logits/rejected": -2.484375, "logps/chosen": -462.0, "logps/rejected": -720.0, "loss": 0.2676, "rewards/accuracies": 0.875, "rewards/chosen": -2.71875, "rewards/margins": 2.75, "rewards/rejected": -5.46875, "step": 4740 }, { "epoch": 0.35738469641110526, "grad_norm": 10.72850088831487, "learning_rate": 4.0574040395717493e-07, "logits/chosen": -2.5625, "logits/rejected": -2.453125, "logps/chosen": -504.0, "logps/rejected": -796.0, "loss": 0.2232, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.03125, "rewards/margins": 3.0, "rewards/rejected": -6.03125, "step": 4750 }, { "epoch": 0.35813708524565496, "grad_norm": 11.110612459192815, "learning_rate": 4.0522621497591813e-07, "logits/chosen": -2.6875, "logits/rejected": -2.53125, "logps/chosen": -494.0, "logps/rejected": -768.0, "loss": 0.2634, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.171875, "rewards/margins": 2.859375, "rewards/rejected": -6.03125, "step": 4760 }, { "epoch": 0.35888947408020466, "grad_norm": 9.181220064708851, "learning_rate": 4.0471095514233933e-07, "logits/chosen": -2.46875, "logits/rejected": -2.15625, "logps/chosen": -458.0, "logps/rejected": -744.0, "loss": 0.2296, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.765625, "rewards/margins": 2.765625, "rewards/rejected": -5.53125, "step": 4770 }, { "epoch": 0.35964186291475436, "grad_norm": 11.499303644666268, "learning_rate": 4.041946280110389e-07, "logits/chosen": -2.578125, "logits/rejected": -2.390625, "logps/chosen": -434.0, "logps/rejected": -708.0, "loss": 0.2293, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.546875, "rewards/margins": 2.890625, "rewards/rejected": -5.4375, "step": 4780 }, { "epoch": 0.36039425174930406, "grad_norm": 12.333655799931284, "learning_rate": 4.0367723714398047e-07, "logits/chosen": -2.578125, "logits/rejected": -2.515625, "logps/chosen": -484.0, "logps/rejected": -824.0, "loss": 0.2321, "rewards/accuracies": 0.9375, "rewards/chosen": -2.859375, "rewards/margins": 3.5625, "rewards/rejected": -6.4375, "step": 4790 }, { "epoch": 0.36114664058385376, "grad_norm": 12.475119877105636, "learning_rate": 4.031587861104657e-07, "logits/chosen": -2.703125, "logits/rejected": -2.6875, "logps/chosen": -500.0, "logps/rejected": -796.0, "loss": 0.2572, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.15625, "rewards/margins": 3.03125, "rewards/rejected": -6.1875, "step": 4800 }, { "epoch": 0.3618990294184034, "grad_norm": 9.799125718820001, "learning_rate": 4.0263927848711026e-07, "logits/chosen": -2.59375, "logits/rejected": -2.53125, "logps/chosen": -476.0, "logps/rejected": -772.0, "loss": 0.2488, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.828125, "rewards/margins": 3.1875, "rewards/rejected": -6.03125, "step": 4810 }, { "epoch": 0.3626514182529531, "grad_norm": 12.448159514651598, "learning_rate": 4.021187178578187e-07, "logits/chosen": -2.4375, "logits/rejected": -2.515625, "logps/chosen": -458.0, "logps/rejected": -748.0, "loss": 0.2385, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.75, "rewards/margins": 2.96875, "rewards/rejected": -5.71875, "step": 4820 }, { "epoch": 0.3634038070875028, "grad_norm": 11.548303365928984, "learning_rate": 4.0159710781375994e-07, "logits/chosen": -2.640625, "logits/rejected": -2.609375, "logps/chosen": -484.0, "logps/rejected": -736.0, "loss": 0.2522, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -3.015625, "rewards/margins": 2.59375, "rewards/rejected": -5.625, "step": 4830 }, { "epoch": 0.3641561959220525, "grad_norm": 14.681020867439685, "learning_rate": 4.0107445195334244e-07, "logits/chosen": -2.6875, "logits/rejected": -2.625, "logps/chosen": -500.0, "logps/rejected": -768.0, "loss": 0.2409, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.015625, "rewards/margins": 2.953125, "rewards/rejected": -5.96875, "step": 4840 }, { "epoch": 0.3649085847566022, "grad_norm": 9.764957828852294, "learning_rate": 4.005507538821894e-07, "logits/chosen": -2.734375, "logits/rejected": -2.5625, "logps/chosen": -478.0, "logps/rejected": -812.0, "loss": 0.2204, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.671875, "rewards/margins": 3.484375, "rewards/rejected": -6.15625, "step": 4850 }, { "epoch": 0.3656609735911519, "grad_norm": 9.575015952484588, "learning_rate": 4.0002601721311393e-07, "logits/chosen": -2.484375, "logits/rejected": -2.546875, "logps/chosen": -480.0, "logps/rejected": -780.0, "loss": 0.2265, "rewards/accuracies": 0.90625, "rewards/chosen": -2.875, "rewards/margins": 3.234375, "rewards/rejected": -6.125, "step": 4860 }, { "epoch": 0.3664133624257016, "grad_norm": 9.905856288823637, "learning_rate": 3.995002455660939e-07, "logits/chosen": -2.78125, "logits/rejected": -2.765625, "logps/chosen": -464.0, "logps/rejected": -808.0, "loss": 0.2187, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -2.78125, "rewards/margins": 3.546875, "rewards/rejected": -6.3125, "step": 4870 }, { "epoch": 0.3671657512602513, "grad_norm": 10.803271067277562, "learning_rate": 3.989734425682473e-07, "logits/chosen": -2.703125, "logits/rejected": -2.78125, "logps/chosen": -464.0, "logps/rejected": -748.0, "loss": 0.2395, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.84375, "rewards/margins": 2.984375, "rewards/rejected": -5.8125, "step": 4880 }, { "epoch": 0.367918140094801, "grad_norm": 10.097977363001046, "learning_rate": 3.984456118538071e-07, "logits/chosen": -2.671875, "logits/rejected": -2.609375, "logps/chosen": -472.0, "logps/rejected": -788.0, "loss": 0.2356, "rewards/accuracies": 0.90625, "rewards/chosen": -2.90625, "rewards/margins": 3.0625, "rewards/rejected": -5.96875, "step": 4890 }, { "epoch": 0.3686705289293507, "grad_norm": 13.680420774110893, "learning_rate": 3.9791675706409593e-07, "logits/chosen": -2.65625, "logits/rejected": -2.421875, "logps/chosen": -470.0, "logps/rejected": -824.0, "loss": 0.2562, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -2.875, "rewards/margins": 3.578125, "rewards/rejected": -6.46875, "step": 4900 }, { "epoch": 0.3694229177639004, "grad_norm": 11.445281227328937, "learning_rate": 3.9738688184750125e-07, "logits/chosen": -2.671875, "logits/rejected": -2.6875, "logps/chosen": -494.0, "logps/rejected": -752.0, "loss": 0.2373, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.890625, "rewards/margins": 2.78125, "rewards/rejected": -5.65625, "step": 4910 }, { "epoch": 0.3701753065984501, "grad_norm": 13.278454775727404, "learning_rate": 3.968559898594502e-07, "logits/chosen": -2.671875, "logits/rejected": -2.875, "logps/chosen": -420.0, "logps/rejected": -692.0, "loss": 0.2503, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.546875, "rewards/margins": 2.890625, "rewards/rejected": -5.4375, "step": 4920 }, { "epoch": 0.37092769543299975, "grad_norm": 11.837169286058144, "learning_rate": 3.9632408476238416e-07, "logits/chosen": -2.578125, "logits/rejected": -2.453125, "logps/chosen": -478.0, "logps/rejected": -720.0, "loss": 0.24, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -2.921875, "rewards/margins": 2.515625, "rewards/rejected": -5.4375, "step": 4930 }, { "epoch": 0.37168008426754945, "grad_norm": 10.527553535270822, "learning_rate": 3.957911702257337e-07, "logits/chosen": -2.6875, "logits/rejected": -2.65625, "logps/chosen": -480.0, "logps/rejected": -816.0, "loss": 0.2367, "rewards/accuracies": 0.90625, "rewards/chosen": -2.734375, "rewards/margins": 3.53125, "rewards/rejected": -6.28125, "step": 4940 }, { "epoch": 0.37243247310209915, "grad_norm": 10.970467877711682, "learning_rate": 3.952572499258929e-07, "logits/chosen": -2.765625, "logits/rejected": -2.71875, "logps/chosen": -474.0, "logps/rejected": -760.0, "loss": 0.2402, "rewards/accuracies": 0.875, "rewards/chosen": -2.859375, "rewards/margins": 3.0625, "rewards/rejected": -5.9375, "step": 4950 }, { "epoch": 0.37318486193664885, "grad_norm": 11.9269319413882, "learning_rate": 3.9472232754619474e-07, "logits/chosen": -2.75, "logits/rejected": -2.640625, "logps/chosen": -476.0, "logps/rejected": -756.0, "loss": 0.2409, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.78125, "rewards/margins": 3.046875, "rewards/rejected": -5.84375, "step": 4960 }, { "epoch": 0.37393725077119855, "grad_norm": 10.97716432080365, "learning_rate": 3.9418640677688464e-07, "logits/chosen": -2.546875, "logits/rejected": -2.625, "logps/chosen": -500.0, "logps/rejected": -736.0, "loss": 0.2603, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.96875, "rewards/margins": 2.625, "rewards/rejected": -5.59375, "step": 4970 }, { "epoch": 0.37468963960574825, "grad_norm": 12.043977435148081, "learning_rate": 3.936494913150961e-07, "logits/chosen": -2.578125, "logits/rejected": -2.546875, "logps/chosen": -498.0, "logps/rejected": -728.0, "loss": 0.2323, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.046875, "rewards/margins": 2.484375, "rewards/rejected": -5.53125, "step": 4980 }, { "epoch": 0.37544202844029795, "grad_norm": 12.313793553318312, "learning_rate": 3.931115848648242e-07, "logits/chosen": -2.734375, "logits/rejected": -2.78125, "logps/chosen": -476.0, "logps/rejected": -784.0, "loss": 0.2251, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.921875, "rewards/margins": 3.28125, "rewards/rejected": -6.1875, "step": 4990 }, { "epoch": 0.37619441727484765, "grad_norm": 11.795523958860512, "learning_rate": 3.925726911369008e-07, "logits/chosen": -2.6875, "logits/rejected": -2.765625, "logps/chosen": -484.0, "logps/rejected": -764.0, "loss": 0.2272, "rewards/accuracies": 0.90625, "rewards/chosen": -2.890625, "rewards/margins": 3.015625, "rewards/rejected": -5.90625, "step": 5000 }, { "epoch": 0.37694680610939735, "grad_norm": 9.068915769578124, "learning_rate": 3.9203281384896856e-07, "logits/chosen": -2.8125, "logits/rejected": -2.78125, "logps/chosen": -484.0, "logps/rejected": -768.0, "loss": 0.2395, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -3.078125, "rewards/margins": 3.015625, "rewards/rejected": -6.09375, "step": 5010 }, { "epoch": 0.37769919494394705, "grad_norm": 11.032306076169599, "learning_rate": 3.9149195672545547e-07, "logits/chosen": -2.640625, "logits/rejected": -2.390625, "logps/chosen": -480.0, "logps/rejected": -760.0, "loss": 0.2491, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -2.921875, "rewards/margins": 2.8125, "rewards/rejected": -5.75, "step": 5020 }, { "epoch": 0.37845158377849675, "grad_norm": 12.36365448814916, "learning_rate": 3.9095012349754897e-07, "logits/chosen": -2.65625, "logits/rejected": -2.5625, "logps/chosen": -494.0, "logps/rejected": -796.0, "loss": 0.2537, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.03125, "rewards/margins": 3.265625, "rewards/rejected": -6.28125, "step": 5030 }, { "epoch": 0.37920397261304645, "grad_norm": 8.751375333127545, "learning_rate": 3.904073179031702e-07, "logits/chosen": -2.703125, "logits/rejected": -2.453125, "logps/chosen": -498.0, "logps/rejected": -816.0, "loss": 0.2198, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.015625, "rewards/margins": 3.1875, "rewards/rejected": -6.21875, "step": 5040 }, { "epoch": 0.3799563614475961, "grad_norm": 11.536841940834115, "learning_rate": 3.898635436869485e-07, "logits/chosen": -2.625, "logits/rejected": -2.609375, "logps/chosen": -490.0, "logps/rejected": -784.0, "loss": 0.2354, "rewards/accuracies": 0.90625, "rewards/chosen": -3.078125, "rewards/margins": 3.109375, "rewards/rejected": -6.1875, "step": 5050 }, { "epoch": 0.3807087502821458, "grad_norm": 14.89859255079245, "learning_rate": 3.8931880460019537e-07, "logits/chosen": -2.640625, "logits/rejected": -2.6875, "logps/chosen": -474.0, "logps/rejected": -704.0, "loss": 0.2457, "rewards/accuracies": 0.875, "rewards/chosen": -2.9375, "rewards/margins": 2.46875, "rewards/rejected": -5.40625, "step": 5060 }, { "epoch": 0.3814611391166955, "grad_norm": 12.920123184221213, "learning_rate": 3.887731044008787e-07, "logits/chosen": -2.515625, "logits/rejected": -2.65625, "logps/chosen": -502.0, "logps/rejected": -796.0, "loss": 0.2215, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.15625, "rewards/margins": 2.9375, "rewards/rejected": -6.09375, "step": 5070 }, { "epoch": 0.3822135279512452, "grad_norm": 12.036197386119813, "learning_rate": 3.8822644685359655e-07, "logits/chosen": -2.578125, "logits/rejected": -2.765625, "logps/chosen": -500.0, "logps/rejected": -816.0, "loss": 0.2535, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.09375, "rewards/margins": 3.1875, "rewards/rejected": -6.28125, "step": 5080 }, { "epoch": 0.3829659167857949, "grad_norm": 16.575852783043803, "learning_rate": 3.876788357295516e-07, "logits/chosen": -2.65625, "logits/rejected": -2.515625, "logps/chosen": -452.0, "logps/rejected": -744.0, "loss": 0.2397, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.71875, "rewards/margins": 2.984375, "rewards/rejected": -5.6875, "step": 5090 }, { "epoch": 0.3837183056203446, "grad_norm": 11.589292095995193, "learning_rate": 3.871302748065246e-07, "logits/chosen": -2.71875, "logits/rejected": -2.59375, "logps/chosen": -432.0, "logps/rejected": -772.0, "loss": 0.2228, "rewards/accuracies": 0.90625, "rewards/chosen": -2.65625, "rewards/margins": 3.0625, "rewards/rejected": -5.71875, "step": 5100 }, { "epoch": 0.3844706944548943, "grad_norm": 11.168759786231032, "learning_rate": 3.8658076786884917e-07, "logits/chosen": -2.875, "logits/rejected": -2.84375, "logps/chosen": -484.0, "logps/rejected": -832.0, "loss": 0.2183, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.015625, "rewards/margins": 3.546875, "rewards/rejected": -6.5625, "step": 5110 }, { "epoch": 0.385223083289444, "grad_norm": 13.184244540670703, "learning_rate": 3.860303187073848e-07, "logits/chosen": -2.765625, "logits/rejected": -2.671875, "logps/chosen": -502.0, "logps/rejected": -832.0, "loss": 0.2117, "rewards/accuracies": 0.875, "rewards/chosen": -3.1875, "rewards/margins": 3.390625, "rewards/rejected": -6.5625, "step": 5120 }, { "epoch": 0.3859754721239937, "grad_norm": 10.625608574202191, "learning_rate": 3.854789311194909e-07, "logits/chosen": -2.71875, "logits/rejected": -2.71875, "logps/chosen": -482.0, "logps/rejected": -752.0, "loss": 0.2325, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.96875, "rewards/margins": 2.90625, "rewards/rejected": -5.875, "step": 5130 }, { "epoch": 0.3867278609585434, "grad_norm": 12.57011905614842, "learning_rate": 3.849266089090013e-07, "logits/chosen": -2.71875, "logits/rejected": -2.8125, "logps/chosen": -478.0, "logps/rejected": -812.0, "loss": 0.2213, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -2.875, "rewards/margins": 3.390625, "rewards/rejected": -6.28125, "step": 5140 }, { "epoch": 0.3874802497930931, "grad_norm": 11.359846835999114, "learning_rate": 3.843733558861969e-07, "logits/chosen": -2.65625, "logits/rejected": -2.53125, "logps/chosen": -462.0, "logps/rejected": -768.0, "loss": 0.2339, "rewards/accuracies": 0.875, "rewards/chosen": -2.765625, "rewards/margins": 3.109375, "rewards/rejected": -5.875, "step": 5150 }, { "epoch": 0.3882326386276428, "grad_norm": 12.325213936894796, "learning_rate": 3.8381917586778036e-07, "logits/chosen": -2.625, "logits/rejected": -2.609375, "logps/chosen": -454.0, "logps/rejected": -720.0, "loss": 0.2377, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.703125, "rewards/margins": 2.8125, "rewards/rejected": -5.53125, "step": 5160 }, { "epoch": 0.38898502746219243, "grad_norm": 11.505274123537891, "learning_rate": 3.8326407267684916e-07, "logits/chosen": -2.703125, "logits/rejected": -2.546875, "logps/chosen": -476.0, "logps/rejected": -832.0, "loss": 0.2368, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.734375, "rewards/margins": 3.734375, "rewards/rejected": -6.46875, "step": 5170 }, { "epoch": 0.38973741629674213, "grad_norm": 15.79862418997355, "learning_rate": 3.827080501428695e-07, "logits/chosen": -2.640625, "logits/rejected": -2.671875, "logps/chosen": -456.0, "logps/rejected": -752.0, "loss": 0.2234, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -2.6875, "rewards/margins": 3.109375, "rewards/rejected": -5.8125, "step": 5180 }, { "epoch": 0.39048980513129183, "grad_norm": 10.583632507267652, "learning_rate": 3.821511121016498e-07, "logits/chosen": -2.828125, "logits/rejected": -2.828125, "logps/chosen": -454.0, "logps/rejected": -748.0, "loss": 0.2329, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.796875, "rewards/margins": 3.0, "rewards/rejected": -5.78125, "step": 5190 }, { "epoch": 0.39124219396584153, "grad_norm": 11.138449144955004, "learning_rate": 3.815932623953142e-07, "logits/chosen": -2.671875, "logits/rejected": -2.640625, "logps/chosen": -474.0, "logps/rejected": -772.0, "loss": 0.2207, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -2.796875, "rewards/margins": 3.109375, "rewards/rejected": -5.90625, "step": 5200 }, { "epoch": 0.39199458280039123, "grad_norm": 37.77866751608172, "learning_rate": 3.810345048722762e-07, "logits/chosen": -2.734375, "logits/rejected": -2.828125, "logps/chosen": -490.0, "logps/rejected": -808.0, "loss": 0.2225, "rewards/accuracies": 0.90625, "rewards/chosen": -2.953125, "rewards/margins": 3.328125, "rewards/rejected": -6.28125, "step": 5210 }, { "epoch": 0.39274697163494093, "grad_norm": 10.896238031089728, "learning_rate": 3.804748433872119e-07, "logits/chosen": -2.546875, "logits/rejected": -2.453125, "logps/chosen": -482.0, "logps/rejected": -764.0, "loss": 0.2106, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.953125, "rewards/margins": 2.953125, "rewards/rejected": -5.90625, "step": 5220 }, { "epoch": 0.39349936046949063, "grad_norm": 12.598795815981047, "learning_rate": 3.7991428180103375e-07, "logits/chosen": -2.640625, "logits/rejected": -2.828125, "logps/chosen": -494.0, "logps/rejected": -832.0, "loss": 0.2401, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.90625, "rewards/margins": 3.59375, "rewards/rejected": -6.5, "step": 5230 }, { "epoch": 0.39425174930404033, "grad_norm": 15.659469614082827, "learning_rate": 3.7935282398086335e-07, "logits/chosen": -2.59375, "logits/rejected": -2.6875, "logps/chosen": -482.0, "logps/rejected": -784.0, "loss": 0.2557, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.84375, "rewards/margins": 3.296875, "rewards/rejected": -6.125, "step": 5240 }, { "epoch": 0.39500413813859003, "grad_norm": 13.994500662425821, "learning_rate": 3.787904738000053e-07, "logits/chosen": -2.65625, "logits/rejected": -2.578125, "logps/chosen": -476.0, "logps/rejected": -772.0, "loss": 0.2392, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.765625, "rewards/margins": 2.96875, "rewards/rejected": -5.75, "step": 5250 }, { "epoch": 0.39575652697313973, "grad_norm": 12.47081952602547, "learning_rate": 3.7822723513792036e-07, "logits/chosen": -2.640625, "logits/rejected": -2.65625, "logps/chosen": -466.0, "logps/rejected": -748.0, "loss": 0.2242, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.75, "rewards/margins": 3.109375, "rewards/rejected": -5.84375, "step": 5260 }, { "epoch": 0.39650891580768943, "grad_norm": 9.709942550279477, "learning_rate": 3.776631118801985e-07, "logits/chosen": -2.578125, "logits/rejected": -2.359375, "logps/chosen": -486.0, "logps/rejected": -772.0, "loss": 0.2173, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -3.015625, "rewards/margins": 3.015625, "rewards/rejected": -6.0, "step": 5270 }, { "epoch": 0.39726130464223913, "grad_norm": 12.160308537210607, "learning_rate": 3.7709810791853224e-07, "logits/chosen": -2.6875, "logits/rejected": -2.625, "logps/chosen": -506.0, "logps/rejected": -864.0, "loss": 0.2458, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.171875, "rewards/margins": 3.71875, "rewards/rejected": -6.875, "step": 5280 }, { "epoch": 0.3980136934767888, "grad_norm": 9.763213249539588, "learning_rate": 3.765322271506898e-07, "logits/chosen": -2.71875, "logits/rejected": -2.65625, "logps/chosen": -480.0, "logps/rejected": -792.0, "loss": 0.2216, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.015625, "rewards/margins": 3.265625, "rewards/rejected": -6.28125, "step": 5290 }, { "epoch": 0.3987660823113385, "grad_norm": 11.099073660492174, "learning_rate": 3.7596547348048806e-07, "logits/chosen": -2.578125, "logits/rejected": -2.421875, "logps/chosen": -468.0, "logps/rejected": -760.0, "loss": 0.2292, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -2.9375, "rewards/margins": 2.859375, "rewards/rejected": -5.78125, "step": 5300 }, { "epoch": 0.3995184711458882, "grad_norm": 12.226061090378838, "learning_rate": 3.753978508177661e-07, "logits/chosen": -2.5625, "logits/rejected": -2.578125, "logps/chosen": -486.0, "logps/rejected": -800.0, "loss": 0.2307, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -2.9375, "rewards/margins": 3.21875, "rewards/rejected": -6.15625, "step": 5310 }, { "epoch": 0.4002708599804379, "grad_norm": 8.36327134314706, "learning_rate": 3.7482936307835737e-07, "logits/chosen": -2.671875, "logits/rejected": -2.765625, "logps/chosen": -468.0, "logps/rejected": -748.0, "loss": 0.2175, "rewards/accuracies": 0.90625, "rewards/chosen": -2.71875, "rewards/margins": 3.03125, "rewards/rejected": -5.75, "step": 5320 }, { "epoch": 0.4010232488149876, "grad_norm": 12.373860217701816, "learning_rate": 3.7426001418406356e-07, "logits/chosen": -2.6875, "logits/rejected": -2.46875, "logps/chosen": -468.0, "logps/rejected": -804.0, "loss": 0.2307, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.78125, "rewards/margins": 3.484375, "rewards/rejected": -6.28125, "step": 5330 }, { "epoch": 0.4017756376495373, "grad_norm": 12.572283921964058, "learning_rate": 3.7368980806262707e-07, "logits/chosen": -2.75, "logits/rejected": -2.78125, "logps/chosen": -478.0, "logps/rejected": -800.0, "loss": 0.2194, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -3.0625, "rewards/margins": 3.328125, "rewards/rejected": -6.375, "step": 5340 }, { "epoch": 0.402528026484087, "grad_norm": 11.113138757060403, "learning_rate": 3.73118748647704e-07, "logits/chosen": -2.734375, "logits/rejected": -2.765625, "logps/chosen": -510.0, "logps/rejected": -796.0, "loss": 0.2452, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.234375, "rewards/margins": 3.0, "rewards/rejected": -6.21875, "step": 5350 }, { "epoch": 0.4032804153186367, "grad_norm": 13.197826932159511, "learning_rate": 3.725468398788372e-07, "logits/chosen": -2.6875, "logits/rejected": -2.46875, "logps/chosen": -520.0, "logps/rejected": -800.0, "loss": 0.2416, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.34375, "rewards/margins": 2.953125, "rewards/rejected": -6.28125, "step": 5360 }, { "epoch": 0.4040328041531864, "grad_norm": 13.244040924316147, "learning_rate": 3.719740857014284e-07, "logits/chosen": -2.734375, "logits/rejected": -2.625, "logps/chosen": -512.0, "logps/rejected": -816.0, "loss": 0.2392, "rewards/accuracies": 0.90625, "rewards/chosen": -3.171875, "rewards/margins": 3.203125, "rewards/rejected": -6.375, "step": 5370 }, { "epoch": 0.4047851929877361, "grad_norm": 13.936961520811911, "learning_rate": 3.714004900667122e-07, "logits/chosen": -2.59375, "logits/rejected": -2.609375, "logps/chosen": -464.0, "logps/rejected": -760.0, "loss": 0.2394, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -2.8125, "rewards/margins": 3.171875, "rewards/rejected": -5.96875, "step": 5380 }, { "epoch": 0.4055375818222858, "grad_norm": 10.660340488693652, "learning_rate": 3.708260569317277e-07, "logits/chosen": -2.6875, "logits/rejected": -2.484375, "logps/chosen": -470.0, "logps/rejected": -744.0, "loss": 0.2335, "rewards/accuracies": 0.90625, "rewards/chosen": -2.90625, "rewards/margins": 2.875, "rewards/rejected": -5.78125, "step": 5390 }, { "epoch": 0.4062899706568355, "grad_norm": 9.036869390727507, "learning_rate": 3.702507902592917e-07, "logits/chosen": -2.5625, "logits/rejected": -2.53125, "logps/chosen": -458.0, "logps/rejected": -796.0, "loss": 0.2127, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -2.765625, "rewards/margins": 3.453125, "rewards/rejected": -6.21875, "step": 5400 }, { "epoch": 0.4070423594913852, "grad_norm": 12.969290751224294, "learning_rate": 3.696746940179713e-07, "logits/chosen": -2.703125, "logits/rejected": -2.65625, "logps/chosen": -502.0, "logps/rejected": -816.0, "loss": 0.2368, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -3.09375, "rewards/margins": 3.1875, "rewards/rejected": -6.28125, "step": 5410 }, { "epoch": 0.4077947483259348, "grad_norm": 12.290955599946319, "learning_rate": 3.690977721820565e-07, "logits/chosen": -2.5, "logits/rejected": -2.5, "logps/chosen": -504.0, "logps/rejected": -808.0, "loss": 0.2322, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.96875, "rewards/margins": 3.3125, "rewards/rejected": -6.28125, "step": 5420 }, { "epoch": 0.4085471371604845, "grad_norm": 26.080156963401, "learning_rate": 3.6852002873153293e-07, "logits/chosen": -2.609375, "logits/rejected": -2.84375, "logps/chosen": -492.0, "logps/rejected": -772.0, "loss": 0.2221, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.125, "rewards/margins": 2.875, "rewards/rejected": -6.0, "step": 5430 }, { "epoch": 0.4092995259950342, "grad_norm": 12.151863559049241, "learning_rate": 3.679414676520541e-07, "logits/chosen": -2.59375, "logits/rejected": -2.640625, "logps/chosen": -496.0, "logps/rejected": -816.0, "loss": 0.2329, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.03125, "rewards/margins": 3.28125, "rewards/rejected": -6.3125, "step": 5440 }, { "epoch": 0.4100519148295839, "grad_norm": 10.605343011686513, "learning_rate": 3.673620929349141e-07, "logits/chosen": -2.578125, "logits/rejected": -2.4375, "logps/chosen": -466.0, "logps/rejected": -800.0, "loss": 0.2147, "rewards/accuracies": 0.90625, "rewards/chosen": -2.90625, "rewards/margins": 3.390625, "rewards/rejected": -6.3125, "step": 5450 }, { "epoch": 0.4108043036641336, "grad_norm": 14.81219999183674, "learning_rate": 3.6678190857701996e-07, "logits/chosen": -2.65625, "logits/rejected": -2.71875, "logps/chosen": -492.0, "logps/rejected": -784.0, "loss": 0.2468, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.0, "rewards/margins": 3.046875, "rewards/rejected": -6.03125, "step": 5460 }, { "epoch": 0.4115566924986833, "grad_norm": 13.705118271502657, "learning_rate": 3.6620091858086426e-07, "logits/chosen": -2.65625, "logits/rejected": -2.625, "logps/chosen": -470.0, "logps/rejected": -812.0, "loss": 0.2196, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.75, "rewards/margins": 3.375, "rewards/rejected": -6.125, "step": 5470 }, { "epoch": 0.412309081333233, "grad_norm": 12.729187577823916, "learning_rate": 3.6561912695449747e-07, "logits/chosen": -2.6875, "logits/rejected": -2.640625, "logps/chosen": -492.0, "logps/rejected": -804.0, "loss": 0.2441, "rewards/accuracies": 0.90625, "rewards/chosen": -2.953125, "rewards/margins": 3.15625, "rewards/rejected": -6.125, "step": 5480 }, { "epoch": 0.4130614701677827, "grad_norm": 10.082458597297515, "learning_rate": 3.6503653771150003e-07, "logits/chosen": -2.734375, "logits/rejected": -2.6875, "logps/chosen": -480.0, "logps/rejected": -784.0, "loss": 0.2086, "rewards/accuracies": 0.90625, "rewards/chosen": -2.828125, "rewards/margins": 3.171875, "rewards/rejected": -6.0, "step": 5490 }, { "epoch": 0.4138138590023324, "grad_norm": 13.56144804638052, "learning_rate": 3.6445315487095487e-07, "logits/chosen": -2.609375, "logits/rejected": -2.734375, "logps/chosen": -512.0, "logps/rejected": -776.0, "loss": 0.2259, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.234375, "rewards/margins": 2.984375, "rewards/rejected": -6.21875, "step": 5500 }, { "epoch": 0.4145662478368821, "grad_norm": 12.90795872389755, "learning_rate": 3.6386898245741987e-07, "logits/chosen": -2.609375, "logits/rejected": -2.390625, "logps/chosen": -460.0, "logps/rejected": -820.0, "loss": 0.2362, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.78125, "rewards/margins": 3.578125, "rewards/rejected": -6.375, "step": 5510 }, { "epoch": 0.4153186366714318, "grad_norm": 13.630745572040611, "learning_rate": 3.632840245008998e-07, "logits/chosen": -2.640625, "logits/rejected": -2.671875, "logps/chosen": -474.0, "logps/rejected": -788.0, "loss": 0.2111, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.890625, "rewards/margins": 3.171875, "rewards/rejected": -6.0625, "step": 5520 }, { "epoch": 0.4160710255059815, "grad_norm": 15.007341789697552, "learning_rate": 3.626982850368185e-07, "logits/chosen": -2.703125, "logits/rejected": -2.40625, "logps/chosen": -502.0, "logps/rejected": -816.0, "loss": 0.2208, "rewards/accuracies": 0.9375, "rewards/chosen": -3.21875, "rewards/margins": 3.15625, "rewards/rejected": -6.375, "step": 5530 }, { "epoch": 0.41682341434053116, "grad_norm": 11.53448448575468, "learning_rate": 3.621117681059914e-07, "logits/chosen": -2.65625, "logits/rejected": -2.46875, "logps/chosen": -528.0, "logps/rejected": -896.0, "loss": 0.2165, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.328125, "rewards/margins": 3.75, "rewards/rejected": -7.0625, "step": 5540 }, { "epoch": 0.41757580317508086, "grad_norm": 10.572092427936775, "learning_rate": 3.6152447775459715e-07, "logits/chosen": -2.75, "logits/rejected": -2.796875, "logps/chosen": -490.0, "logps/rejected": -772.0, "loss": 0.2281, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.125, "rewards/margins": 3.0625, "rewards/rejected": -6.1875, "step": 5550 }, { "epoch": 0.41832819200963056, "grad_norm": 10.143929195717025, "learning_rate": 3.609364180341503e-07, "logits/chosen": -2.609375, "logits/rejected": -2.734375, "logps/chosen": -476.0, "logps/rejected": -772.0, "loss": 0.2133, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -2.828125, "rewards/margins": 3.109375, "rewards/rejected": -5.9375, "step": 5560 }, { "epoch": 0.41908058084418026, "grad_norm": 12.005131327993782, "learning_rate": 3.603475930014726e-07, "logits/chosen": -2.53125, "logits/rejected": -2.59375, "logps/chosen": -506.0, "logps/rejected": -792.0, "loss": 0.2246, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.0625, "rewards/margins": 3.15625, "rewards/rejected": -6.21875, "step": 5570 }, { "epoch": 0.41983296967872996, "grad_norm": 18.495106648920054, "learning_rate": 3.5975800671866575e-07, "logits/chosen": -2.640625, "logits/rejected": -2.734375, "logps/chosen": -520.0, "logps/rejected": -852.0, "loss": 0.2345, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.28125, "rewards/margins": 3.4375, "rewards/rejected": -6.71875, "step": 5580 }, { "epoch": 0.42058535851327966, "grad_norm": 12.611697832119427, "learning_rate": 3.591676632530829e-07, "logits/chosen": -2.671875, "logits/rejected": -2.6875, "logps/chosen": -478.0, "logps/rejected": -812.0, "loss": 0.2129, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.921875, "rewards/margins": 3.546875, "rewards/rejected": -6.46875, "step": 5590 }, { "epoch": 0.42133774734782936, "grad_norm": 13.11447327706276, "learning_rate": 3.5857656667730074e-07, "logits/chosen": -2.5, "logits/rejected": -2.40625, "logps/chosen": -478.0, "logps/rejected": -816.0, "loss": 0.2109, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -2.796875, "rewards/margins": 3.515625, "rewards/rejected": -6.3125, "step": 5600 }, { "epoch": 0.42209013618237906, "grad_norm": 12.844766842759714, "learning_rate": 3.579847210690915e-07, "logits/chosen": -2.6875, "logits/rejected": -2.703125, "logps/chosen": -548.0, "logps/rejected": -840.0, "loss": 0.2259, "rewards/accuracies": 0.90625, "rewards/chosen": -3.421875, "rewards/margins": 3.265625, "rewards/rejected": -6.6875, "step": 5610 }, { "epoch": 0.42284252501692876, "grad_norm": 11.24935134645234, "learning_rate": 3.573921305113947e-07, "logits/chosen": -2.734375, "logits/rejected": -2.75, "logps/chosen": -496.0, "logps/rejected": -816.0, "loss": 0.208, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.1875, "rewards/margins": 3.296875, "rewards/rejected": -6.5, "step": 5620 }, { "epoch": 0.42359491385147846, "grad_norm": 12.835294988479383, "learning_rate": 3.5679879909228895e-07, "logits/chosen": -2.78125, "logits/rejected": -2.796875, "logps/chosen": -490.0, "logps/rejected": -780.0, "loss": 0.2581, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -3.0625, "rewards/margins": 3.015625, "rewards/rejected": -6.09375, "step": 5630 }, { "epoch": 0.42434730268602816, "grad_norm": 14.251601397370445, "learning_rate": 3.562047309049638e-07, "logits/chosen": -2.625, "logits/rejected": -2.6875, "logps/chosen": -480.0, "logps/rejected": -804.0, "loss": 0.2348, "rewards/accuracies": 0.90625, "rewards/chosen": -3.0, "rewards/margins": 3.375, "rewards/rejected": -6.375, "step": 5640 }, { "epoch": 0.42509969152057786, "grad_norm": 9.433203035403032, "learning_rate": 3.5560993004769156e-07, "logits/chosen": -2.65625, "logits/rejected": -2.515625, "logps/chosen": -502.0, "logps/rejected": -800.0, "loss": 0.2429, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.15625, "rewards/margins": 3.078125, "rewards/rejected": -6.25, "step": 5650 }, { "epoch": 0.4258520803551275, "grad_norm": 16.76020161431076, "learning_rate": 3.550144006237991e-07, "logits/chosen": -2.703125, "logits/rejected": -2.515625, "logps/chosen": -506.0, "logps/rejected": -784.0, "loss": 0.2337, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.125, "rewards/margins": 2.90625, "rewards/rejected": -6.03125, "step": 5660 }, { "epoch": 0.4266044691896772, "grad_norm": 13.90002969920596, "learning_rate": 3.5441814674163914e-07, "logits/chosen": -2.609375, "logits/rejected": -2.65625, "logps/chosen": -494.0, "logps/rejected": -792.0, "loss": 0.2071, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.875, "rewards/margins": 3.234375, "rewards/rejected": -6.125, "step": 5670 }, { "epoch": 0.4273568580242269, "grad_norm": 9.562718051581403, "learning_rate": 3.5382117251456245e-07, "logits/chosen": -2.671875, "logits/rejected": -2.546875, "logps/chosen": -516.0, "logps/rejected": -824.0, "loss": 0.2094, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.203125, "rewards/margins": 3.296875, "rewards/rejected": -6.5, "step": 5680 }, { "epoch": 0.4281092468587766, "grad_norm": 12.678071395055253, "learning_rate": 3.5322348206088903e-07, "logits/chosen": -2.78125, "logits/rejected": -2.765625, "logps/chosen": -468.0, "logps/rejected": -856.0, "loss": 0.1988, "rewards/accuracies": 0.9375, "rewards/chosen": -2.828125, "rewards/margins": 3.71875, "rewards/rejected": -6.53125, "step": 5690 }, { "epoch": 0.4288616356933263, "grad_norm": 9.447255777618386, "learning_rate": 3.526250795038801e-07, "logits/chosen": -2.59375, "logits/rejected": -2.625, "logps/chosen": -482.0, "logps/rejected": -808.0, "loss": 0.2203, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.015625, "rewards/margins": 3.328125, "rewards/rejected": -6.34375, "step": 5700 }, { "epoch": 0.429614024527876, "grad_norm": 19.18933483787395, "learning_rate": 3.520259689717091e-07, "logits/chosen": -2.875, "logits/rejected": -2.90625, "logps/chosen": -528.0, "logps/rejected": -856.0, "loss": 0.2394, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -3.46875, "rewards/margins": 3.28125, "rewards/rejected": -6.75, "step": 5710 }, { "epoch": 0.4303664133624257, "grad_norm": 16.603393523341584, "learning_rate": 3.514261545974338e-07, "logits/chosen": -2.5625, "logits/rejected": -2.53125, "logps/chosen": -528.0, "logps/rejected": -860.0, "loss": 0.2352, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.078125, "rewards/margins": 3.6875, "rewards/rejected": -6.75, "step": 5720 }, { "epoch": 0.4311188021969754, "grad_norm": 12.269423469709606, "learning_rate": 3.508256405189675e-07, "logits/chosen": -2.671875, "logits/rejected": -2.875, "logps/chosen": -496.0, "logps/rejected": -740.0, "loss": 0.2309, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.9375, "rewards/margins": 2.84375, "rewards/rejected": -5.78125, "step": 5730 }, { "epoch": 0.4318711910315251, "grad_norm": 12.5238734341765, "learning_rate": 3.502244308790506e-07, "logits/chosen": -2.65625, "logits/rejected": -2.546875, "logps/chosen": -482.0, "logps/rejected": -792.0, "loss": 0.2183, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.84375, "rewards/margins": 3.265625, "rewards/rejected": -6.09375, "step": 5740 }, { "epoch": 0.4326235798660748, "grad_norm": 11.357261499176934, "learning_rate": 3.496225298252216e-07, "logits/chosen": -2.71875, "logits/rejected": -2.671875, "logps/chosen": -510.0, "logps/rejected": -792.0, "loss": 0.2375, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.203125, "rewards/margins": 2.984375, "rewards/rejected": -6.1875, "step": 5750 }, { "epoch": 0.4333759687006245, "grad_norm": 11.359098894553041, "learning_rate": 3.490199415097892e-07, "logits/chosen": -2.75, "logits/rejected": -2.65625, "logps/chosen": -506.0, "logps/rejected": -816.0, "loss": 0.2217, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.375, "rewards/margins": 3.125, "rewards/rejected": -6.5, "step": 5760 }, { "epoch": 0.4341283575351742, "grad_norm": 13.2825894464909, "learning_rate": 3.4841667008980316e-07, "logits/chosen": -2.71875, "logits/rejected": -2.609375, "logps/chosen": -500.0, "logps/rejected": -872.0, "loss": 0.2118, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.21875, "rewards/margins": 3.828125, "rewards/rejected": -7.0625, "step": 5770 }, { "epoch": 0.43488074636972385, "grad_norm": 8.538860424010183, "learning_rate": 3.478127197270257e-07, "logits/chosen": -2.6875, "logits/rejected": -2.53125, "logps/chosen": -506.0, "logps/rejected": -868.0, "loss": 0.207, "rewards/accuracies": 0.9375, "rewards/chosen": -3.28125, "rewards/margins": 3.640625, "rewards/rejected": -6.90625, "step": 5780 }, { "epoch": 0.43563313520427355, "grad_norm": 10.29013564496307, "learning_rate": 3.4720809458790277e-07, "logits/chosen": -2.6875, "logits/rejected": -2.515625, "logps/chosen": -502.0, "logps/rejected": -900.0, "loss": 0.2402, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.09375, "rewards/margins": 4.03125, "rewards/rejected": -7.09375, "step": 5790 }, { "epoch": 0.43638552403882325, "grad_norm": 9.649929425542561, "learning_rate": 3.466027988435356e-07, "logits/chosen": -2.875, "logits/rejected": -2.59375, "logps/chosen": -462.0, "logps/rejected": -848.0, "loss": 0.2124, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -2.78125, "rewards/margins": 3.765625, "rewards/rejected": -6.5625, "step": 5800 }, { "epoch": 0.43713791287337295, "grad_norm": 12.046351127230505, "learning_rate": 3.459968366696515e-07, "logits/chosen": -2.703125, "logits/rejected": -2.78125, "logps/chosen": -452.0, "logps/rejected": -800.0, "loss": 0.2174, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.609375, "rewards/margins": 3.671875, "rewards/rejected": -6.28125, "step": 5810 }, { "epoch": 0.43789030170792265, "grad_norm": 12.09299913268705, "learning_rate": 3.453902122465753e-07, "logits/chosen": -2.8125, "logits/rejected": -2.8125, "logps/chosen": -510.0, "logps/rejected": -832.0, "loss": 0.2483, "rewards/accuracies": 0.875, "rewards/chosen": -3.28125, "rewards/margins": 3.40625, "rewards/rejected": -6.6875, "step": 5820 }, { "epoch": 0.43864269054247235, "grad_norm": 10.100609551752274, "learning_rate": 3.4478292975920043e-07, "logits/chosen": -2.6875, "logits/rejected": -2.640625, "logps/chosen": -478.0, "logps/rejected": -772.0, "loss": 0.2195, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -2.96875, "rewards/margins": 3.140625, "rewards/rejected": -6.09375, "step": 5830 }, { "epoch": 0.43939507937702205, "grad_norm": 11.553479831356274, "learning_rate": 3.441749933969601e-07, "logits/chosen": -2.5625, "logits/rejected": -2.65625, "logps/chosen": -516.0, "logps/rejected": -784.0, "loss": 0.2122, "rewards/accuracies": 0.9375, "rewards/chosen": -3.15625, "rewards/margins": 3.046875, "rewards/rejected": -6.1875, "step": 5840 }, { "epoch": 0.44014746821157175, "grad_norm": 8.825301572948195, "learning_rate": 3.435664073537985e-07, "logits/chosen": -2.6875, "logits/rejected": -2.703125, "logps/chosen": -528.0, "logps/rejected": -804.0, "loss": 0.2226, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.359375, "rewards/margins": 2.96875, "rewards/rejected": -6.3125, "step": 5850 }, { "epoch": 0.44089985704612145, "grad_norm": 13.545540673720017, "learning_rate": 3.4295717582814143e-07, "logits/chosen": -2.484375, "logits/rejected": -2.484375, "logps/chosen": -508.0, "logps/rejected": -780.0, "loss": 0.2305, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -3.140625, "rewards/margins": 2.890625, "rewards/rejected": -6.03125, "step": 5860 }, { "epoch": 0.44165224588067115, "grad_norm": 13.272379332685626, "learning_rate": 3.4234730302286814e-07, "logits/chosen": -2.65625, "logits/rejected": -2.71875, "logps/chosen": -490.0, "logps/rejected": -788.0, "loss": 0.2024, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.0625, "rewards/margins": 3.171875, "rewards/rejected": -6.25, "step": 5870 }, { "epoch": 0.44240463471522085, "grad_norm": 11.388798737252163, "learning_rate": 3.417367931452813e-07, "logits/chosen": -2.484375, "logits/rejected": -2.28125, "logps/chosen": -536.0, "logps/rejected": -916.0, "loss": 0.219, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.359375, "rewards/margins": 3.8125, "rewards/rejected": -7.15625, "step": 5880 }, { "epoch": 0.44315702354977055, "grad_norm": 12.830124879423986, "learning_rate": 3.4112565040707893e-07, "logits/chosen": -2.71875, "logits/rejected": -2.59375, "logps/chosen": -512.0, "logps/rejected": -788.0, "loss": 0.2377, "rewards/accuracies": 0.8125, "rewards/chosen": -3.390625, "rewards/margins": 2.8125, "rewards/rejected": -6.21875, "step": 5890 }, { "epoch": 0.4439094123843202, "grad_norm": 9.797803786566087, "learning_rate": 3.405138790243248e-07, "logits/chosen": -2.515625, "logits/rejected": -2.546875, "logps/chosen": -508.0, "logps/rejected": -796.0, "loss": 0.2252, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.078125, "rewards/margins": 3.09375, "rewards/rejected": -6.15625, "step": 5900 }, { "epoch": 0.4446618012188699, "grad_norm": 15.023129404397592, "learning_rate": 3.399014832174194e-07, "logits/chosen": -2.828125, "logits/rejected": -2.546875, "logps/chosen": -494.0, "logps/rejected": -816.0, "loss": 0.2183, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.140625, "rewards/margins": 3.296875, "rewards/rejected": -6.4375, "step": 5910 }, { "epoch": 0.4454141900534196, "grad_norm": 16.94013635111925, "learning_rate": 3.3928846721107104e-07, "logits/chosen": -2.71875, "logits/rejected": -2.609375, "logps/chosen": -532.0, "logps/rejected": -876.0, "loss": 0.2202, "rewards/accuracies": 0.90625, "rewards/chosen": -3.375, "rewards/margins": 3.5625, "rewards/rejected": -6.9375, "step": 5920 }, { "epoch": 0.4461665788879693, "grad_norm": 11.534076162868773, "learning_rate": 3.386748352342664e-07, "logits/chosen": -2.671875, "logits/rejected": -2.703125, "logps/chosen": -492.0, "logps/rejected": -788.0, "loss": 0.2148, "rewards/accuracies": 0.90625, "rewards/chosen": -3.0625, "rewards/margins": 3.171875, "rewards/rejected": -6.21875, "step": 5930 }, { "epoch": 0.446918967722519, "grad_norm": 15.176485624549207, "learning_rate": 3.380605915202419e-07, "logits/chosen": -2.703125, "logits/rejected": -2.75, "logps/chosen": -516.0, "logps/rejected": -832.0, "loss": 0.2345, "rewards/accuracies": 0.90625, "rewards/chosen": -3.296875, "rewards/margins": 3.3125, "rewards/rejected": -6.625, "step": 5940 }, { "epoch": 0.4476713565570687, "grad_norm": 11.1260127238389, "learning_rate": 3.3744574030645366e-07, "logits/chosen": -2.5625, "logits/rejected": -2.578125, "logps/chosen": -516.0, "logps/rejected": -772.0, "loss": 0.2408, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -3.234375, "rewards/margins": 2.75, "rewards/rejected": -5.96875, "step": 5950 }, { "epoch": 0.4484237453916184, "grad_norm": 11.403188571071032, "learning_rate": 3.3683028583454896e-07, "logits/chosen": -2.625, "logits/rejected": -2.5625, "logps/chosen": -480.0, "logps/rejected": -784.0, "loss": 0.2249, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.828125, "rewards/margins": 3.140625, "rewards/rejected": -5.96875, "step": 5960 }, { "epoch": 0.4491761342261681, "grad_norm": 12.284363008159993, "learning_rate": 3.3621423235033687e-07, "logits/chosen": -2.640625, "logits/rejected": -2.703125, "logps/chosen": -494.0, "logps/rejected": -796.0, "loss": 0.2286, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.046875, "rewards/margins": 3.203125, "rewards/rejected": -6.25, "step": 5970 }, { "epoch": 0.4499285230607178, "grad_norm": 15.099975033660488, "learning_rate": 3.355975841037585e-07, "logits/chosen": -2.625, "logits/rejected": -2.5625, "logps/chosen": -494.0, "logps/rejected": -800.0, "loss": 0.2274, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.171875, "rewards/margins": 3.0625, "rewards/rejected": -6.25, "step": 5980 }, { "epoch": 0.4506809118952675, "grad_norm": 10.340300051406846, "learning_rate": 3.3498034534885844e-07, "logits/chosen": -2.59375, "logits/rejected": -2.6875, "logps/chosen": -500.0, "logps/rejected": -796.0, "loss": 0.1992, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.1875, "rewards/margins": 3.171875, "rewards/rejected": -6.34375, "step": 5990 }, { "epoch": 0.4514333007298172, "grad_norm": 11.19879008668907, "learning_rate": 3.343625203437547e-07, "logits/chosen": -2.75, "logits/rejected": -2.640625, "logps/chosen": -496.0, "logps/rejected": -808.0, "loss": 0.2121, "rewards/accuracies": 0.90625, "rewards/chosen": -3.171875, "rewards/margins": 3.296875, "rewards/rejected": -6.46875, "step": 6000 }, { "epoch": 0.4521856895643669, "grad_norm": 10.867336606891005, "learning_rate": 3.337441133506096e-07, "logits/chosen": -2.734375, "logits/rejected": -2.703125, "logps/chosen": -510.0, "logps/rejected": -824.0, "loss": 0.2389, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.265625, "rewards/margins": 3.25, "rewards/rejected": -6.5, "step": 6010 }, { "epoch": 0.45293807839891653, "grad_norm": 14.432572023827102, "learning_rate": 3.3312512863560053e-07, "logits/chosen": -2.59375, "logits/rejected": -2.6875, "logps/chosen": -476.0, "logps/rejected": -780.0, "loss": 0.2257, "rewards/accuracies": 0.875, "rewards/chosen": -3.03125, "rewards/margins": 3.140625, "rewards/rejected": -6.15625, "step": 6020 }, { "epoch": 0.45369046723346623, "grad_norm": 7.50385454184873, "learning_rate": 3.325055704688906e-07, "logits/chosen": -2.640625, "logits/rejected": -2.671875, "logps/chosen": -456.0, "logps/rejected": -740.0, "loss": 0.2129, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.75, "rewards/margins": 3.0625, "rewards/rejected": -5.8125, "step": 6030 }, { "epoch": 0.45444285606801593, "grad_norm": 12.459022713096395, "learning_rate": 3.318854431245984e-07, "logits/chosen": -2.65625, "logits/rejected": -2.515625, "logps/chosen": -472.0, "logps/rejected": -804.0, "loss": 0.2086, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -2.875, "rewards/margins": 3.5625, "rewards/rejected": -6.4375, "step": 6040 }, { "epoch": 0.45519524490256563, "grad_norm": 13.172996096265305, "learning_rate": 3.312647508807696e-07, "logits/chosen": -2.59375, "logits/rejected": -2.640625, "logps/chosen": -520.0, "logps/rejected": -828.0, "loss": 0.2342, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.328125, "rewards/margins": 3.09375, "rewards/rejected": -6.4375, "step": 6050 }, { "epoch": 0.45594763373711533, "grad_norm": 13.914971708689533, "learning_rate": 3.3064349801934664e-07, "logits/chosen": -2.75, "logits/rejected": -2.5, "logps/chosen": -484.0, "logps/rejected": -832.0, "loss": 0.1923, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -2.9375, "rewards/margins": 3.578125, "rewards/rejected": -6.5, "step": 6060 }, { "epoch": 0.45670002257166503, "grad_norm": 8.374247592527045, "learning_rate": 3.300216888261396e-07, "logits/chosen": -2.703125, "logits/rejected": -2.703125, "logps/chosen": -540.0, "logps/rejected": -868.0, "loss": 0.212, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.34375, "rewards/margins": 3.453125, "rewards/rejected": -6.8125, "step": 6070 }, { "epoch": 0.45745241140621473, "grad_norm": 11.982044300178051, "learning_rate": 3.2939932759079655e-07, "logits/chosen": -2.671875, "logits/rejected": -2.734375, "logps/chosen": -510.0, "logps/rejected": -816.0, "loss": 0.2233, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.125, "rewards/margins": 3.28125, "rewards/rejected": -6.40625, "step": 6080 }, { "epoch": 0.45820480024076443, "grad_norm": 10.152962356413779, "learning_rate": 3.2877641860677357e-07, "logits/chosen": -2.796875, "logits/rejected": -2.671875, "logps/chosen": -500.0, "logps/rejected": -852.0, "loss": 0.2055, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.453125, "rewards/margins": 3.25, "rewards/rejected": -6.6875, "step": 6090 }, { "epoch": 0.45895718907531413, "grad_norm": 12.41380170447185, "learning_rate": 3.2815296617130585e-07, "logits/chosen": -2.703125, "logits/rejected": -2.421875, "logps/chosen": -528.0, "logps/rejected": -836.0, "loss": 0.2258, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.40625, "rewards/margins": 3.09375, "rewards/rejected": -6.5, "step": 6100 }, { "epoch": 0.45970957790986383, "grad_norm": 12.47730585580999, "learning_rate": 3.275289745853775e-07, "logits/chosen": -2.734375, "logits/rejected": -2.734375, "logps/chosen": -510.0, "logps/rejected": -828.0, "loss": 0.2083, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.09375, "rewards/margins": 3.390625, "rewards/rejected": -6.5, "step": 6110 }, { "epoch": 0.46046196674441353, "grad_norm": 7.770042867915527, "learning_rate": 3.269044481536921e-07, "logits/chosen": -2.78125, "logits/rejected": -2.9375, "logps/chosen": -502.0, "logps/rejected": -836.0, "loss": 0.2348, "rewards/accuracies": 0.90625, "rewards/chosen": -3.15625, "rewards/margins": 3.5, "rewards/rejected": -6.65625, "step": 6120 }, { "epoch": 0.46121435557896323, "grad_norm": 14.207089454991694, "learning_rate": 3.2627939118464296e-07, "logits/chosen": -2.71875, "logits/rejected": -2.890625, "logps/chosen": -470.0, "logps/rejected": -784.0, "loss": 0.2168, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -2.90625, "rewards/margins": 3.21875, "rewards/rejected": -6.125, "step": 6130 }, { "epoch": 0.46196674441351293, "grad_norm": 9.086015402781731, "learning_rate": 3.256538079902833e-07, "logits/chosen": -2.6875, "logits/rejected": -2.515625, "logps/chosen": -512.0, "logps/rejected": -844.0, "loss": 0.2342, "rewards/accuracies": 0.90625, "rewards/chosen": -3.25, "rewards/margins": 3.453125, "rewards/rejected": -6.6875, "step": 6140 }, { "epoch": 0.4627191332480626, "grad_norm": 13.351840651864237, "learning_rate": 3.2502770288629655e-07, "logits/chosen": -2.78125, "logits/rejected": -2.625, "logps/chosen": -506.0, "logps/rejected": -812.0, "loss": 0.2205, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.0625, "rewards/margins": 3.234375, "rewards/rejected": -6.3125, "step": 6150 }, { "epoch": 0.4634715220826123, "grad_norm": 12.282210072685414, "learning_rate": 3.244010801919668e-07, "logits/chosen": -2.828125, "logits/rejected": -2.609375, "logps/chosen": -512.0, "logps/rejected": -812.0, "loss": 0.2115, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.203125, "rewards/margins": 3.28125, "rewards/rejected": -6.5, "step": 6160 }, { "epoch": 0.464223910917162, "grad_norm": 13.21591491499129, "learning_rate": 3.237739442301487e-07, "logits/chosen": -2.796875, "logits/rejected": -2.765625, "logps/chosen": -516.0, "logps/rejected": -892.0, "loss": 0.2086, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.265625, "rewards/margins": 3.96875, "rewards/rejected": -7.25, "step": 6170 }, { "epoch": 0.4649762997517117, "grad_norm": 10.778082361585705, "learning_rate": 3.231462993272377e-07, "logits/chosen": -2.75, "logits/rejected": -2.734375, "logps/chosen": -544.0, "logps/rejected": -880.0, "loss": 0.2188, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.53125, "rewards/margins": 3.421875, "rewards/rejected": -6.96875, "step": 6180 }, { "epoch": 0.4657286885862614, "grad_norm": 13.177076178212616, "learning_rate": 3.225181498131404e-07, "logits/chosen": -2.65625, "logits/rejected": -2.453125, "logps/chosen": -502.0, "logps/rejected": -896.0, "loss": 0.1933, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.15625, "rewards/margins": 3.984375, "rewards/rejected": -7.125, "step": 6190 }, { "epoch": 0.4664810774208111, "grad_norm": 16.176099946608478, "learning_rate": 3.218895000212445e-07, "logits/chosen": -2.453125, "logits/rejected": -2.46875, "logps/chosen": -482.0, "logps/rejected": -816.0, "loss": 0.2369, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -2.953125, "rewards/margins": 3.28125, "rewards/rejected": -6.25, "step": 6200 }, { "epoch": 0.4672334662553608, "grad_norm": 8.698871229514632, "learning_rate": 3.2126035428838874e-07, "logits/chosen": -2.59375, "logits/rejected": -2.65625, "logps/chosen": -532.0, "logps/rejected": -812.0, "loss": 0.2144, "rewards/accuracies": 0.875, "rewards/chosen": -3.1875, "rewards/margins": 3.0, "rewards/rejected": -6.1875, "step": 6210 }, { "epoch": 0.4679858550899105, "grad_norm": 16.191235093963822, "learning_rate": 3.2063071695483354e-07, "logits/chosen": -2.765625, "logits/rejected": -2.71875, "logps/chosen": -510.0, "logps/rejected": -824.0, "loss": 0.2165, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -3.328125, "rewards/margins": 3.046875, "rewards/rejected": -6.375, "step": 6220 }, { "epoch": 0.4687382439244602, "grad_norm": 11.08419422121178, "learning_rate": 3.200005923642305e-07, "logits/chosen": -2.65625, "logits/rejected": -2.65625, "logps/chosen": -496.0, "logps/rejected": -840.0, "loss": 0.2266, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.125, "rewards/margins": 3.421875, "rewards/rejected": -6.5625, "step": 6230 }, { "epoch": 0.4694906327590099, "grad_norm": 13.166991037571217, "learning_rate": 3.193699848635925e-07, "logits/chosen": -2.59375, "logits/rejected": -2.546875, "logps/chosen": -512.0, "logps/rejected": -824.0, "loss": 0.2021, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.21875, "rewards/margins": 3.234375, "rewards/rejected": -6.46875, "step": 6240 }, { "epoch": 0.4702430215935596, "grad_norm": 11.73042815146686, "learning_rate": 3.1873889880326425e-07, "logits/chosen": -2.53125, "logits/rejected": -2.65625, "logps/chosen": -486.0, "logps/rejected": -760.0, "loss": 0.2315, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.03125, "rewards/margins": 2.890625, "rewards/rejected": -5.9375, "step": 6250 }, { "epoch": 0.4709954104281093, "grad_norm": 10.316245437578518, "learning_rate": 3.1810733853689153e-07, "logits/chosen": -2.71875, "logits/rejected": -2.453125, "logps/chosen": -528.0, "logps/rejected": -836.0, "loss": 0.2241, "rewards/accuracies": 0.90625, "rewards/chosen": -3.09375, "rewards/margins": 3.15625, "rewards/rejected": -6.25, "step": 6260 }, { "epoch": 0.4717477992626589, "grad_norm": 15.634378016325218, "learning_rate": 3.1747530842139157e-07, "logits/chosen": -2.78125, "logits/rejected": -2.59375, "logps/chosen": -506.0, "logps/rejected": -840.0, "loss": 0.2456, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.21875, "rewards/margins": 3.375, "rewards/rejected": -6.59375, "step": 6270 }, { "epoch": 0.4725001880972086, "grad_norm": 11.310463960803373, "learning_rate": 3.1684281281692305e-07, "logits/chosen": -2.78125, "logits/rejected": -2.609375, "logps/chosen": -506.0, "logps/rejected": -808.0, "loss": 0.2348, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.25, "rewards/margins": 3.078125, "rewards/rejected": -6.3125, "step": 6280 }, { "epoch": 0.4732525769317583, "grad_norm": 13.9020705727266, "learning_rate": 3.162098560868558e-07, "logits/chosen": -2.5, "logits/rejected": -2.75, "logps/chosen": -488.0, "logps/rejected": -760.0, "loss": 0.1974, "rewards/accuracies": 0.90625, "rewards/chosen": -3.015625, "rewards/margins": 2.921875, "rewards/rejected": -5.9375, "step": 6290 }, { "epoch": 0.474004965766308, "grad_norm": 14.821562019175834, "learning_rate": 3.155764425977408e-07, "logits/chosen": -2.796875, "logits/rejected": -2.90625, "logps/chosen": -528.0, "logps/rejected": -808.0, "loss": 0.1965, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.328125, "rewards/margins": 3.21875, "rewards/rejected": -6.53125, "step": 6300 }, { "epoch": 0.4747573546008577, "grad_norm": 11.788764515923882, "learning_rate": 3.1494257671928006e-07, "logits/chosen": -2.671875, "logits/rejected": -2.953125, "logps/chosen": -548.0, "logps/rejected": -820.0, "loss": 0.232, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.703125, "rewards/margins": 2.90625, "rewards/rejected": -6.59375, "step": 6310 }, { "epoch": 0.4755097434354074, "grad_norm": 14.52534973914942, "learning_rate": 3.1430826282429653e-07, "logits/chosen": -2.734375, "logits/rejected": -2.984375, "logps/chosen": -520.0, "logps/rejected": -824.0, "loss": 0.2062, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.34375, "rewards/margins": 3.265625, "rewards/rejected": -6.625, "step": 6320 }, { "epoch": 0.4762621322699571, "grad_norm": 11.457702282528732, "learning_rate": 3.136735052887038e-07, "logits/chosen": -2.6875, "logits/rejected": -2.71875, "logps/chosen": -524.0, "logps/rejected": -788.0, "loss": 0.2264, "rewards/accuracies": 0.90625, "rewards/chosen": -3.203125, "rewards/margins": 2.984375, "rewards/rejected": -6.1875, "step": 6330 }, { "epoch": 0.4770145211045068, "grad_norm": 9.535103242797428, "learning_rate": 3.1303830849147594e-07, "logits/chosen": -2.765625, "logits/rejected": -2.765625, "logps/chosen": -498.0, "logps/rejected": -752.0, "loss": 0.2298, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -3.03125, "rewards/margins": 2.8125, "rewards/rejected": -5.84375, "step": 6340 }, { "epoch": 0.4777669099390565, "grad_norm": 12.960297535421116, "learning_rate": 3.124026768146176e-07, "logits/chosen": -2.703125, "logits/rejected": -2.671875, "logps/chosen": -464.0, "logps/rejected": -776.0, "loss": 0.2214, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.78125, "rewards/margins": 3.140625, "rewards/rejected": -5.9375, "step": 6350 }, { "epoch": 0.4785192987736062, "grad_norm": 9.11757489098743, "learning_rate": 3.117666146431331e-07, "logits/chosen": -2.625, "logits/rejected": -2.6875, "logps/chosen": -510.0, "logps/rejected": -796.0, "loss": 0.2255, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.21875, "rewards/margins": 2.96875, "rewards/rejected": -6.1875, "step": 6360 }, { "epoch": 0.4792716876081559, "grad_norm": 11.175667760706547, "learning_rate": 3.111301263649969e-07, "logits/chosen": -2.609375, "logits/rejected": -2.578125, "logps/chosen": -498.0, "logps/rejected": -776.0, "loss": 0.2314, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.15625, "rewards/margins": 2.84375, "rewards/rejected": -6.0, "step": 6370 }, { "epoch": 0.4800240764427056, "grad_norm": 11.74868637492868, "learning_rate": 3.1049321637112303e-07, "logits/chosen": -2.703125, "logits/rejected": -2.65625, "logps/chosen": -472.0, "logps/rejected": -792.0, "loss": 0.2296, "rewards/accuracies": 0.9375, "rewards/chosen": -2.84375, "rewards/margins": 3.3125, "rewards/rejected": -6.15625, "step": 6380 }, { "epoch": 0.48077646527725526, "grad_norm": 12.976127890094924, "learning_rate": 3.0985588905533456e-07, "logits/chosen": -2.59375, "logits/rejected": -2.609375, "logps/chosen": -464.0, "logps/rejected": -764.0, "loss": 0.2071, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.71875, "rewards/margins": 3.265625, "rewards/rejected": -6.0, "step": 6390 }, { "epoch": 0.48152885411180496, "grad_norm": 12.855785443106829, "learning_rate": 3.0921814881433373e-07, "logits/chosen": -2.421875, "logits/rejected": -2.28125, "logps/chosen": -470.0, "logps/rejected": -796.0, "loss": 0.2316, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.828125, "rewards/margins": 3.21875, "rewards/rejected": -6.0625, "step": 6400 }, { "epoch": 0.48228124294635466, "grad_norm": 9.250938406698088, "learning_rate": 3.0858000004767125e-07, "logits/chosen": -2.640625, "logits/rejected": -2.546875, "logps/chosen": -512.0, "logps/rejected": -784.0, "loss": 0.2122, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.203125, "rewards/margins": 2.890625, "rewards/rejected": -6.09375, "step": 6410 }, { "epoch": 0.48303363178090436, "grad_norm": 11.726331836831472, "learning_rate": 3.079414471577163e-07, "logits/chosen": -2.515625, "logits/rejected": -2.53125, "logps/chosen": -496.0, "logps/rejected": -804.0, "loss": 0.2242, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.109375, "rewards/margins": 3.3125, "rewards/rejected": -6.40625, "step": 6420 }, { "epoch": 0.48378602061545406, "grad_norm": 9.881904663043088, "learning_rate": 3.073024945496258e-07, "logits/chosen": -2.5, "logits/rejected": -2.703125, "logps/chosen": -484.0, "logps/rejected": -804.0, "loss": 0.2187, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.828125, "rewards/margins": 3.40625, "rewards/rejected": -6.21875, "step": 6430 }, { "epoch": 0.48453840945000376, "grad_norm": 13.325286307510412, "learning_rate": 3.0666314663131435e-07, "logits/chosen": -2.65625, "logits/rejected": -2.75, "logps/chosen": -508.0, "logps/rejected": -800.0, "loss": 0.2127, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.171875, "rewards/margins": 3.171875, "rewards/rejected": -6.34375, "step": 6440 }, { "epoch": 0.48529079828455346, "grad_norm": 13.27128103817623, "learning_rate": 3.0602340781342343e-07, "logits/chosen": -2.75, "logits/rejected": -2.890625, "logps/chosen": -512.0, "logps/rejected": -796.0, "loss": 0.2159, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.09375, "rewards/margins": 3.15625, "rewards/rejected": -6.25, "step": 6450 }, { "epoch": 0.48604318711910316, "grad_norm": 11.449490973989493, "learning_rate": 3.0538328250929147e-07, "logits/chosen": -2.921875, "logits/rejected": -2.6875, "logps/chosen": -490.0, "logps/rejected": -836.0, "loss": 0.2409, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.15625, "rewards/margins": 3.5625, "rewards/rejected": -6.71875, "step": 6460 }, { "epoch": 0.48679557595365286, "grad_norm": 12.04924763572949, "learning_rate": 3.047427751349228e-07, "logits/chosen": -2.71875, "logits/rejected": -2.671875, "logps/chosen": -524.0, "logps/rejected": -792.0, "loss": 0.2138, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.09375, "rewards/margins": 3.09375, "rewards/rejected": -6.1875, "step": 6470 }, { "epoch": 0.48754796478820256, "grad_norm": 14.827881777140593, "learning_rate": 3.041018901089579e-07, "logits/chosen": -2.859375, "logits/rejected": -2.65625, "logps/chosen": -468.0, "logps/rejected": -776.0, "loss": 0.2097, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.96875, "rewards/margins": 2.953125, "rewards/rejected": -5.9375, "step": 6480 }, { "epoch": 0.48830035362275226, "grad_norm": 10.281291207622756, "learning_rate": 3.034606318526423e-07, "logits/chosen": -2.734375, "logits/rejected": -2.8125, "logps/chosen": -476.0, "logps/rejected": -784.0, "loss": 0.2026, "rewards/accuracies": 0.9375, "rewards/chosen": -2.859375, "rewards/margins": 3.3125, "rewards/rejected": -6.15625, "step": 6490 }, { "epoch": 0.48905274245730196, "grad_norm": 9.444324195959767, "learning_rate": 3.028190047897964e-07, "logits/chosen": -2.8125, "logits/rejected": -2.734375, "logps/chosen": -502.0, "logps/rejected": -844.0, "loss": 0.2245, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.296875, "rewards/margins": 3.5, "rewards/rejected": -6.78125, "step": 6500 }, { "epoch": 0.4898051312918516, "grad_norm": 13.448251054322352, "learning_rate": 3.021770133467848e-07, "logits/chosen": -2.6875, "logits/rejected": -2.578125, "logps/chosen": -524.0, "logps/rejected": -840.0, "loss": 0.2166, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.3125, "rewards/margins": 3.09375, "rewards/rejected": -6.40625, "step": 6510 }, { "epoch": 0.4905575201264013, "grad_norm": 12.429928081559261, "learning_rate": 3.015346619524859e-07, "logits/chosen": -2.59375, "logits/rejected": -2.515625, "logps/chosen": -524.0, "logps/rejected": -832.0, "loss": 0.1956, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.328125, "rewards/margins": 3.25, "rewards/rejected": -6.59375, "step": 6520 }, { "epoch": 0.491309908960951, "grad_norm": 11.887749239590674, "learning_rate": 3.008919550382613e-07, "logits/chosen": -2.734375, "logits/rejected": -2.71875, "logps/chosen": -508.0, "logps/rejected": -828.0, "loss": 0.2303, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.125, "rewards/margins": 3.265625, "rewards/rejected": -6.375, "step": 6530 }, { "epoch": 0.4920622977955007, "grad_norm": 11.570563056140749, "learning_rate": 3.0024889703792513e-07, "logits/chosen": -2.65625, "logits/rejected": -2.59375, "logps/chosen": -520.0, "logps/rejected": -792.0, "loss": 0.2181, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.234375, "rewards/margins": 2.90625, "rewards/rejected": -6.125, "step": 6540 }, { "epoch": 0.4928146866300504, "grad_norm": 14.63840811224487, "learning_rate": 2.996054923877136e-07, "logits/chosen": -2.8125, "logits/rejected": -2.625, "logps/chosen": -506.0, "logps/rejected": -788.0, "loss": 0.2285, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.34375, "rewards/margins": 2.859375, "rewards/rejected": -6.21875, "step": 6550 }, { "epoch": 0.4935670754646001, "grad_norm": 11.673378033333575, "learning_rate": 2.9896174552625427e-07, "logits/chosen": -2.75, "logits/rejected": -2.484375, "logps/chosen": -516.0, "logps/rejected": -880.0, "loss": 0.2077, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.109375, "rewards/margins": 3.84375, "rewards/rejected": -6.9375, "step": 6560 }, { "epoch": 0.4943194642991498, "grad_norm": 10.921304940423882, "learning_rate": 2.983176608945356e-07, "logits/chosen": -2.6875, "logits/rejected": -2.546875, "logps/chosen": -524.0, "logps/rejected": -880.0, "loss": 0.2267, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.40625, "rewards/margins": 3.59375, "rewards/rejected": -7.0, "step": 6570 }, { "epoch": 0.4950718531336995, "grad_norm": 15.562571925813984, "learning_rate": 2.9767324293587617e-07, "logits/chosen": -2.59375, "logits/rejected": -2.4375, "logps/chosen": -510.0, "logps/rejected": -828.0, "loss": 0.2196, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.140625, "rewards/margins": 3.3125, "rewards/rejected": -6.4375, "step": 6580 }, { "epoch": 0.4958242419682492, "grad_norm": 15.9592724320002, "learning_rate": 2.9702849609589403e-07, "logits/chosen": -2.84375, "logits/rejected": -2.640625, "logps/chosen": -504.0, "logps/rejected": -828.0, "loss": 0.2271, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.265625, "rewards/margins": 3.078125, "rewards/rejected": -6.34375, "step": 6590 }, { "epoch": 0.4965766308027989, "grad_norm": 12.154841187349797, "learning_rate": 2.96383424822476e-07, "logits/chosen": -2.625, "logits/rejected": -2.484375, "logps/chosen": -490.0, "logps/rejected": -776.0, "loss": 0.2151, "rewards/accuracies": 0.90625, "rewards/chosen": -2.953125, "rewards/margins": 3.171875, "rewards/rejected": -6.125, "step": 6600 }, { "epoch": 0.4973290196373486, "grad_norm": 14.587778938362858, "learning_rate": 2.957380335657473e-07, "logits/chosen": -2.609375, "logits/rejected": -2.59375, "logps/chosen": -496.0, "logps/rejected": -848.0, "loss": 0.2157, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.125, "rewards/margins": 3.578125, "rewards/rejected": -6.6875, "step": 6610 }, { "epoch": 0.4980814084718983, "grad_norm": 11.773458826296299, "learning_rate": 2.950923267780405e-07, "logits/chosen": -2.796875, "logits/rejected": -2.53125, "logps/chosen": -510.0, "logps/rejected": -792.0, "loss": 0.2182, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.25, "rewards/margins": 2.953125, "rewards/rejected": -6.21875, "step": 6620 }, { "epoch": 0.49883379730644795, "grad_norm": 12.55637501977933, "learning_rate": 2.944463089138648e-07, "logits/chosen": -2.703125, "logits/rejected": -2.625, "logps/chosen": -476.0, "logps/rejected": -784.0, "loss": 0.2235, "rewards/accuracies": 0.9375, "rewards/chosen": -2.90625, "rewards/margins": 3.265625, "rewards/rejected": -6.15625, "step": 6630 }, { "epoch": 0.49958618614099765, "grad_norm": 11.002392611973209, "learning_rate": 2.937999844298753e-07, "logits/chosen": -2.828125, "logits/rejected": -2.84375, "logps/chosen": -464.0, "logps/rejected": -772.0, "loss": 0.2144, "rewards/accuracies": 0.9375, "rewards/chosen": -2.90625, "rewards/margins": 3.234375, "rewards/rejected": -6.15625, "step": 6640 }, { "epoch": 0.5003385749755473, "grad_norm": 18.722291877914323, "learning_rate": 2.931533577848428e-07, "logits/chosen": -2.75, "logits/rejected": -2.6875, "logps/chosen": -540.0, "logps/rejected": -872.0, "loss": 0.2103, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.390625, "rewards/margins": 3.625, "rewards/rejected": -7.0, "step": 6650 }, { "epoch": 0.5010909638100971, "grad_norm": 11.499324539492386, "learning_rate": 2.9250643343962216e-07, "logits/chosen": -2.578125, "logits/rejected": -2.484375, "logps/chosen": -536.0, "logps/rejected": -852.0, "loss": 0.2058, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.359375, "rewards/margins": 3.34375, "rewards/rejected": -6.6875, "step": 6660 }, { "epoch": 0.5018433526446467, "grad_norm": 15.513440720344319, "learning_rate": 2.918592158571223e-07, "logits/chosen": -2.765625, "logits/rejected": -2.75, "logps/chosen": -496.0, "logps/rejected": -832.0, "loss": 0.2162, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.140625, "rewards/margins": 3.390625, "rewards/rejected": -6.53125, "step": 6670 }, { "epoch": 0.5025957414791965, "grad_norm": 17.99994916999856, "learning_rate": 2.912117095022749e-07, "logits/chosen": -2.765625, "logits/rejected": -2.640625, "logps/chosen": -520.0, "logps/rejected": -868.0, "loss": 0.2204, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.25, "rewards/margins": 3.515625, "rewards/rejected": -6.75, "step": 6680 }, { "epoch": 0.5033481303137461, "grad_norm": 10.117991296035967, "learning_rate": 2.9056391884200374e-07, "logits/chosen": -2.8125, "logits/rejected": -2.875, "logps/chosen": -486.0, "logps/rejected": -808.0, "loss": 0.2, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -2.984375, "rewards/margins": 3.421875, "rewards/rejected": -6.40625, "step": 6690 }, { "epoch": 0.5041005191482958, "grad_norm": 11.79301932972393, "learning_rate": 2.8991584834519405e-07, "logits/chosen": -2.703125, "logits/rejected": -2.78125, "logps/chosen": -490.0, "logps/rejected": -788.0, "loss": 0.2227, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -2.90625, "rewards/margins": 3.0625, "rewards/rejected": -5.96875, "step": 6700 }, { "epoch": 0.5048529079828455, "grad_norm": 9.784676662657027, "learning_rate": 2.8926750248266165e-07, "logits/chosen": -2.765625, "logits/rejected": -2.75, "logps/chosen": -524.0, "logps/rejected": -844.0, "loss": 0.205, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.1875, "rewards/margins": 3.40625, "rewards/rejected": -6.59375, "step": 6710 }, { "epoch": 0.5056052968173952, "grad_norm": 30.88333630338591, "learning_rate": 2.886188857271217e-07, "logits/chosen": -2.71875, "logits/rejected": -2.6875, "logps/chosen": -504.0, "logps/rejected": -824.0, "loss": 0.2306, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.171875, "rewards/margins": 3.375, "rewards/rejected": -6.53125, "step": 6720 }, { "epoch": 0.5063576856519449, "grad_norm": 13.26241030105115, "learning_rate": 2.879700025531583e-07, "logits/chosen": -2.6875, "logits/rejected": -2.59375, "logps/chosen": -480.0, "logps/rejected": -768.0, "loss": 0.2172, "rewards/accuracies": 0.90625, "rewards/chosen": -3.03125, "rewards/margins": 3.15625, "rewards/rejected": -6.1875, "step": 6730 }, { "epoch": 0.5071100744864946, "grad_norm": 11.26182304283782, "learning_rate": 2.873208574371937e-07, "logits/chosen": -2.625, "logits/rejected": -2.640625, "logps/chosen": -498.0, "logps/rejected": -792.0, "loss": 0.2297, "rewards/accuracies": 0.90625, "rewards/chosen": -3.0625, "rewards/margins": 3.09375, "rewards/rejected": -6.15625, "step": 6740 }, { "epoch": 0.5078624633210443, "grad_norm": 13.592699598326204, "learning_rate": 2.8667145485745684e-07, "logits/chosen": -2.625, "logits/rejected": -2.5, "logps/chosen": -536.0, "logps/rejected": -864.0, "loss": 0.2162, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.203125, "rewards/margins": 3.59375, "rewards/rejected": -6.78125, "step": 6750 }, { "epoch": 0.508614852155594, "grad_norm": 16.244295722395606, "learning_rate": 2.860217992939532e-07, "logits/chosen": -2.609375, "logits/rejected": -2.484375, "logps/chosen": -504.0, "logps/rejected": -816.0, "loss": 0.2198, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.15625, "rewards/margins": 3.203125, "rewards/rejected": -6.375, "step": 6760 }, { "epoch": 0.5093672409901437, "grad_norm": 13.453948677061739, "learning_rate": 2.853718952284331e-07, "logits/chosen": -2.734375, "logits/rejected": -2.53125, "logps/chosen": -528.0, "logps/rejected": -832.0, "loss": 0.2154, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.25, "rewards/margins": 3.34375, "rewards/rejected": -6.59375, "step": 6770 }, { "epoch": 0.5101196298246934, "grad_norm": 11.42810643796632, "learning_rate": 2.8472174714436137e-07, "logits/chosen": -2.6875, "logits/rejected": -2.53125, "logps/chosen": -500.0, "logps/rejected": -848.0, "loss": 0.2079, "rewards/accuracies": 0.9375, "rewards/chosen": -3.21875, "rewards/margins": 3.59375, "rewards/rejected": -6.8125, "step": 6780 }, { "epoch": 0.5108720186592431, "grad_norm": 11.50437768321476, "learning_rate": 2.8407135952688634e-07, "logits/chosen": -2.703125, "logits/rejected": -2.703125, "logps/chosen": -520.0, "logps/rejected": -828.0, "loss": 0.1843, "rewards/accuracies": 0.875, "rewards/chosen": -3.375, "rewards/margins": 3.25, "rewards/rejected": -6.625, "step": 6790 }, { "epoch": 0.5116244074937928, "grad_norm": 12.082948254009233, "learning_rate": 2.834207368628088e-07, "logits/chosen": -2.859375, "logits/rejected": -2.5625, "logps/chosen": -516.0, "logps/rejected": -848.0, "loss": 0.2154, "rewards/accuracies": 0.9375, "rewards/chosen": -3.34375, "rewards/margins": 3.359375, "rewards/rejected": -6.6875, "step": 6800 }, { "epoch": 0.5123767963283424, "grad_norm": 9.774017190407273, "learning_rate": 2.8276988364055076e-07, "logits/chosen": -2.71875, "logits/rejected": -2.609375, "logps/chosen": -510.0, "logps/rejected": -880.0, "loss": 0.1818, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.0625, "rewards/margins": 3.984375, "rewards/rejected": -7.03125, "step": 6810 }, { "epoch": 0.5131291851628922, "grad_norm": 12.283190043909658, "learning_rate": 2.821188043501251e-07, "logits/chosen": -2.96875, "logits/rejected": -2.9375, "logps/chosen": -528.0, "logps/rejected": -844.0, "loss": 0.2156, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.546875, "rewards/margins": 3.265625, "rewards/rejected": -6.8125, "step": 6820 }, { "epoch": 0.5138815739974418, "grad_norm": 13.18184365584575, "learning_rate": 2.8146750348310406e-07, "logits/chosen": -2.6875, "logits/rejected": -2.8125, "logps/chosen": -516.0, "logps/rejected": -840.0, "loss": 0.2327, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.171875, "rewards/margins": 3.4375, "rewards/rejected": -6.59375, "step": 6830 }, { "epoch": 0.5146339628319916, "grad_norm": 11.926991942662301, "learning_rate": 2.8081598553258863e-07, "logits/chosen": -2.78125, "logits/rejected": -2.796875, "logps/chosen": -524.0, "logps/rejected": -844.0, "loss": 0.2045, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.171875, "rewards/margins": 3.390625, "rewards/rejected": -6.5625, "step": 6840 }, { "epoch": 0.5153863516665412, "grad_norm": 12.383046583959175, "learning_rate": 2.801642549931773e-07, "logits/chosen": -2.75, "logits/rejected": -2.671875, "logps/chosen": -524.0, "logps/rejected": -832.0, "loss": 0.1952, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.234375, "rewards/margins": 3.265625, "rewards/rejected": -6.5, "step": 6850 }, { "epoch": 0.516138740501091, "grad_norm": 10.784514669087978, "learning_rate": 2.7951231636093496e-07, "logits/chosen": -2.71875, "logits/rejected": -2.78125, "logps/chosen": -516.0, "logps/rejected": -884.0, "loss": 0.1779, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.171875, "rewards/margins": 3.9375, "rewards/rejected": -7.09375, "step": 6860 }, { "epoch": 0.5168911293356406, "grad_norm": 15.291011895417745, "learning_rate": 2.788601741333623e-07, "logits/chosen": -2.828125, "logits/rejected": -2.84375, "logps/chosen": -536.0, "logps/rejected": -916.0, "loss": 0.1897, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.421875, "rewards/margins": 3.96875, "rewards/rejected": -7.375, "step": 6870 }, { "epoch": 0.5176435181701904, "grad_norm": 9.41629817865714, "learning_rate": 2.782078328093646e-07, "logits/chosen": -2.71875, "logits/rejected": -2.65625, "logps/chosen": -484.0, "logps/rejected": -852.0, "loss": 0.2008, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.0, "rewards/margins": 3.578125, "rewards/rejected": -6.59375, "step": 6880 }, { "epoch": 0.51839590700474, "grad_norm": 13.191321206098243, "learning_rate": 2.775552968892205e-07, "logits/chosen": -2.71875, "logits/rejected": -2.640625, "logps/chosen": -506.0, "logps/rejected": -796.0, "loss": 0.1997, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.21875, "rewards/margins": 3.0625, "rewards/rejected": -6.28125, "step": 6890 }, { "epoch": 0.5191482958392898, "grad_norm": 14.068614000910328, "learning_rate": 2.76902570874551e-07, "logits/chosen": -2.84375, "logits/rejected": -2.6875, "logps/chosen": -508.0, "logps/rejected": -876.0, "loss": 0.2142, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.234375, "rewards/margins": 3.78125, "rewards/rejected": -7.0, "step": 6900 }, { "epoch": 0.5199006846738394, "grad_norm": 13.195987284971265, "learning_rate": 2.7624965926828856e-07, "logits/chosen": -2.71875, "logits/rejected": -2.875, "logps/chosen": -456.0, "logps/rejected": -800.0, "loss": 0.2166, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -2.84375, "rewards/margins": 3.5, "rewards/rejected": -6.34375, "step": 6910 }, { "epoch": 0.5206530735083892, "grad_norm": 14.334970265844388, "learning_rate": 2.7559656657464615e-07, "logits/chosen": -2.78125, "logits/rejected": -2.75, "logps/chosen": -520.0, "logps/rejected": -812.0, "loss": 0.2309, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.203125, "rewards/margins": 3.109375, "rewards/rejected": -6.3125, "step": 6920 }, { "epoch": 0.5214054623429388, "grad_norm": 15.628835672335851, "learning_rate": 2.7494329729908585e-07, "logits/chosen": -2.859375, "logits/rejected": -2.546875, "logps/chosen": -508.0, "logps/rejected": -892.0, "loss": 0.207, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.34375, "rewards/margins": 3.78125, "rewards/rejected": -7.125, "step": 6930 }, { "epoch": 0.5221578511774885, "grad_norm": 13.809076404696354, "learning_rate": 2.7428985594828785e-07, "logits/chosen": -2.65625, "logits/rejected": -2.8125, "logps/chosen": -512.0, "logps/rejected": -880.0, "loss": 0.231, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.109375, "rewards/margins": 3.765625, "rewards/rejected": -6.875, "step": 6940 }, { "epoch": 0.5229102400120382, "grad_norm": 12.66772018086389, "learning_rate": 2.736362470301195e-07, "logits/chosen": -2.734375, "logits/rejected": -2.875, "logps/chosen": -482.0, "logps/rejected": -772.0, "loss": 0.1956, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.140625, "rewards/margins": 3.03125, "rewards/rejected": -6.15625, "step": 6950 }, { "epoch": 0.5236626288465879, "grad_norm": 10.694377106569316, "learning_rate": 2.7298247505360414e-07, "logits/chosen": -2.8125, "logits/rejected": -2.828125, "logps/chosen": -500.0, "logps/rejected": -864.0, "loss": 0.2026, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.046875, "rewards/margins": 3.8125, "rewards/rejected": -6.84375, "step": 6960 }, { "epoch": 0.5244150176811376, "grad_norm": 13.760636549491295, "learning_rate": 2.723285445288902e-07, "logits/chosen": -2.65625, "logits/rejected": -2.828125, "logps/chosen": -486.0, "logps/rejected": -792.0, "loss": 0.2102, "rewards/accuracies": 0.9375, "rewards/chosen": -2.96875, "rewards/margins": 3.265625, "rewards/rejected": -6.25, "step": 6970 }, { "epoch": 0.5251674065156873, "grad_norm": 17.818636230621593, "learning_rate": 2.7167445996721954e-07, "logits/chosen": -2.828125, "logits/rejected": -2.875, "logps/chosen": -502.0, "logps/rejected": -824.0, "loss": 0.2059, "rewards/accuracies": 0.90625, "rewards/chosen": -3.109375, "rewards/margins": 3.359375, "rewards/rejected": -6.46875, "step": 6980 }, { "epoch": 0.525919795350237, "grad_norm": 10.845423159568714, "learning_rate": 2.710202258808967e-07, "logits/chosen": -2.765625, "logits/rejected": -2.8125, "logps/chosen": -520.0, "logps/rejected": -848.0, "loss": 0.2121, "rewards/accuracies": 0.9375, "rewards/chosen": -3.203125, "rewards/margins": 3.484375, "rewards/rejected": -6.6875, "step": 6990 }, { "epoch": 0.5266721841847867, "grad_norm": 16.839175598155585, "learning_rate": 2.70365846783258e-07, "logits/chosen": -2.734375, "logits/rejected": -2.640625, "logps/chosen": -510.0, "logps/rejected": -856.0, "loss": 0.2188, "rewards/accuracies": 0.90625, "rewards/chosen": -3.21875, "rewards/margins": 3.484375, "rewards/rejected": -6.6875, "step": 7000 }, { "epoch": 0.5274245730193364, "grad_norm": 9.430797008824985, "learning_rate": 2.6971132718864005e-07, "logits/chosen": -2.421875, "logits/rejected": -2.578125, "logps/chosen": -488.0, "logps/rejected": -792.0, "loss": 0.2111, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.921875, "rewards/margins": 3.265625, "rewards/rejected": -6.1875, "step": 7010 }, { "epoch": 0.5281769618538861, "grad_norm": 13.1875738662973, "learning_rate": 2.6905667161234844e-07, "logits/chosen": -2.59375, "logits/rejected": -2.65625, "logps/chosen": -486.0, "logps/rejected": -784.0, "loss": 0.2338, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.0625, "rewards/margins": 3.0, "rewards/rejected": -6.0625, "step": 7020 }, { "epoch": 0.5289293506884358, "grad_norm": 11.788705539881553, "learning_rate": 2.6840188457062725e-07, "logits/chosen": -2.625, "logits/rejected": -2.734375, "logps/chosen": -540.0, "logps/rejected": -840.0, "loss": 0.1972, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.171875, "rewards/margins": 3.484375, "rewards/rejected": -6.65625, "step": 7030 }, { "epoch": 0.5296817395229855, "grad_norm": 16.580437213024343, "learning_rate": 2.6774697058062713e-07, "logits/chosen": -2.765625, "logits/rejected": -2.625, "logps/chosen": -528.0, "logps/rejected": -844.0, "loss": 0.2136, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.421875, "rewards/margins": 3.328125, "rewards/rejected": -6.75, "step": 7040 }, { "epoch": 0.5304341283575352, "grad_norm": 9.014707059971963, "learning_rate": 2.6709193416037475e-07, "logits/chosen": -2.578125, "logits/rejected": -2.515625, "logps/chosen": -498.0, "logps/rejected": -828.0, "loss": 0.206, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -3.140625, "rewards/margins": 3.296875, "rewards/rejected": -6.4375, "step": 7050 }, { "epoch": 0.5311865171920849, "grad_norm": 13.129400436073599, "learning_rate": 2.6643677982874133e-07, "logits/chosen": -2.6875, "logits/rejected": -2.796875, "logps/chosen": -484.0, "logps/rejected": -812.0, "loss": 0.2235, "rewards/accuracies": 0.90625, "rewards/chosen": -3.15625, "rewards/margins": 3.375, "rewards/rejected": -6.53125, "step": 7060 }, { "epoch": 0.5319389060266345, "grad_norm": 15.104143138920092, "learning_rate": 2.657815121054115e-07, "logits/chosen": -2.828125, "logits/rejected": -2.78125, "logps/chosen": -516.0, "logps/rejected": -844.0, "loss": 0.208, "rewards/accuracies": 0.90625, "rewards/chosen": -3.15625, "rewards/margins": 3.390625, "rewards/rejected": -6.5625, "step": 7070 }, { "epoch": 0.5326912948611843, "grad_norm": 11.705955666035939, "learning_rate": 2.6512613551085214e-07, "logits/chosen": -2.6875, "logits/rejected": -2.75, "logps/chosen": -556.0, "logps/rejected": -852.0, "loss": 0.2062, "rewards/accuracies": 0.90625, "rewards/chosen": -3.46875, "rewards/margins": 3.265625, "rewards/rejected": -6.75, "step": 7080 }, { "epoch": 0.5334436836957339, "grad_norm": 9.419880272255284, "learning_rate": 2.6447065456628105e-07, "logits/chosen": -2.8125, "logits/rejected": -2.625, "logps/chosen": -528.0, "logps/rejected": -892.0, "loss": 0.1985, "rewards/accuracies": 0.90625, "rewards/chosen": -3.359375, "rewards/margins": 3.75, "rewards/rejected": -7.125, "step": 7090 }, { "epoch": 0.5341960725302837, "grad_norm": 10.299902340849993, "learning_rate": 2.6381507379363627e-07, "logits/chosen": -2.734375, "logits/rejected": -2.9375, "logps/chosen": -488.0, "logps/rejected": -816.0, "loss": 0.2028, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.875, "rewards/margins": 3.515625, "rewards/rejected": -6.375, "step": 7100 }, { "epoch": 0.5349484613648333, "grad_norm": 13.335379245092094, "learning_rate": 2.6315939771554407e-07, "logits/chosen": -2.765625, "logits/rejected": -2.6875, "logps/chosen": -490.0, "logps/rejected": -816.0, "loss": 0.2028, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.953125, "rewards/margins": 3.453125, "rewards/rejected": -6.40625, "step": 7110 }, { "epoch": 0.5357008501993831, "grad_norm": 11.056398019870423, "learning_rate": 2.6250363085528867e-07, "logits/chosen": -2.59375, "logits/rejected": -2.546875, "logps/chosen": -470.0, "logps/rejected": -780.0, "loss": 0.2262, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.90625, "rewards/margins": 3.109375, "rewards/rejected": -6.03125, "step": 7120 }, { "epoch": 0.5364532390339327, "grad_norm": 9.36041849181895, "learning_rate": 2.618477777367801e-07, "logits/chosen": -2.71875, "logits/rejected": -2.546875, "logps/chosen": -488.0, "logps/rejected": -832.0, "loss": 0.221, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -2.90625, "rewards/margins": 3.734375, "rewards/rejected": -6.625, "step": 7130 }, { "epoch": 0.5372056278684825, "grad_norm": 16.521092041100363, "learning_rate": 2.6119184288452377e-07, "logits/chosen": -2.71875, "logits/rejected": -2.78125, "logps/chosen": -472.0, "logps/rejected": -836.0, "loss": 0.2034, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.875, "rewards/margins": 3.671875, "rewards/rejected": -6.5625, "step": 7140 }, { "epoch": 0.5379580167030321, "grad_norm": 12.263696112235495, "learning_rate": 2.6053583082358887e-07, "logits/chosen": -2.640625, "logits/rejected": -2.75, "logps/chosen": -466.0, "logps/rejected": -800.0, "loss": 0.1964, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.828125, "rewards/margins": 3.46875, "rewards/rejected": -6.3125, "step": 7150 }, { "epoch": 0.5387104055375819, "grad_norm": 15.121645995211553, "learning_rate": 2.598797460795772e-07, "logits/chosen": -2.6875, "logits/rejected": -2.859375, "logps/chosen": -504.0, "logps/rejected": -776.0, "loss": 0.2132, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.078125, "rewards/margins": 3.078125, "rewards/rejected": -6.15625, "step": 7160 }, { "epoch": 0.5394627943721315, "grad_norm": 10.69999153512066, "learning_rate": 2.5922359317859195e-07, "logits/chosen": -2.75, "logits/rejected": -2.6875, "logps/chosen": -490.0, "logps/rejected": -840.0, "loss": 0.2159, "rewards/accuracies": 0.90625, "rewards/chosen": -3.015625, "rewards/margins": 3.65625, "rewards/rejected": -6.6875, "step": 7170 }, { "epoch": 0.5402151832066812, "grad_norm": 16.844700291624754, "learning_rate": 2.585673766472065e-07, "logits/chosen": -2.609375, "logits/rejected": -2.5, "logps/chosen": -516.0, "logps/rejected": -852.0, "loss": 0.2093, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.21875, "rewards/margins": 3.515625, "rewards/rejected": -6.71875, "step": 7180 }, { "epoch": 0.5409675720412309, "grad_norm": 12.430083948895065, "learning_rate": 2.5791110101243337e-07, "logits/chosen": -2.703125, "logits/rejected": -2.59375, "logps/chosen": -504.0, "logps/rejected": -844.0, "loss": 0.2185, "rewards/accuracies": 0.875, "rewards/chosen": -3.265625, "rewards/margins": 3.484375, "rewards/rejected": -6.75, "step": 7190 }, { "epoch": 0.5417199608757806, "grad_norm": 11.218687591815497, "learning_rate": 2.5725477080169243e-07, "logits/chosen": -2.75, "logits/rejected": -2.640625, "logps/chosen": -524.0, "logps/rejected": -852.0, "loss": 0.2105, "rewards/accuracies": 0.875, "rewards/chosen": -3.28125, "rewards/margins": 3.4375, "rewards/rejected": -6.71875, "step": 7200 }, { "epoch": 0.5424723497103303, "grad_norm": 12.044298745775524, "learning_rate": 2.565983905427806e-07, "logits/chosen": -2.953125, "logits/rejected": -2.84375, "logps/chosen": -524.0, "logps/rejected": -864.0, "loss": 0.2078, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -3.515625, "rewards/margins": 3.390625, "rewards/rejected": -6.90625, "step": 7210 }, { "epoch": 0.54322473854488, "grad_norm": 16.22010438297265, "learning_rate": 2.559419647638395e-07, "logits/chosen": -2.765625, "logits/rejected": -2.75, "logps/chosen": -516.0, "logps/rejected": -832.0, "loss": 0.2128, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -3.375, "rewards/margins": 3.296875, "rewards/rejected": -6.65625, "step": 7220 }, { "epoch": 0.5439771273794297, "grad_norm": 12.148317145404636, "learning_rate": 2.5528549799332515e-07, "logits/chosen": -2.640625, "logits/rejected": -2.453125, "logps/chosen": -516.0, "logps/rejected": -836.0, "loss": 0.2143, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.40625, "rewards/margins": 3.359375, "rewards/rejected": -6.75, "step": 7230 }, { "epoch": 0.5447295162139794, "grad_norm": 12.029829859835436, "learning_rate": 2.5462899475997617e-07, "logits/chosen": -2.84375, "logits/rejected": -2.75, "logps/chosen": -502.0, "logps/rejected": -848.0, "loss": 0.2213, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.15625, "rewards/margins": 3.671875, "rewards/rejected": -6.84375, "step": 7240 }, { "epoch": 0.5454819050485291, "grad_norm": 15.128051306445242, "learning_rate": 2.5397245959278284e-07, "logits/chosen": -2.796875, "logits/rejected": -2.59375, "logps/chosen": -508.0, "logps/rejected": -852.0, "loss": 0.2158, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.171875, "rewards/margins": 3.484375, "rewards/rejected": -6.65625, "step": 7250 }, { "epoch": 0.5462342938830788, "grad_norm": 15.768624060066504, "learning_rate": 2.533158970209558e-07, "logits/chosen": -2.640625, "logits/rejected": -2.453125, "logps/chosen": -528.0, "logps/rejected": -844.0, "loss": 0.2152, "rewards/accuracies": 0.90625, "rewards/chosen": -3.25, "rewards/margins": 3.28125, "rewards/rejected": -6.53125, "step": 7260 }, { "epoch": 0.5469866827176285, "grad_norm": 11.012500094663995, "learning_rate": 2.526593115738945e-07, "logits/chosen": -2.796875, "logits/rejected": -2.828125, "logps/chosen": -528.0, "logps/rejected": -832.0, "loss": 0.2004, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.359375, "rewards/margins": 3.28125, "rewards/rejected": -6.625, "step": 7270 }, { "epoch": 0.5477390715521782, "grad_norm": 11.36044663288735, "learning_rate": 2.5200270778115634e-07, "logits/chosen": -2.625, "logits/rejected": -2.5, "logps/chosen": -498.0, "logps/rejected": -832.0, "loss": 0.2011, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.96875, "rewards/margins": 3.65625, "rewards/rejected": -6.625, "step": 7280 }, { "epoch": 0.5484914603867279, "grad_norm": 14.90707377333477, "learning_rate": 2.513460901724253e-07, "logits/chosen": -2.703125, "logits/rejected": -2.75, "logps/chosen": -536.0, "logps/rejected": -876.0, "loss": 0.2069, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.453125, "rewards/margins": 3.578125, "rewards/rejected": -7.03125, "step": 7290 }, { "epoch": 0.5492438492212776, "grad_norm": 11.080057908849495, "learning_rate": 2.5068946327748073e-07, "logits/chosen": -2.71875, "logits/rejected": -2.71875, "logps/chosen": -476.0, "logps/rejected": -888.0, "loss": 0.1988, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -2.921875, "rewards/margins": 4.03125, "rewards/rejected": -6.96875, "step": 7300 }, { "epoch": 0.5499962380558272, "grad_norm": 12.913749572245399, "learning_rate": 2.5003283162616585e-07, "logits/chosen": -2.71875, "logits/rejected": -2.640625, "logps/chosen": -510.0, "logps/rejected": -848.0, "loss": 0.1872, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.09375, "rewards/margins": 3.578125, "rewards/rejected": -6.65625, "step": 7310 }, { "epoch": 0.550748626890377, "grad_norm": 15.047401998527171, "learning_rate": 2.493761997483569e-07, "logits/chosen": -2.765625, "logits/rejected": -2.671875, "logps/chosen": -508.0, "logps/rejected": -860.0, "loss": 0.2265, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.265625, "rewards/margins": 3.515625, "rewards/rejected": -6.78125, "step": 7320 }, { "epoch": 0.5515010157249266, "grad_norm": 17.405441825129305, "learning_rate": 2.4871957217393155e-07, "logits/chosen": -2.671875, "logits/rejected": -2.6875, "logps/chosen": -536.0, "logps/rejected": -872.0, "loss": 0.2191, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.375, "rewards/margins": 3.46875, "rewards/rejected": -6.8125, "step": 7330 }, { "epoch": 0.5522534045594764, "grad_norm": 13.616746562190682, "learning_rate": 2.480629534327378e-07, "logits/chosen": -2.78125, "logits/rejected": -2.609375, "logps/chosen": -494.0, "logps/rejected": -832.0, "loss": 0.2024, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.03125, "rewards/margins": 3.515625, "rewards/rejected": -6.5625, "step": 7340 }, { "epoch": 0.553005793394026, "grad_norm": 11.019238053486845, "learning_rate": 2.474063480545628e-07, "logits/chosen": -2.78125, "logits/rejected": -2.71875, "logps/chosen": -476.0, "logps/rejected": -792.0, "loss": 0.1978, "rewards/accuracies": 0.90625, "rewards/chosen": -2.984375, "rewards/margins": 3.203125, "rewards/rejected": -6.1875, "step": 7350 }, { "epoch": 0.5537581822285758, "grad_norm": 12.447332476730946, "learning_rate": 2.4674976056910136e-07, "logits/chosen": -2.71875, "logits/rejected": -2.46875, "logps/chosen": -494.0, "logps/rejected": -788.0, "loss": 0.2045, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.0625, "rewards/margins": 3.125, "rewards/rejected": -6.1875, "step": 7360 }, { "epoch": 0.5545105710631254, "grad_norm": 11.169885755814962, "learning_rate": 2.460931955059251e-07, "logits/chosen": -2.84375, "logits/rejected": -2.71875, "logps/chosen": -498.0, "logps/rejected": -864.0, "loss": 0.1983, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.28125, "rewards/margins": 3.6875, "rewards/rejected": -6.96875, "step": 7370 }, { "epoch": 0.5552629598976752, "grad_norm": 14.64407848519302, "learning_rate": 2.4543665739445054e-07, "logits/chosen": -2.78125, "logits/rejected": -2.765625, "logps/chosen": -504.0, "logps/rejected": -864.0, "loss": 0.1924, "rewards/accuracies": 0.9375, "rewards/chosen": -3.1875, "rewards/margins": 3.734375, "rewards/rejected": -6.9375, "step": 7380 }, { "epoch": 0.5560153487322248, "grad_norm": 15.28279471820109, "learning_rate": 2.447801507639087e-07, "logits/chosen": -2.828125, "logits/rejected": -2.828125, "logps/chosen": -532.0, "logps/rejected": -876.0, "loss": 0.2075, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.5, "rewards/margins": 3.46875, "rewards/rejected": -6.96875, "step": 7390 }, { "epoch": 0.5567677375667746, "grad_norm": 18.75353672233982, "learning_rate": 2.4412368014331326e-07, "logits/chosen": -2.765625, "logits/rejected": -2.71875, "logps/chosen": -490.0, "logps/rejected": -860.0, "loss": 0.2286, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.234375, "rewards/margins": 3.6875, "rewards/rejected": -6.90625, "step": 7400 }, { "epoch": 0.5575201264013242, "grad_norm": 12.160798244198634, "learning_rate": 2.434672500614294e-07, "logits/chosen": -2.6875, "logits/rejected": -2.609375, "logps/chosen": -486.0, "logps/rejected": -816.0, "loss": 0.2117, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.015625, "rewards/margins": 3.40625, "rewards/rejected": -6.4375, "step": 7410 }, { "epoch": 0.5582725152358738, "grad_norm": 15.547233970260622, "learning_rate": 2.428108650467427e-07, "logits/chosen": -2.6875, "logits/rejected": -2.65625, "logps/chosen": -544.0, "logps/rejected": -844.0, "loss": 0.2257, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.453125, "rewards/margins": 3.15625, "rewards/rejected": -6.625, "step": 7420 }, { "epoch": 0.5590249040704236, "grad_norm": 13.687304985986827, "learning_rate": 2.4215452962742795e-07, "logits/chosen": -2.796875, "logits/rejected": -2.625, "logps/chosen": -516.0, "logps/rejected": -828.0, "loss": 0.1944, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.390625, "rewards/margins": 3.21875, "rewards/rejected": -6.59375, "step": 7430 }, { "epoch": 0.5597772929049732, "grad_norm": 13.125138423232695, "learning_rate": 2.4149824833131766e-07, "logits/chosen": -2.78125, "logits/rejected": -3.0, "logps/chosen": -506.0, "logps/rejected": -872.0, "loss": 0.196, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.28125, "rewards/margins": 3.921875, "rewards/rejected": -7.1875, "step": 7440 }, { "epoch": 0.560529681739523, "grad_norm": 13.96561512643986, "learning_rate": 2.408420256858709e-07, "logits/chosen": -2.84375, "logits/rejected": -2.859375, "logps/chosen": -508.0, "logps/rejected": -836.0, "loss": 0.2042, "rewards/accuracies": 0.90625, "rewards/chosen": -3.34375, "rewards/margins": 3.28125, "rewards/rejected": -6.625, "step": 7450 }, { "epoch": 0.5612820705740726, "grad_norm": 11.607180459577807, "learning_rate": 2.4018586621814245e-07, "logits/chosen": -2.796875, "logits/rejected": -2.671875, "logps/chosen": -520.0, "logps/rejected": -880.0, "loss": 0.1916, "rewards/accuracies": 0.9375, "rewards/chosen": -3.3125, "rewards/margins": 3.671875, "rewards/rejected": -7.0, "step": 7460 }, { "epoch": 0.5620344594086224, "grad_norm": 12.446218298176449, "learning_rate": 2.395297744547507e-07, "logits/chosen": -2.734375, "logits/rejected": -2.671875, "logps/chosen": -540.0, "logps/rejected": -844.0, "loss": 0.2344, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.546875, "rewards/margins": 3.140625, "rewards/rejected": -6.6875, "step": 7470 }, { "epoch": 0.562786848243172, "grad_norm": 10.96807769206434, "learning_rate": 2.3887375492184754e-07, "logits/chosen": -2.859375, "logits/rejected": -2.6875, "logps/chosen": -536.0, "logps/rejected": -852.0, "loss": 0.1873, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.5, "rewards/margins": 3.296875, "rewards/rejected": -6.78125, "step": 7480 }, { "epoch": 0.5635392370777218, "grad_norm": 11.793833909151953, "learning_rate": 2.3821781214508624e-07, "logits/chosen": -2.71875, "logits/rejected": -2.78125, "logps/chosen": -540.0, "logps/rejected": -820.0, "loss": 0.2244, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.34375, "rewards/margins": 3.0, "rewards/rejected": -6.34375, "step": 7490 }, { "epoch": 0.5642916259122714, "grad_norm": 10.508332658042427, "learning_rate": 2.3756195064959062e-07, "logits/chosen": -2.75, "logits/rejected": -2.515625, "logps/chosen": -512.0, "logps/rejected": -820.0, "loss": 0.19, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -3.1875, "rewards/margins": 3.296875, "rewards/rejected": -6.46875, "step": 7500 }, { "epoch": 0.5650440147468212, "grad_norm": 10.075619073024296, "learning_rate": 2.369061749599238e-07, "logits/chosen": -2.71875, "logits/rejected": -2.671875, "logps/chosen": -580.0, "logps/rejected": -872.0, "loss": 0.2041, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.6875, "rewards/margins": 3.203125, "rewards/rejected": -6.90625, "step": 7510 }, { "epoch": 0.5657964035813708, "grad_norm": 18.589276214793955, "learning_rate": 2.36250489600057e-07, "logits/chosen": -2.71875, "logits/rejected": -2.875, "logps/chosen": -512.0, "logps/rejected": -808.0, "loss": 0.2027, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.15625, "rewards/margins": 3.265625, "rewards/rejected": -6.40625, "step": 7520 }, { "epoch": 0.5665487924159206, "grad_norm": 16.22132873094457, "learning_rate": 2.3559489909333812e-07, "logits/chosen": -2.765625, "logits/rejected": -2.71875, "logps/chosen": -536.0, "logps/rejected": -868.0, "loss": 0.1884, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.515625, "rewards/margins": 3.359375, "rewards/rejected": -6.875, "step": 7530 }, { "epoch": 0.5673011812504702, "grad_norm": 15.512210382268346, "learning_rate": 2.3493940796246088e-07, "logits/chosen": -2.9375, "logits/rejected": -2.96875, "logps/chosen": -520.0, "logps/rejected": -888.0, "loss": 0.2055, "rewards/accuracies": 0.90625, "rewards/chosen": -3.484375, "rewards/margins": 3.578125, "rewards/rejected": -7.0625, "step": 7540 }, { "epoch": 0.5680535700850199, "grad_norm": 11.710533088781727, "learning_rate": 2.342840207294335e-07, "logits/chosen": -2.796875, "logits/rejected": -2.96875, "logps/chosen": -528.0, "logps/rejected": -904.0, "loss": 0.1923, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.40625, "rewards/margins": 3.890625, "rewards/rejected": -7.3125, "step": 7550 }, { "epoch": 0.5688059589195696, "grad_norm": 13.507723366199807, "learning_rate": 2.33628741915547e-07, "logits/chosen": -2.8125, "logits/rejected": -2.671875, "logps/chosen": -498.0, "logps/rejected": -888.0, "loss": 0.1651, "rewards/accuracies": 0.9375, "rewards/chosen": -3.3125, "rewards/margins": 3.875, "rewards/rejected": -7.1875, "step": 7560 }, { "epoch": 0.5695583477541193, "grad_norm": 12.977679097598456, "learning_rate": 2.32973576041345e-07, "logits/chosen": -2.78125, "logits/rejected": -2.65625, "logps/chosen": -512.0, "logps/rejected": -904.0, "loss": 0.2092, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -3.203125, "rewards/margins": 3.984375, "rewards/rejected": -7.1875, "step": 7570 }, { "epoch": 0.570310736588669, "grad_norm": 15.4070694549532, "learning_rate": 2.3231852762659165e-07, "logits/chosen": -2.875, "logits/rejected": -2.78125, "logps/chosen": -474.0, "logps/rejected": -856.0, "loss": 0.2127, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.875, "rewards/margins": 3.96875, "rewards/rejected": -6.84375, "step": 7580 }, { "epoch": 0.5710631254232187, "grad_norm": 10.639584903346911, "learning_rate": 2.3166360119024094e-07, "logits/chosen": -2.6875, "logits/rejected": -2.59375, "logps/chosen": -544.0, "logps/rejected": -880.0, "loss": 0.1813, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.3125, "rewards/margins": 3.625, "rewards/rejected": -6.9375, "step": 7590 }, { "epoch": 0.5718155142577684, "grad_norm": 9.046535825509498, "learning_rate": 2.310088012504053e-07, "logits/chosen": -2.90625, "logits/rejected": -2.765625, "logps/chosen": -504.0, "logps/rejected": -856.0, "loss": 0.2013, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.25, "rewards/margins": 3.515625, "rewards/rejected": -6.78125, "step": 7600 }, { "epoch": 0.5725679030923181, "grad_norm": 9.855751150862275, "learning_rate": 2.303541323243246e-07, "logits/chosen": -2.78125, "logits/rejected": -2.890625, "logps/chosen": -500.0, "logps/rejected": -824.0, "loss": 0.2074, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.171875, "rewards/margins": 3.390625, "rewards/rejected": -6.5625, "step": 7610 }, { "epoch": 0.5733202919268678, "grad_norm": 12.945105053806856, "learning_rate": 2.2969959892833474e-07, "logits/chosen": -2.84375, "logits/rejected": -2.671875, "logps/chosen": -494.0, "logps/rejected": -852.0, "loss": 0.1936, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.171875, "rewards/margins": 3.484375, "rewards/rejected": -6.65625, "step": 7620 }, { "epoch": 0.5740726807614175, "grad_norm": 10.526143498948, "learning_rate": 2.290452055778367e-07, "logits/chosen": -2.8125, "logits/rejected": -2.8125, "logps/chosen": -536.0, "logps/rejected": -864.0, "loss": 0.214, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.421875, "rewards/margins": 3.53125, "rewards/rejected": -6.96875, "step": 7630 }, { "epoch": 0.5748250695959672, "grad_norm": 15.749010709574826, "learning_rate": 2.2839095678726553e-07, "logits/chosen": -2.921875, "logits/rejected": -2.90625, "logps/chosen": -492.0, "logps/rejected": -860.0, "loss": 0.1979, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.109375, "rewards/margins": 3.65625, "rewards/rejected": -6.75, "step": 7640 }, { "epoch": 0.5755774584305169, "grad_norm": 10.89756635815755, "learning_rate": 2.2773685707005863e-07, "logits/chosen": -2.71875, "logits/rejected": -2.65625, "logps/chosen": -568.0, "logps/rejected": -880.0, "loss": 0.2019, "rewards/accuracies": 0.90625, "rewards/chosen": -3.671875, "rewards/margins": 3.25, "rewards/rejected": -6.9375, "step": 7650 }, { "epoch": 0.5763298472650666, "grad_norm": 14.913763643381511, "learning_rate": 2.270829109386253e-07, "logits/chosen": -2.703125, "logits/rejected": -2.640625, "logps/chosen": -524.0, "logps/rejected": -828.0, "loss": 0.2243, "rewards/accuracies": 0.875, "rewards/chosen": -3.25, "rewards/margins": 3.28125, "rewards/rejected": -6.53125, "step": 7660 }, { "epoch": 0.5770822360996163, "grad_norm": 8.727092670482827, "learning_rate": 2.2642912290431518e-07, "logits/chosen": -2.71875, "logits/rejected": -2.640625, "logps/chosen": -494.0, "logps/rejected": -864.0, "loss": 0.2051, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.09375, "rewards/margins": 3.796875, "rewards/rejected": -6.875, "step": 7670 }, { "epoch": 0.5778346249341659, "grad_norm": 11.666334427151252, "learning_rate": 2.257754974773873e-07, "logits/chosen": -2.8125, "logits/rejected": -2.625, "logps/chosen": -528.0, "logps/rejected": -876.0, "loss": 0.215, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.328125, "rewards/margins": 3.59375, "rewards/rejected": -6.9375, "step": 7680 }, { "epoch": 0.5785870137687157, "grad_norm": 10.001204235798024, "learning_rate": 2.2512203916697892e-07, "logits/chosen": -2.703125, "logits/rejected": -2.78125, "logps/chosen": -512.0, "logps/rejected": -844.0, "loss": 0.1887, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.203125, "rewards/margins": 3.4375, "rewards/rejected": -6.625, "step": 7690 }, { "epoch": 0.5793394026032653, "grad_norm": 9.384138761412927, "learning_rate": 2.2446875248107432e-07, "logits/chosen": -2.953125, "logits/rejected": -2.578125, "logps/chosen": -532.0, "logps/rejected": -884.0, "loss": 0.2013, "rewards/accuracies": 0.9375, "rewards/chosen": -3.453125, "rewards/margins": 3.640625, "rewards/rejected": -7.09375, "step": 7700 }, { "epoch": 0.5800917914378151, "grad_norm": 11.476721759126246, "learning_rate": 2.2381564192647396e-07, "logits/chosen": -2.828125, "logits/rejected": -2.671875, "logps/chosen": -508.0, "logps/rejected": -864.0, "loss": 0.2037, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.203125, "rewards/margins": 3.625, "rewards/rejected": -6.84375, "step": 7710 }, { "epoch": 0.5808441802723647, "grad_norm": 10.547195849553724, "learning_rate": 2.231627120087631e-07, "logits/chosen": -2.84375, "logits/rejected": -2.59375, "logps/chosen": -484.0, "logps/rejected": -900.0, "loss": 0.1952, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -3.03125, "rewards/margins": 4.1875, "rewards/rejected": -7.25, "step": 7720 }, { "epoch": 0.5815965691069145, "grad_norm": 12.295988135008125, "learning_rate": 2.2250996723228104e-07, "logits/chosen": -2.90625, "logits/rejected": -2.96875, "logps/chosen": -544.0, "logps/rejected": -900.0, "loss": 0.198, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.625, "rewards/margins": 3.59375, "rewards/rejected": -7.21875, "step": 7730 }, { "epoch": 0.5823489579414641, "grad_norm": 13.745510020253832, "learning_rate": 2.2185741210008947e-07, "logits/chosen": -2.734375, "logits/rejected": -2.71875, "logps/chosen": -540.0, "logps/rejected": -900.0, "loss": 0.19, "rewards/accuracies": 0.9375, "rewards/chosen": -3.3125, "rewards/margins": 3.71875, "rewards/rejected": -7.03125, "step": 7740 }, { "epoch": 0.5831013467760139, "grad_norm": 12.544664335012413, "learning_rate": 2.2120505111394224e-07, "logits/chosen": -2.859375, "logits/rejected": -2.75, "logps/chosen": -564.0, "logps/rejected": -904.0, "loss": 0.1948, "rewards/accuracies": 0.90625, "rewards/chosen": -3.6875, "rewards/margins": 3.484375, "rewards/rejected": -7.1875, "step": 7750 }, { "epoch": 0.5838537356105635, "grad_norm": 11.622510013895388, "learning_rate": 2.205528887742535e-07, "logits/chosen": -2.6875, "logits/rejected": -2.609375, "logps/chosen": -584.0, "logps/rejected": -944.0, "loss": 0.1967, "rewards/accuracies": 0.90625, "rewards/chosen": -3.765625, "rewards/margins": 3.6875, "rewards/rejected": -7.4375, "step": 7760 }, { "epoch": 0.5846061244451133, "grad_norm": 10.142912167898485, "learning_rate": 2.199009295800672e-07, "logits/chosen": -2.6875, "logits/rejected": -2.515625, "logps/chosen": -556.0, "logps/rejected": -888.0, "loss": 0.1789, "rewards/accuracies": 0.90625, "rewards/chosen": -3.71875, "rewards/margins": 3.453125, "rewards/rejected": -7.1875, "step": 7770 }, { "epoch": 0.5853585132796629, "grad_norm": 15.145830486179621, "learning_rate": 2.192491780290259e-07, "logits/chosen": -2.796875, "logits/rejected": -2.8125, "logps/chosen": -540.0, "logps/rejected": -888.0, "loss": 0.1915, "rewards/accuracies": 0.90625, "rewards/chosen": -3.375, "rewards/margins": 3.6875, "rewards/rejected": -7.0625, "step": 7780 }, { "epoch": 0.5861109021142126, "grad_norm": 18.827220047958438, "learning_rate": 2.1859763861733947e-07, "logits/chosen": -2.859375, "logits/rejected": -2.921875, "logps/chosen": -488.0, "logps/rejected": -792.0, "loss": 0.2109, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.09375, "rewards/margins": 3.25, "rewards/rejected": -6.34375, "step": 7790 }, { "epoch": 0.5868632909487623, "grad_norm": 15.028086560471248, "learning_rate": 2.179463158397545e-07, "logits/chosen": -2.796875, "logits/rejected": -2.859375, "logps/chosen": -510.0, "logps/rejected": -848.0, "loss": 0.2061, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.296875, "rewards/margins": 3.515625, "rewards/rejected": -6.8125, "step": 7800 }, { "epoch": 0.587615679783312, "grad_norm": 11.286258141261165, "learning_rate": 2.1729521418952304e-07, "logits/chosen": -2.84375, "logits/rejected": -2.90625, "logps/chosen": -484.0, "logps/rejected": -888.0, "loss": 0.1919, "rewards/accuracies": 0.9375, "rewards/chosen": -2.953125, "rewards/margins": 4.1875, "rewards/rejected": -7.15625, "step": 7810 }, { "epoch": 0.5883680686178617, "grad_norm": 11.573615061254767, "learning_rate": 2.166443381583718e-07, "logits/chosen": -2.765625, "logits/rejected": -2.828125, "logps/chosen": -528.0, "logps/rejected": -896.0, "loss": 0.1785, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.484375, "rewards/margins": 3.578125, "rewards/rejected": -7.0625, "step": 7820 }, { "epoch": 0.5891204574524114, "grad_norm": 13.017097218784468, "learning_rate": 2.1599369223647068e-07, "logits/chosen": -2.75, "logits/rejected": -2.96875, "logps/chosen": -528.0, "logps/rejected": -832.0, "loss": 0.2018, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.390625, "rewards/margins": 3.328125, "rewards/rejected": -6.71875, "step": 7830 }, { "epoch": 0.5898728462869611, "grad_norm": 15.619795856682215, "learning_rate": 2.1534328091240238e-07, "logits/chosen": -2.90625, "logits/rejected": -2.890625, "logps/chosen": -524.0, "logps/rejected": -864.0, "loss": 0.2131, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.375, "rewards/margins": 3.4375, "rewards/rejected": -6.8125, "step": 7840 }, { "epoch": 0.5906252351215108, "grad_norm": 10.786549931470189, "learning_rate": 2.1469310867313118e-07, "logits/chosen": -2.765625, "logits/rejected": -2.734375, "logps/chosen": -564.0, "logps/rejected": -896.0, "loss": 0.2104, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -3.515625, "rewards/margins": 3.640625, "rewards/rejected": -7.15625, "step": 7850 }, { "epoch": 0.5913776239560605, "grad_norm": 12.238818017166324, "learning_rate": 2.1404318000397192e-07, "logits/chosen": -2.75, "logits/rejected": -2.828125, "logps/chosen": -516.0, "logps/rejected": -848.0, "loss": 0.1908, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.25, "rewards/margins": 3.515625, "rewards/rejected": -6.78125, "step": 7860 }, { "epoch": 0.5921300127906102, "grad_norm": 13.719882793698565, "learning_rate": 2.1339349938855929e-07, "logits/chosen": -2.953125, "logits/rejected": -2.890625, "logps/chosen": -504.0, "logps/rejected": -824.0, "loss": 0.1991, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.3125, "rewards/margins": 3.375, "rewards/rejected": -6.6875, "step": 7870 }, { "epoch": 0.5928824016251599, "grad_norm": 12.708526671508313, "learning_rate": 2.127440713088165e-07, "logits/chosen": -2.75, "logits/rejected": -2.828125, "logps/chosen": -520.0, "logps/rejected": -868.0, "loss": 0.2027, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.28125, "rewards/margins": 3.625, "rewards/rejected": -6.90625, "step": 7880 }, { "epoch": 0.5936347904597096, "grad_norm": 15.63000246941594, "learning_rate": 2.1209490024492477e-07, "logits/chosen": -2.84375, "logits/rejected": -2.8125, "logps/chosen": -512.0, "logps/rejected": -840.0, "loss": 0.2154, "rewards/accuracies": 0.9375, "rewards/chosen": -3.28125, "rewards/margins": 3.4375, "rewards/rejected": -6.71875, "step": 7890 }, { "epoch": 0.5943871792942593, "grad_norm": 13.560858689221693, "learning_rate": 2.114459906752923e-07, "logits/chosen": -2.765625, "logits/rejected": -2.890625, "logps/chosen": -528.0, "logps/rejected": -840.0, "loss": 0.2072, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.34375, "rewards/margins": 3.328125, "rewards/rejected": -6.6875, "step": 7900 }, { "epoch": 0.595139568128809, "grad_norm": 11.125494655976635, "learning_rate": 2.1079734707652315e-07, "logits/chosen": -2.6875, "logits/rejected": -2.671875, "logps/chosen": -520.0, "logps/rejected": -812.0, "loss": 0.2243, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -3.203125, "rewards/margins": 3.203125, "rewards/rejected": -6.40625, "step": 7910 }, { "epoch": 0.5958919569633586, "grad_norm": 11.25773514806983, "learning_rate": 2.101489739233867e-07, "logits/chosen": -2.609375, "logits/rejected": -2.671875, "logps/chosen": -516.0, "logps/rejected": -880.0, "loss": 0.1898, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.25, "rewards/margins": 3.78125, "rewards/rejected": -7.03125, "step": 7920 }, { "epoch": 0.5966443457979084, "grad_norm": 8.815393202473977, "learning_rate": 2.0950087568878643e-07, "logits/chosen": -2.90625, "logits/rejected": -2.765625, "logps/chosen": -516.0, "logps/rejected": -876.0, "loss": 0.2085, "rewards/accuracies": 0.90625, "rewards/chosen": -3.3125, "rewards/margins": 3.75, "rewards/rejected": -7.0625, "step": 7930 }, { "epoch": 0.597396734632458, "grad_norm": 11.462686650117979, "learning_rate": 2.088530568437295e-07, "logits/chosen": -2.859375, "logits/rejected": -2.828125, "logps/chosen": -512.0, "logps/rejected": -872.0, "loss": 0.2091, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.34375, "rewards/margins": 3.65625, "rewards/rejected": -7.0, "step": 7940 }, { "epoch": 0.5981491234670078, "grad_norm": 11.46220360647605, "learning_rate": 2.0820552185729553e-07, "logits/chosen": -2.765625, "logits/rejected": -2.765625, "logps/chosen": -528.0, "logps/rejected": -876.0, "loss": 0.1899, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.3125, "rewards/margins": 3.71875, "rewards/rejected": -7.03125, "step": 7950 }, { "epoch": 0.5989015123015574, "grad_norm": 11.839921531754774, "learning_rate": 2.0755827519660585e-07, "logits/chosen": -2.671875, "logits/rejected": -2.71875, "logps/chosen": -520.0, "logps/rejected": -864.0, "loss": 0.2063, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.203125, "rewards/margins": 3.75, "rewards/rejected": -6.9375, "step": 7960 }, { "epoch": 0.5996539011361072, "grad_norm": 11.751163180472034, "learning_rate": 2.069113213267928e-07, "logits/chosen": -2.78125, "logits/rejected": -2.921875, "logps/chosen": -524.0, "logps/rejected": -848.0, "loss": 0.1924, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.34375, "rewards/margins": 3.34375, "rewards/rejected": -6.6875, "step": 7970 }, { "epoch": 0.6004062899706568, "grad_norm": 12.970126841893364, "learning_rate": 2.062646647109688e-07, "logits/chosen": -2.828125, "logits/rejected": -2.84375, "logps/chosen": -470.0, "logps/rejected": -812.0, "loss": 0.209, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.875, "rewards/margins": 3.625, "rewards/rejected": -6.5, "step": 7980 }, { "epoch": 0.6011586788052066, "grad_norm": 14.29463407634808, "learning_rate": 2.0561830981019582e-07, "logits/chosen": -2.765625, "logits/rejected": -2.75, "logps/chosen": -506.0, "logps/rejected": -852.0, "loss": 0.2053, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.15625, "rewards/margins": 3.59375, "rewards/rejected": -6.75, "step": 7990 }, { "epoch": 0.6019110676397562, "grad_norm": 11.160122190924627, "learning_rate": 2.04972261083454e-07, "logits/chosen": -2.78125, "logits/rejected": -2.5625, "logps/chosen": -528.0, "logps/rejected": -884.0, "loss": 0.1717, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.265625, "rewards/margins": 3.640625, "rewards/rejected": -6.90625, "step": 8000 }, { "epoch": 0.602663456474306, "grad_norm": 16.006048923165142, "learning_rate": 2.0432652298761176e-07, "logits/chosen": -2.875, "logits/rejected": -2.90625, "logps/chosen": -524.0, "logps/rejected": -860.0, "loss": 0.2071, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.375, "rewards/margins": 3.375, "rewards/rejected": -6.75, "step": 8010 }, { "epoch": 0.6034158453088556, "grad_norm": 8.583406451827797, "learning_rate": 2.0368109997739415e-07, "logits/chosen": -2.875, "logits/rejected": -2.78125, "logps/chosen": -484.0, "logps/rejected": -872.0, "loss": 0.1828, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.0, "rewards/margins": 3.84375, "rewards/rejected": -6.84375, "step": 8020 }, { "epoch": 0.6041682341434053, "grad_norm": 12.702894858562102, "learning_rate": 2.0303599650535283e-07, "logits/chosen": -2.640625, "logits/rejected": -2.484375, "logps/chosen": -528.0, "logps/rejected": -844.0, "loss": 0.205, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.28125, "rewards/margins": 3.390625, "rewards/rejected": -6.6875, "step": 8030 }, { "epoch": 0.604920622977955, "grad_norm": 11.463209075851706, "learning_rate": 2.0239121702183505e-07, "logits/chosen": -2.796875, "logits/rejected": -2.828125, "logps/chosen": -468.0, "logps/rejected": -836.0, "loss": 0.2118, "rewards/accuracies": 0.9375, "rewards/chosen": -2.984375, "rewards/margins": 3.671875, "rewards/rejected": -6.65625, "step": 8040 }, { "epoch": 0.6056730118125047, "grad_norm": 10.824183564144633, "learning_rate": 2.017467659749528e-07, "logits/chosen": -2.828125, "logits/rejected": -2.734375, "logps/chosen": -540.0, "logps/rejected": -848.0, "loss": 0.1902, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.1875, "rewards/margins": 3.59375, "rewards/rejected": -6.78125, "step": 8050 }, { "epoch": 0.6064254006470544, "grad_norm": 11.428851498777538, "learning_rate": 2.011026478105525e-07, "logits/chosen": -2.75, "logits/rejected": -2.828125, "logps/chosen": -512.0, "logps/rejected": -864.0, "loss": 0.196, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.171875, "rewards/margins": 3.609375, "rewards/rejected": -6.78125, "step": 8060 }, { "epoch": 0.6071777894816041, "grad_norm": 11.135437202183654, "learning_rate": 2.004588669721841e-07, "logits/chosen": -2.90625, "logits/rejected": -2.671875, "logps/chosen": -524.0, "logps/rejected": -900.0, "loss": 0.1942, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.390625, "rewards/margins": 3.765625, "rewards/rejected": -7.15625, "step": 8070 }, { "epoch": 0.6079301783161538, "grad_norm": 11.412330608730766, "learning_rate": 1.9981542790107032e-07, "logits/chosen": -2.8125, "logits/rejected": -2.796875, "logps/chosen": -524.0, "logps/rejected": -888.0, "loss": 0.1955, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.140625, "rewards/margins": 3.765625, "rewards/rejected": -6.90625, "step": 8080 }, { "epoch": 0.6086825671507035, "grad_norm": 12.83353510592892, "learning_rate": 1.9917233503607625e-07, "logits/chosen": -2.953125, "logits/rejected": -2.8125, "logps/chosen": -506.0, "logps/rejected": -900.0, "loss": 0.1761, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.265625, "rewards/margins": 4.03125, "rewards/rejected": -7.3125, "step": 8090 }, { "epoch": 0.6094349559852532, "grad_norm": 14.531779315775934, "learning_rate": 1.9852959281367872e-07, "logits/chosen": -2.859375, "logits/rejected": -2.859375, "logps/chosen": -516.0, "logps/rejected": -864.0, "loss": 0.1895, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.375, "rewards/margins": 3.5625, "rewards/rejected": -6.9375, "step": 8100 }, { "epoch": 0.6101873448198029, "grad_norm": 10.089410588155014, "learning_rate": 1.9788720566793527e-07, "logits/chosen": -2.84375, "logits/rejected": -2.859375, "logps/chosen": -506.0, "logps/rejected": -836.0, "loss": 0.1914, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.25, "rewards/margins": 3.421875, "rewards/rejected": -6.6875, "step": 8110 }, { "epoch": 0.6109397336543526, "grad_norm": 11.138074850756247, "learning_rate": 1.9724517803045417e-07, "logits/chosen": -2.796875, "logits/rejected": -2.65625, "logps/chosen": -508.0, "logps/rejected": -856.0, "loss": 0.2084, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -3.25, "rewards/margins": 3.390625, "rewards/rejected": -6.625, "step": 8120 }, { "epoch": 0.6116921224889023, "grad_norm": 10.875176843728601, "learning_rate": 1.966035143303636e-07, "logits/chosen": -2.75, "logits/rejected": -2.765625, "logps/chosen": -548.0, "logps/rejected": -836.0, "loss": 0.1894, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.375, "rewards/margins": 3.234375, "rewards/rejected": -6.625, "step": 8130 }, { "epoch": 0.612444511323452, "grad_norm": 9.866156191761272, "learning_rate": 1.959622189942808e-07, "logits/chosen": -2.796875, "logits/rejected": -2.734375, "logps/chosen": -496.0, "logps/rejected": -848.0, "loss": 0.1905, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.109375, "rewards/margins": 3.5, "rewards/rejected": -6.59375, "step": 8140 }, { "epoch": 0.6131969001580017, "grad_norm": 14.055705396458091, "learning_rate": 1.9532129644628204e-07, "logits/chosen": -2.84375, "logits/rejected": -2.78125, "logps/chosen": -504.0, "logps/rejected": -828.0, "loss": 0.2068, "rewards/accuracies": 0.90625, "rewards/chosen": -3.21875, "rewards/margins": 3.34375, "rewards/rejected": -6.5625, "step": 8150 }, { "epoch": 0.6139492889925513, "grad_norm": 16.146177730033074, "learning_rate": 1.946807511078718e-07, "logits/chosen": -2.703125, "logits/rejected": -2.890625, "logps/chosen": -512.0, "logps/rejected": -848.0, "loss": 0.2052, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.078125, "rewards/margins": 3.65625, "rewards/rejected": -6.75, "step": 8160 }, { "epoch": 0.6147016778271011, "grad_norm": 12.081719111333763, "learning_rate": 1.9404058739795217e-07, "logits/chosen": -2.71875, "logits/rejected": -2.828125, "logps/chosen": -520.0, "logps/rejected": -864.0, "loss": 0.1915, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.1875, "rewards/margins": 3.734375, "rewards/rejected": -6.9375, "step": 8170 }, { "epoch": 0.6154540666616507, "grad_norm": 11.658471284427655, "learning_rate": 1.9340080973279268e-07, "logits/chosen": -2.765625, "logits/rejected": -2.59375, "logps/chosen": -496.0, "logps/rejected": -864.0, "loss": 0.1767, "rewards/accuracies": 0.9375, "rewards/chosen": -3.046875, "rewards/margins": 3.75, "rewards/rejected": -6.8125, "step": 8180 }, { "epoch": 0.6162064554962005, "grad_norm": 19.92535314433597, "learning_rate": 1.9276142252599971e-07, "logits/chosen": -2.78125, "logits/rejected": -2.875, "logps/chosen": -500.0, "logps/rejected": -832.0, "loss": 0.1968, "rewards/accuracies": 0.9375, "rewards/chosen": -3.1875, "rewards/margins": 3.515625, "rewards/rejected": -6.6875, "step": 8190 }, { "epoch": 0.6169588443307501, "grad_norm": 14.060838972246453, "learning_rate": 1.9212243018848572e-07, "logits/chosen": -2.6875, "logits/rejected": -2.75, "logps/chosen": -506.0, "logps/rejected": -892.0, "loss": 0.1955, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.171875, "rewards/margins": 3.96875, "rewards/rejected": -7.125, "step": 8200 }, { "epoch": 0.6177112331652999, "grad_norm": 10.641794519209762, "learning_rate": 1.9148383712843946e-07, "logits/chosen": -2.796875, "logits/rejected": -2.765625, "logps/chosen": -532.0, "logps/rejected": -832.0, "loss": 0.2109, "rewards/accuracies": 0.875, "rewards/chosen": -3.296875, "rewards/margins": 3.296875, "rewards/rejected": -6.59375, "step": 8210 }, { "epoch": 0.6184636219998495, "grad_norm": 10.701959851990978, "learning_rate": 1.908456477512949e-07, "logits/chosen": -2.640625, "logits/rejected": -2.453125, "logps/chosen": -506.0, "logps/rejected": -848.0, "loss": 0.1948, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.125, "rewards/margins": 3.578125, "rewards/rejected": -6.6875, "step": 8220 }, { "epoch": 0.6192160108343993, "grad_norm": 14.875642454271361, "learning_rate": 1.9020786645970132e-07, "logits/chosen": -2.65625, "logits/rejected": -2.5625, "logps/chosen": -516.0, "logps/rejected": -864.0, "loss": 0.2291, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.234375, "rewards/margins": 3.59375, "rewards/rejected": -6.8125, "step": 8230 }, { "epoch": 0.6199683996689489, "grad_norm": 14.911405035776056, "learning_rate": 1.8957049765349275e-07, "logits/chosen": -2.765625, "logits/rejected": -2.609375, "logps/chosen": -484.0, "logps/rejected": -828.0, "loss": 0.2025, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.078125, "rewards/margins": 3.546875, "rewards/rejected": -6.625, "step": 8240 }, { "epoch": 0.6207207885034987, "grad_norm": 11.59662712085765, "learning_rate": 1.8893354572965764e-07, "logits/chosen": -2.6875, "logits/rejected": -2.796875, "logps/chosen": -494.0, "logps/rejected": -840.0, "loss": 0.1933, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.265625, "rewards/margins": 3.40625, "rewards/rejected": -6.6875, "step": 8250 }, { "epoch": 0.6214731773380483, "grad_norm": 11.818064942135853, "learning_rate": 1.882970150823083e-07, "logits/chosen": -2.84375, "logits/rejected": -2.703125, "logps/chosen": -512.0, "logps/rejected": -864.0, "loss": 0.2042, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.1875, "rewards/margins": 3.53125, "rewards/rejected": -6.71875, "step": 8260 }, { "epoch": 0.6222255661725979, "grad_norm": 10.698556073226928, "learning_rate": 1.8766091010265107e-07, "logits/chosen": -2.6875, "logits/rejected": -2.796875, "logps/chosen": -506.0, "logps/rejected": -860.0, "loss": 0.1806, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.078125, "rewards/margins": 3.78125, "rewards/rejected": -6.875, "step": 8270 }, { "epoch": 0.6229779550071477, "grad_norm": 10.61343444536279, "learning_rate": 1.870252351789557e-07, "logits/chosen": -2.734375, "logits/rejected": -2.765625, "logps/chosen": -494.0, "logps/rejected": -840.0, "loss": 0.194, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.109375, "rewards/margins": 3.5625, "rewards/rejected": -6.65625, "step": 8280 }, { "epoch": 0.6237303438416973, "grad_norm": 10.86749165802413, "learning_rate": 1.8638999469652485e-07, "logits/chosen": -2.84375, "logits/rejected": -2.828125, "logps/chosen": -494.0, "logps/rejected": -844.0, "loss": 0.2224, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.21875, "rewards/margins": 3.46875, "rewards/rejected": -6.6875, "step": 8290 }, { "epoch": 0.6244827326762471, "grad_norm": 10.111549395492688, "learning_rate": 1.8575519303766456e-07, "logits/chosen": -2.671875, "logits/rejected": -2.78125, "logps/chosen": -532.0, "logps/rejected": -868.0, "loss": 0.2083, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.296875, "rewards/margins": 3.703125, "rewards/rejected": -7.0, "step": 8300 }, { "epoch": 0.6252351215107967, "grad_norm": 10.47556787354029, "learning_rate": 1.8512083458165322e-07, "logits/chosen": -2.859375, "logits/rejected": -2.703125, "logps/chosen": -528.0, "logps/rejected": -852.0, "loss": 0.2153, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.421875, "rewards/margins": 3.4375, "rewards/rejected": -6.84375, "step": 8310 }, { "epoch": 0.6259875103453465, "grad_norm": 13.390586289221135, "learning_rate": 1.8448692370471197e-07, "logits/chosen": -2.71875, "logits/rejected": -2.59375, "logps/chosen": -484.0, "logps/rejected": -828.0, "loss": 0.1999, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.015625, "rewards/margins": 3.46875, "rewards/rejected": -6.5, "step": 8320 }, { "epoch": 0.6267398991798961, "grad_norm": 11.863226652794859, "learning_rate": 1.838534647799741e-07, "logits/chosen": -2.75, "logits/rejected": -2.46875, "logps/chosen": -516.0, "logps/rejected": -796.0, "loss": 0.2099, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.171875, "rewards/margins": 2.984375, "rewards/rejected": -6.15625, "step": 8330 }, { "epoch": 0.6274922880144459, "grad_norm": 15.475802182185896, "learning_rate": 1.83220462177455e-07, "logits/chosen": -2.75, "logits/rejected": -2.59375, "logps/chosen": -510.0, "logps/rejected": -884.0, "loss": 0.2049, "rewards/accuracies": 0.9375, "rewards/chosen": -3.265625, "rewards/margins": 3.84375, "rewards/rejected": -7.125, "step": 8340 }, { "epoch": 0.6282446768489955, "grad_norm": 11.193474180186774, "learning_rate": 1.825879202640222e-07, "logits/chosen": -2.796875, "logits/rejected": -2.640625, "logps/chosen": -528.0, "logps/rejected": -872.0, "loss": 0.2004, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.375, "rewards/margins": 3.40625, "rewards/rejected": -6.78125, "step": 8350 }, { "epoch": 0.6289970656835453, "grad_norm": 12.165498689317088, "learning_rate": 1.81955843403365e-07, "logits/chosen": -2.828125, "logits/rejected": -2.71875, "logps/chosen": -502.0, "logps/rejected": -860.0, "loss": 0.1977, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -3.28125, "rewards/margins": 3.65625, "rewards/rejected": -6.9375, "step": 8360 }, { "epoch": 0.6297494545180949, "grad_norm": 8.714375314969942, "learning_rate": 1.8132423595596464e-07, "logits/chosen": -2.71875, "logits/rejected": -2.640625, "logps/chosen": -524.0, "logps/rejected": -860.0, "loss": 0.2027, "rewards/accuracies": 0.9375, "rewards/chosen": -3.34375, "rewards/margins": 3.5, "rewards/rejected": -6.84375, "step": 8370 }, { "epoch": 0.6305018433526447, "grad_norm": 12.21323723731653, "learning_rate": 1.8069310227906365e-07, "logits/chosen": -2.671875, "logits/rejected": -2.703125, "logps/chosen": -536.0, "logps/rejected": -840.0, "loss": 0.2081, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.203125, "rewards/margins": 3.4375, "rewards/rejected": -6.65625, "step": 8380 }, { "epoch": 0.6312542321871943, "grad_norm": 12.178629224373406, "learning_rate": 1.8006244672663644e-07, "logits/chosen": -2.625, "logits/rejected": -2.546875, "logps/chosen": -512.0, "logps/rejected": -852.0, "loss": 0.1876, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.28125, "rewards/margins": 3.421875, "rewards/rejected": -6.6875, "step": 8390 }, { "epoch": 0.632006621021744, "grad_norm": 10.619159652067346, "learning_rate": 1.794322736493591e-07, "logits/chosen": -2.75, "logits/rejected": -2.75, "logps/chosen": -536.0, "logps/rejected": -888.0, "loss": 0.2053, "rewards/accuracies": 0.90625, "rewards/chosen": -3.359375, "rewards/margins": 3.75, "rewards/rejected": -7.09375, "step": 8400 }, { "epoch": 0.6327590098562937, "grad_norm": 10.083804134105026, "learning_rate": 1.7880258739457905e-07, "logits/chosen": -2.875, "logits/rejected": -2.90625, "logps/chosen": -544.0, "logps/rejected": -896.0, "loss": 0.2095, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.515625, "rewards/margins": 3.671875, "rewards/rejected": -7.1875, "step": 8410 }, { "epoch": 0.6335113986908434, "grad_norm": 10.869727911183146, "learning_rate": 1.7817339230628553e-07, "logits/chosen": -2.859375, "logits/rejected": -2.90625, "logps/chosen": -552.0, "logps/rejected": -876.0, "loss": 0.2189, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.46875, "rewards/margins": 3.546875, "rewards/rejected": -7.0, "step": 8420 }, { "epoch": 0.6342637875253931, "grad_norm": 10.989316835419913, "learning_rate": 1.7754469272507914e-07, "logits/chosen": -2.78125, "logits/rejected": -2.484375, "logps/chosen": -504.0, "logps/rejected": -916.0, "loss": 0.1871, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -3.234375, "rewards/margins": 4.0625, "rewards/rejected": -7.28125, "step": 8430 }, { "epoch": 0.6350161763599428, "grad_norm": 13.49315802854144, "learning_rate": 1.7691649298814232e-07, "logits/chosen": -2.90625, "logits/rejected": -2.859375, "logps/chosen": -520.0, "logps/rejected": -844.0, "loss": 0.1935, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.390625, "rewards/margins": 3.3125, "rewards/rejected": -6.6875, "step": 8440 }, { "epoch": 0.6357685651944925, "grad_norm": 10.693059040806757, "learning_rate": 1.7628879742920928e-07, "logits/chosen": -2.8125, "logits/rejected": -2.625, "logps/chosen": -512.0, "logps/rejected": -904.0, "loss": 0.2059, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -3.296875, "rewards/margins": 3.90625, "rewards/rejected": -7.1875, "step": 8450 }, { "epoch": 0.6365209540290422, "grad_norm": 11.411361228173503, "learning_rate": 1.7566161037853594e-07, "logits/chosen": -2.6875, "logits/rejected": -2.625, "logps/chosen": -512.0, "logps/rejected": -880.0, "loss": 0.1983, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.140625, "rewards/margins": 3.84375, "rewards/rejected": -7.0, "step": 8460 }, { "epoch": 0.6372733428635919, "grad_norm": 14.918979305776368, "learning_rate": 1.7503493616287023e-07, "logits/chosen": -2.921875, "logits/rejected": -2.859375, "logps/chosen": -500.0, "logps/rejected": -860.0, "loss": 0.2181, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.234375, "rewards/margins": 3.625, "rewards/rejected": -6.84375, "step": 8470 }, { "epoch": 0.6380257316981416, "grad_norm": 10.32487466998491, "learning_rate": 1.7440877910542225e-07, "logits/chosen": -2.78125, "logits/rejected": -2.578125, "logps/chosen": -524.0, "logps/rejected": -872.0, "loss": 0.1937, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.109375, "rewards/margins": 3.78125, "rewards/rejected": -6.875, "step": 8480 }, { "epoch": 0.6387781205326913, "grad_norm": 12.934439407168218, "learning_rate": 1.7378314352583446e-07, "logits/chosen": -2.75, "logits/rejected": -2.78125, "logps/chosen": -536.0, "logps/rejected": -904.0, "loss": 0.1833, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.296875, "rewards/margins": 3.65625, "rewards/rejected": -6.9375, "step": 8490 }, { "epoch": 0.639530509367241, "grad_norm": 14.450898779783333, "learning_rate": 1.731580337401517e-07, "logits/chosen": -2.84375, "logits/rejected": -2.71875, "logps/chosen": -528.0, "logps/rejected": -872.0, "loss": 0.1968, "rewards/accuracies": 0.90625, "rewards/chosen": -3.328125, "rewards/margins": 3.515625, "rewards/rejected": -6.84375, "step": 8500 }, { "epoch": 0.6402828982017907, "grad_norm": 14.559814322872993, "learning_rate": 1.7253345406079161e-07, "logits/chosen": -2.8125, "logits/rejected": -2.765625, "logps/chosen": -580.0, "logps/rejected": -904.0, "loss": 0.2057, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.78125, "rewards/margins": 3.4375, "rewards/rejected": -7.21875, "step": 8510 }, { "epoch": 0.6410352870363404, "grad_norm": 12.728604517405135, "learning_rate": 1.719094087965148e-07, "logits/chosen": -2.6875, "logits/rejected": -2.546875, "logps/chosen": -544.0, "logps/rejected": -912.0, "loss": 0.1844, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.53125, "rewards/margins": 3.625, "rewards/rejected": -7.15625, "step": 8520 }, { "epoch": 0.64178767587089, "grad_norm": 12.11489187841079, "learning_rate": 1.7128590225239515e-07, "logits/chosen": -2.796875, "logits/rejected": -2.78125, "logps/chosen": -510.0, "logps/rejected": -864.0, "loss": 0.2219, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.265625, "rewards/margins": 3.703125, "rewards/rejected": -6.96875, "step": 8530 }, { "epoch": 0.6425400647054398, "grad_norm": 14.36192278631842, "learning_rate": 1.7066293872979005e-07, "logits/chosen": -2.6875, "logits/rejected": -2.28125, "logps/chosen": -564.0, "logps/rejected": -916.0, "loss": 0.2104, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.484375, "rewards/margins": 3.671875, "rewards/rejected": -7.15625, "step": 8540 }, { "epoch": 0.6432924535399894, "grad_norm": 12.560112776742477, "learning_rate": 1.7004052252631085e-07, "logits/chosen": -2.78125, "logits/rejected": -2.671875, "logps/chosen": -536.0, "logps/rejected": -888.0, "loss": 0.2138, "rewards/accuracies": 0.90625, "rewards/chosen": -3.453125, "rewards/margins": 3.59375, "rewards/rejected": -7.0625, "step": 8550 }, { "epoch": 0.6440448423745392, "grad_norm": 13.212672354571334, "learning_rate": 1.6941865793579302e-07, "logits/chosen": -2.78125, "logits/rejected": -2.5, "logps/chosen": -490.0, "logps/rejected": -900.0, "loss": 0.1985, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.046875, "rewards/margins": 4.0625, "rewards/rejected": -7.09375, "step": 8560 }, { "epoch": 0.6447972312090888, "grad_norm": 10.429375790395905, "learning_rate": 1.687973492482666e-07, "logits/chosen": -2.71875, "logits/rejected": -2.578125, "logps/chosen": -528.0, "logps/rejected": -852.0, "loss": 0.1979, "rewards/accuracies": 0.90625, "rewards/chosen": -3.296875, "rewards/margins": 3.3125, "rewards/rejected": -6.59375, "step": 8570 }, { "epoch": 0.6455496200436386, "grad_norm": 10.34031299971184, "learning_rate": 1.6817660074992694e-07, "logits/chosen": -2.640625, "logits/rejected": -2.84375, "logps/chosen": -510.0, "logps/rejected": -824.0, "loss": 0.1778, "rewards/accuracies": 0.90625, "rewards/chosen": -3.296875, "rewards/margins": 3.421875, "rewards/rejected": -6.71875, "step": 8580 }, { "epoch": 0.6463020088781882, "grad_norm": 11.013136402594386, "learning_rate": 1.6755641672310456e-07, "logits/chosen": -2.78125, "logits/rejected": -2.59375, "logps/chosen": -498.0, "logps/rejected": -860.0, "loss": 0.1891, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.359375, "rewards/margins": 3.609375, "rewards/rejected": -6.96875, "step": 8590 }, { "epoch": 0.647054397712738, "grad_norm": 16.063303875666236, "learning_rate": 1.6693680144623594e-07, "logits/chosen": -2.765625, "logits/rejected": -2.859375, "logps/chosen": -544.0, "logps/rejected": -888.0, "loss": 0.1856, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.53125, "rewards/margins": 3.59375, "rewards/rejected": -7.125, "step": 8600 }, { "epoch": 0.6478067865472876, "grad_norm": 15.372034857876354, "learning_rate": 1.6631775919383398e-07, "logits/chosen": -2.6875, "logits/rejected": -2.5, "logps/chosen": -524.0, "logps/rejected": -904.0, "loss": 0.1992, "rewards/accuracies": 0.9375, "rewards/chosen": -3.28125, "rewards/margins": 4.0, "rewards/rejected": -7.28125, "step": 8610 }, { "epoch": 0.6485591753818374, "grad_norm": 13.532973926416721, "learning_rate": 1.656992942364585e-07, "logits/chosen": -2.890625, "logits/rejected": -2.6875, "logps/chosen": -552.0, "logps/rejected": -916.0, "loss": 0.1781, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.640625, "rewards/margins": 3.53125, "rewards/rejected": -7.1875, "step": 8620 }, { "epoch": 0.649311564216387, "grad_norm": 14.04543514297853, "learning_rate": 1.6508141084068682e-07, "logits/chosen": -2.78125, "logits/rejected": -2.71875, "logps/chosen": -512.0, "logps/rejected": -864.0, "loss": 0.1844, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.34375, "rewards/margins": 3.640625, "rewards/rejected": -7.0, "step": 8630 }, { "epoch": 0.6500639530509367, "grad_norm": 12.231401353451382, "learning_rate": 1.6446411326908415e-07, "logits/chosen": -2.875, "logits/rejected": -2.59375, "logps/chosen": -540.0, "logps/rejected": -928.0, "loss": 0.1906, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.546875, "rewards/margins": 4.09375, "rewards/rejected": -7.65625, "step": 8640 }, { "epoch": 0.6508163418854864, "grad_norm": 10.386067798814706, "learning_rate": 1.6384740578017419e-07, "logits/chosen": -2.859375, "logits/rejected": -2.78125, "logps/chosen": -528.0, "logps/rejected": -912.0, "loss": 0.2106, "rewards/accuracies": 0.90625, "rewards/chosen": -3.453125, "rewards/margins": 3.921875, "rewards/rejected": -7.375, "step": 8650 }, { "epoch": 0.6515687307200361, "grad_norm": 14.722060833646424, "learning_rate": 1.6323129262841016e-07, "logits/chosen": -2.703125, "logits/rejected": -2.90625, "logps/chosen": -488.0, "logps/rejected": -828.0, "loss": 0.1704, "rewards/accuracies": 0.9375, "rewards/chosen": -3.109375, "rewards/margins": 3.5625, "rewards/rejected": -6.65625, "step": 8660 }, { "epoch": 0.6523211195545858, "grad_norm": 16.679500219529533, "learning_rate": 1.626157780641449e-07, "logits/chosen": -2.875, "logits/rejected": -2.8125, "logps/chosen": -508.0, "logps/rejected": -840.0, "loss": 0.1884, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.140625, "rewards/margins": 3.609375, "rewards/rejected": -6.75, "step": 8670 }, { "epoch": 0.6530735083891355, "grad_norm": 12.009104254750707, "learning_rate": 1.6200086633360203e-07, "logits/chosen": -2.75, "logits/rejected": -2.65625, "logps/chosen": -516.0, "logps/rejected": -916.0, "loss": 0.1811, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.265625, "rewards/margins": 4.1875, "rewards/rejected": -7.4375, "step": 8680 }, { "epoch": 0.6538258972236852, "grad_norm": 15.091384082791228, "learning_rate": 1.6138656167884615e-07, "logits/chosen": -2.96875, "logits/rejected": -2.734375, "logps/chosen": -504.0, "logps/rejected": -864.0, "loss": 0.2182, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.328125, "rewards/margins": 3.6875, "rewards/rejected": -7.03125, "step": 8690 }, { "epoch": 0.6545782860582349, "grad_norm": 10.00941576812804, "learning_rate": 1.6077286833775407e-07, "logits/chosen": -2.75, "logits/rejected": -2.765625, "logps/chosen": -502.0, "logps/rejected": -884.0, "loss": 0.1775, "rewards/accuracies": 0.9375, "rewards/chosen": -3.21875, "rewards/margins": 3.859375, "rewards/rejected": -7.0625, "step": 8700 }, { "epoch": 0.6553306748927846, "grad_norm": 12.859743584132628, "learning_rate": 1.6015979054398537e-07, "logits/chosen": -2.8125, "logits/rejected": -2.78125, "logps/chosen": -500.0, "logps/rejected": -824.0, "loss": 0.2027, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.140625, "rewards/margins": 3.46875, "rewards/rejected": -6.625, "step": 8710 }, { "epoch": 0.6560830637273343, "grad_norm": 7.312284428965653, "learning_rate": 1.5954733252695297e-07, "logits/chosen": -2.65625, "logits/rejected": -2.65625, "logps/chosen": -484.0, "logps/rejected": -832.0, "loss": 0.204, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.09375, "rewards/margins": 3.578125, "rewards/rejected": -6.65625, "step": 8720 }, { "epoch": 0.656835452561884, "grad_norm": 8.53401040734989, "learning_rate": 1.5893549851179443e-07, "logits/chosen": -2.78125, "logits/rejected": -2.421875, "logps/chosen": -520.0, "logps/rejected": -884.0, "loss": 0.1824, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.390625, "rewards/margins": 3.671875, "rewards/rejected": -7.0625, "step": 8730 }, { "epoch": 0.6575878413964337, "grad_norm": 14.087811931758406, "learning_rate": 1.5832429271934216e-07, "logits/chosen": -2.75, "logits/rejected": -2.671875, "logps/chosen": -536.0, "logps/rejected": -868.0, "loss": 0.2049, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.46875, "rewards/margins": 3.515625, "rewards/rejected": -6.96875, "step": 8740 }, { "epoch": 0.6583402302309834, "grad_norm": 15.864873683716384, "learning_rate": 1.5771371936609512e-07, "logits/chosen": -2.8125, "logits/rejected": -2.765625, "logps/chosen": -548.0, "logps/rejected": -864.0, "loss": 0.2156, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.65625, "rewards/margins": 3.359375, "rewards/rejected": -7.0, "step": 8750 }, { "epoch": 0.6590926190655331, "grad_norm": 11.138715320052327, "learning_rate": 1.5710378266418909e-07, "logits/chosen": -2.71875, "logits/rejected": -2.40625, "logps/chosen": -524.0, "logps/rejected": -920.0, "loss": 0.2103, "rewards/accuracies": 0.9375, "rewards/chosen": -3.421875, "rewards/margins": 3.96875, "rewards/rejected": -7.40625, "step": 8760 }, { "epoch": 0.6598450079000827, "grad_norm": 13.599514882405696, "learning_rate": 1.5649448682136768e-07, "logits/chosen": -2.65625, "logits/rejected": -2.609375, "logps/chosen": -474.0, "logps/rejected": -800.0, "loss": 0.2, "rewards/accuracies": 0.9375, "rewards/chosen": -2.9375, "rewards/margins": 3.390625, "rewards/rejected": -6.34375, "step": 8770 }, { "epoch": 0.6605973967346325, "grad_norm": 12.24977824800274, "learning_rate": 1.558858360409536e-07, "logits/chosen": -2.703125, "logits/rejected": -2.609375, "logps/chosen": -464.0, "logps/rejected": -808.0, "loss": 0.1966, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -2.9375, "rewards/margins": 3.34375, "rewards/rejected": -6.28125, "step": 8780 }, { "epoch": 0.6613497855691821, "grad_norm": 12.800011473265384, "learning_rate": 1.5527783452181947e-07, "logits/chosen": -2.703125, "logits/rejected": -2.6875, "logps/chosen": -532.0, "logps/rejected": -856.0, "loss": 0.2091, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.40625, "rewards/margins": 3.4375, "rewards/rejected": -6.84375, "step": 8790 }, { "epoch": 0.6621021744037319, "grad_norm": 19.85385293543991, "learning_rate": 1.5467048645835895e-07, "logits/chosen": -2.65625, "logits/rejected": -2.640625, "logps/chosen": -524.0, "logps/rejected": -860.0, "loss": 0.1883, "rewards/accuracies": 0.90625, "rewards/chosen": -3.265625, "rewards/margins": 3.5625, "rewards/rejected": -6.84375, "step": 8800 }, { "epoch": 0.6628545632382815, "grad_norm": 11.963115460518358, "learning_rate": 1.540637960404575e-07, "logits/chosen": -2.734375, "logits/rejected": -2.671875, "logps/chosen": -506.0, "logps/rejected": -808.0, "loss": 0.2013, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.203125, "rewards/margins": 3.09375, "rewards/rejected": -6.28125, "step": 8810 }, { "epoch": 0.6636069520728313, "grad_norm": 14.242870291033535, "learning_rate": 1.53457767453464e-07, "logits/chosen": -2.734375, "logits/rejected": -2.71875, "logps/chosen": -504.0, "logps/rejected": -812.0, "loss": 0.2127, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.140625, "rewards/margins": 3.3125, "rewards/rejected": -6.4375, "step": 8820 }, { "epoch": 0.6643593409073809, "grad_norm": 10.841951027110344, "learning_rate": 1.528524048781613e-07, "logits/chosen": -2.71875, "logits/rejected": -2.796875, "logps/chosen": -464.0, "logps/rejected": -832.0, "loss": 0.1944, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -2.890625, "rewards/margins": 3.75, "rewards/rejected": -6.625, "step": 8830 }, { "epoch": 0.6651117297419307, "grad_norm": 10.304792315119537, "learning_rate": 1.5224771249073787e-07, "logits/chosen": -2.796875, "logits/rejected": -2.53125, "logps/chosen": -504.0, "logps/rejected": -832.0, "loss": 0.1869, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.1875, "rewards/margins": 3.25, "rewards/rejected": -6.4375, "step": 8840 }, { "epoch": 0.6658641185764803, "grad_norm": 16.184679054573397, "learning_rate": 1.5164369446275878e-07, "logits/chosen": -2.78125, "logits/rejected": -2.875, "logps/chosen": -540.0, "logps/rejected": -868.0, "loss": 0.1928, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.5, "rewards/margins": 3.5625, "rewards/rejected": -7.0625, "step": 8850 }, { "epoch": 0.6666165074110301, "grad_norm": 9.61552162749813, "learning_rate": 1.5104035496113672e-07, "logits/chosen": -2.75, "logits/rejected": -2.84375, "logps/chosen": -540.0, "logps/rejected": -884.0, "loss": 0.171, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -3.375, "rewards/margins": 3.765625, "rewards/rejected": -7.125, "step": 8860 }, { "epoch": 0.6673688962455797, "grad_norm": 12.0884478093952, "learning_rate": 1.5043769814810374e-07, "logits/chosen": -2.734375, "logits/rejected": -2.359375, "logps/chosen": -512.0, "logps/rejected": -920.0, "loss": 0.1953, "rewards/accuracies": 0.90625, "rewards/chosen": -3.21875, "rewards/margins": 4.0625, "rewards/rejected": -7.28125, "step": 8870 }, { "epoch": 0.6681212850801294, "grad_norm": 15.438595863260414, "learning_rate": 1.4983572818118213e-07, "logits/chosen": -2.625, "logits/rejected": -2.328125, "logps/chosen": -510.0, "logps/rejected": -880.0, "loss": 0.2079, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.1875, "rewards/margins": 3.609375, "rewards/rejected": -6.8125, "step": 8880 }, { "epoch": 0.6688736739146791, "grad_norm": 16.02770918503472, "learning_rate": 1.4923444921315575e-07, "logits/chosen": -2.671875, "logits/rejected": -2.53125, "logps/chosen": -498.0, "logps/rejected": -884.0, "loss": 0.2061, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -3.09375, "rewards/margins": 3.84375, "rewards/rejected": -6.9375, "step": 8890 }, { "epoch": 0.6696260627492288, "grad_norm": 10.729389818543897, "learning_rate": 1.4863386539204167e-07, "logits/chosen": -2.8125, "logits/rejected": -2.84375, "logps/chosen": -524.0, "logps/rejected": -836.0, "loss": 0.1919, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.234375, "rewards/margins": 3.359375, "rewards/rejected": -6.59375, "step": 8900 }, { "epoch": 0.6703784515837785, "grad_norm": 11.746159573431683, "learning_rate": 1.480339808610614e-07, "logits/chosen": -2.859375, "logits/rejected": -2.671875, "logps/chosen": -486.0, "logps/rejected": -860.0, "loss": 0.2099, "rewards/accuracies": 0.9375, "rewards/chosen": -3.0, "rewards/margins": 3.75, "rewards/rejected": -6.75, "step": 8910 }, { "epoch": 0.6711308404183282, "grad_norm": 14.168879089426735, "learning_rate": 1.47434799758612e-07, "logits/chosen": -2.625, "logits/rejected": -2.6875, "logps/chosen": -516.0, "logps/rejected": -800.0, "loss": 0.208, "rewards/accuracies": 0.875, "rewards/chosen": -3.21875, "rewards/margins": 3.109375, "rewards/rejected": -6.34375, "step": 8920 }, { "epoch": 0.6718832292528779, "grad_norm": 13.018545454917328, "learning_rate": 1.4683632621823822e-07, "logits/chosen": -2.625, "logits/rejected": -2.703125, "logps/chosen": -484.0, "logps/rejected": -772.0, "loss": 0.2058, "rewards/accuracies": 0.9375, "rewards/chosen": -2.875, "rewards/margins": 3.203125, "rewards/rejected": -6.09375, "step": 8930 }, { "epoch": 0.6726356180874276, "grad_norm": 16.84004583231802, "learning_rate": 1.4623856436860322e-07, "logits/chosen": -2.8125, "logits/rejected": -2.578125, "logps/chosen": -478.0, "logps/rejected": -860.0, "loss": 0.2029, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -2.953125, "rewards/margins": 3.71875, "rewards/rejected": -6.65625, "step": 8940 }, { "epoch": 0.6733880069219773, "grad_norm": 13.13121328343648, "learning_rate": 1.4564151833346072e-07, "logits/chosen": -2.75, "logits/rejected": -2.6875, "logps/chosen": -528.0, "logps/rejected": -892.0, "loss": 0.2132, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -3.296875, "rewards/margins": 3.8125, "rewards/rejected": -7.125, "step": 8950 }, { "epoch": 0.674140395756527, "grad_norm": 12.12769346337225, "learning_rate": 1.4504519223162602e-07, "logits/chosen": -2.828125, "logits/rejected": -2.828125, "logps/chosen": -512.0, "logps/rejected": -844.0, "loss": 0.1969, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.25, "rewards/margins": 3.40625, "rewards/rejected": -6.65625, "step": 8960 }, { "epoch": 0.6748927845910767, "grad_norm": 12.92028671878834, "learning_rate": 1.4444959017694827e-07, "logits/chosen": -2.71875, "logits/rejected": -2.8125, "logps/chosen": -516.0, "logps/rejected": -844.0, "loss": 0.1918, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.25, "rewards/margins": 3.4375, "rewards/rejected": -6.6875, "step": 8970 }, { "epoch": 0.6756451734256264, "grad_norm": 10.079310633879503, "learning_rate": 1.4385471627828112e-07, "logits/chosen": -2.75, "logits/rejected": -2.71875, "logps/chosen": -536.0, "logps/rejected": -900.0, "loss": 0.1814, "rewards/accuracies": 0.96875, "rewards/chosen": -3.390625, "rewards/margins": 3.75, "rewards/rejected": -7.125, "step": 8980 }, { "epoch": 0.6763975622601761, "grad_norm": 12.745172780431192, "learning_rate": 1.4326057463945544e-07, "logits/chosen": -2.8125, "logits/rejected": -2.703125, "logps/chosen": -504.0, "logps/rejected": -892.0, "loss": 0.1799, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.25, "rewards/margins": 3.78125, "rewards/rejected": -7.03125, "step": 8990 }, { "epoch": 0.6771499510947258, "grad_norm": 13.720032710209182, "learning_rate": 1.4266716935925025e-07, "logits/chosen": -2.796875, "logits/rejected": -2.828125, "logps/chosen": -510.0, "logps/rejected": -828.0, "loss": 0.1699, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.21875, "rewards/margins": 3.5625, "rewards/rejected": -6.78125, "step": 9000 }, { "epoch": 0.6779023399292754, "grad_norm": 11.970963144307479, "learning_rate": 1.4207450453136434e-07, "logits/chosen": -2.703125, "logits/rejected": -2.84375, "logps/chosen": -524.0, "logps/rejected": -900.0, "loss": 0.2038, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.328125, "rewards/margins": 3.90625, "rewards/rejected": -7.21875, "step": 9010 }, { "epoch": 0.6786547287638252, "grad_norm": 15.840567952336716, "learning_rate": 1.4148258424438912e-07, "logits/chosen": -2.78125, "logits/rejected": -2.875, "logps/chosen": -544.0, "logps/rejected": -904.0, "loss": 0.1913, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.453125, "rewards/margins": 3.765625, "rewards/rejected": -7.21875, "step": 9020 }, { "epoch": 0.6794071175983748, "grad_norm": 18.641993273970186, "learning_rate": 1.4089141258177894e-07, "logits/chosen": -2.703125, "logits/rejected": -2.953125, "logps/chosen": -498.0, "logps/rejected": -824.0, "loss": 0.1972, "rewards/accuracies": 0.9375, "rewards/chosen": -3.109375, "rewards/margins": 3.46875, "rewards/rejected": -6.59375, "step": 9030 }, { "epoch": 0.6801595064329246, "grad_norm": 17.601846328249795, "learning_rate": 1.4030099362182429e-07, "logits/chosen": -2.828125, "logits/rejected": -2.8125, "logps/chosen": -520.0, "logps/rejected": -912.0, "loss": 0.1751, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.34375, "rewards/margins": 4.03125, "rewards/rejected": -7.375, "step": 9040 }, { "epoch": 0.6809118952674742, "grad_norm": 20.85100695141935, "learning_rate": 1.3971133143762255e-07, "logits/chosen": -2.796875, "logits/rejected": -2.703125, "logps/chosen": -502.0, "logps/rejected": -900.0, "loss": 0.2017, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.203125, "rewards/margins": 3.953125, "rewards/rejected": -7.15625, "step": 9050 }, { "epoch": 0.681664284102024, "grad_norm": 11.89756126881172, "learning_rate": 1.3912243009705043e-07, "logits/chosen": -2.734375, "logits/rejected": -2.71875, "logps/chosen": -528.0, "logps/rejected": -844.0, "loss": 0.2137, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.375, "rewards/margins": 3.28125, "rewards/rejected": -6.65625, "step": 9060 }, { "epoch": 0.6824166729365736, "grad_norm": 10.376821979508703, "learning_rate": 1.3853429366273617e-07, "logits/chosen": -2.765625, "logits/rejected": -2.484375, "logps/chosen": -508.0, "logps/rejected": -848.0, "loss": 0.2024, "rewards/accuracies": 0.90625, "rewards/chosen": -3.203125, "rewards/margins": 3.515625, "rewards/rejected": -6.71875, "step": 9070 }, { "epoch": 0.6831690617711234, "grad_norm": 14.058256036812386, "learning_rate": 1.379469261920308e-07, "logits/chosen": -2.609375, "logits/rejected": -2.546875, "logps/chosen": -532.0, "logps/rejected": -888.0, "loss": 0.1812, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.328125, "rewards/margins": 3.828125, "rewards/rejected": -7.15625, "step": 9080 }, { "epoch": 0.683921450605673, "grad_norm": 12.929793805385328, "learning_rate": 1.373603317369807e-07, "logits/chosen": -2.671875, "logits/rejected": -2.609375, "logps/chosen": -540.0, "logps/rejected": -900.0, "loss": 0.1878, "rewards/accuracies": 0.9375, "rewards/chosen": -3.5, "rewards/margins": 3.6875, "rewards/rejected": -7.1875, "step": 9090 }, { "epoch": 0.6846738394402228, "grad_norm": 13.594135234529695, "learning_rate": 1.3677451434429945e-07, "logits/chosen": -2.765625, "logits/rejected": -2.875, "logps/chosen": -504.0, "logps/rejected": -876.0, "loss": 0.1873, "rewards/accuracies": 0.9375, "rewards/chosen": -3.28125, "rewards/margins": 3.859375, "rewards/rejected": -7.15625, "step": 9100 }, { "epoch": 0.6854262282747724, "grad_norm": 11.957325831285708, "learning_rate": 1.3618947805533993e-07, "logits/chosen": -2.59375, "logits/rejected": -2.609375, "logps/chosen": -536.0, "logps/rejected": -876.0, "loss": 0.1791, "rewards/accuracies": 0.9375, "rewards/chosen": -3.53125, "rewards/margins": 3.515625, "rewards/rejected": -7.0625, "step": 9110 }, { "epoch": 0.6861786171093222, "grad_norm": 9.886855721361252, "learning_rate": 1.3560522690606657e-07, "logits/chosen": -2.84375, "logits/rejected": -2.5, "logps/chosen": -532.0, "logps/rejected": -880.0, "loss": 0.1856, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.59375, "rewards/margins": 3.515625, "rewards/rejected": -7.125, "step": 9120 }, { "epoch": 0.6869310059438718, "grad_norm": 13.741041073613257, "learning_rate": 1.3502176492702732e-07, "logits/chosen": -2.734375, "logits/rejected": -2.703125, "logps/chosen": -532.0, "logps/rejected": -924.0, "loss": 0.1929, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.328125, "rewards/margins": 4.125, "rewards/rejected": -7.4375, "step": 9130 }, { "epoch": 0.6876833947784214, "grad_norm": 18.180361997290014, "learning_rate": 1.344390961433257e-07, "logits/chosen": -2.75, "logits/rejected": -2.625, "logps/chosen": -516.0, "logps/rejected": -884.0, "loss": 0.1993, "rewards/accuracies": 0.9375, "rewards/chosen": -3.234375, "rewards/margins": 3.875, "rewards/rejected": -7.125, "step": 9140 }, { "epoch": 0.6884357836129712, "grad_norm": 11.393134782039885, "learning_rate": 1.338572245745937e-07, "logits/chosen": -2.75, "logits/rejected": -2.8125, "logps/chosen": -482.0, "logps/rejected": -856.0, "loss": 0.2005, "rewards/accuracies": 0.9375, "rewards/chosen": -3.125, "rewards/margins": 3.8125, "rewards/rejected": -6.9375, "step": 9150 }, { "epoch": 0.6891881724475208, "grad_norm": 12.835262215534366, "learning_rate": 1.3327615423496324e-07, "logits/chosen": -2.71875, "logits/rejected": -2.625, "logps/chosen": -528.0, "logps/rejected": -900.0, "loss": 0.1846, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.296875, "rewards/margins": 3.921875, "rewards/rejected": -7.21875, "step": 9160 }, { "epoch": 0.6899405612820706, "grad_norm": 10.318980464903818, "learning_rate": 1.326958891330388e-07, "logits/chosen": -2.75, "logits/rejected": -2.578125, "logps/chosen": -540.0, "logps/rejected": -924.0, "loss": 0.1969, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.484375, "rewards/margins": 3.78125, "rewards/rejected": -7.28125, "step": 9170 }, { "epoch": 0.6906929501166202, "grad_norm": 15.020933279825002, "learning_rate": 1.3211643327187028e-07, "logits/chosen": -2.609375, "logits/rejected": -2.53125, "logps/chosen": -532.0, "logps/rejected": -912.0, "loss": 0.2037, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.5, "rewards/margins": 3.890625, "rewards/rejected": -7.375, "step": 9180 }, { "epoch": 0.69144533895117, "grad_norm": 15.31761167602629, "learning_rate": 1.3153779064892417e-07, "logits/chosen": -2.625, "logits/rejected": -2.828125, "logps/chosen": -532.0, "logps/rejected": -884.0, "loss": 0.1976, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.359375, "rewards/margins": 3.75, "rewards/rejected": -7.09375, "step": 9190 }, { "epoch": 0.6921977277857196, "grad_norm": 18.800184627446868, "learning_rate": 1.309599652560574e-07, "logits/chosen": -2.828125, "logits/rejected": -2.6875, "logps/chosen": -492.0, "logps/rejected": -892.0, "loss": 0.1824, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.203125, "rewards/margins": 4.0, "rewards/rejected": -7.21875, "step": 9200 }, { "epoch": 0.6929501166202694, "grad_norm": 12.57322210871062, "learning_rate": 1.3038296107948872e-07, "logits/chosen": -2.765625, "logits/rejected": -2.5, "logps/chosen": -512.0, "logps/rejected": -884.0, "loss": 0.1892, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.109375, "rewards/margins": 3.75, "rewards/rejected": -6.875, "step": 9210 }, { "epoch": 0.693702505454819, "grad_norm": 13.298682942666238, "learning_rate": 1.2980678209977158e-07, "logits/chosen": -2.65625, "logits/rejected": -2.515625, "logps/chosen": -528.0, "logps/rejected": -888.0, "loss": 0.1921, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.453125, "rewards/margins": 3.609375, "rewards/rejected": -7.0625, "step": 9220 }, { "epoch": 0.6944548942893688, "grad_norm": 10.802032722863032, "learning_rate": 1.2923143229176696e-07, "logits/chosen": -2.6875, "logits/rejected": -2.5625, "logps/chosen": -540.0, "logps/rejected": -872.0, "loss": 0.2056, "rewards/accuracies": 0.90625, "rewards/chosen": -3.375, "rewards/margins": 3.625, "rewards/rejected": -7.0, "step": 9230 }, { "epoch": 0.6952072831239184, "grad_norm": 10.302952018659623, "learning_rate": 1.286569156246154e-07, "logits/chosen": -2.8125, "logits/rejected": -2.59375, "logps/chosen": -524.0, "logps/rejected": -900.0, "loss": 0.1734, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -3.421875, "rewards/margins": 3.859375, "rewards/rejected": -7.28125, "step": 9240 }, { "epoch": 0.6959596719584681, "grad_norm": 14.615217121688161, "learning_rate": 1.2808323606171006e-07, "logits/chosen": -2.8125, "logits/rejected": -2.75, "logps/chosen": -536.0, "logps/rejected": -836.0, "loss": 0.1975, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.46875, "rewards/margins": 3.3125, "rewards/rejected": -6.78125, "step": 9250 }, { "epoch": 0.6967120607930178, "grad_norm": 11.203534988546581, "learning_rate": 1.2751039756066907e-07, "logits/chosen": -2.65625, "logits/rejected": -2.453125, "logps/chosen": -504.0, "logps/rejected": -852.0, "loss": 0.1851, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.03125, "rewards/margins": 3.609375, "rewards/rejected": -6.65625, "step": 9260 }, { "epoch": 0.6974644496275675, "grad_norm": 13.965159007705639, "learning_rate": 1.2693840407330838e-07, "logits/chosen": -2.65625, "logits/rejected": -2.78125, "logps/chosen": -488.0, "logps/rejected": -816.0, "loss": 0.2014, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.0625, "rewards/margins": 3.40625, "rewards/rejected": -6.46875, "step": 9270 }, { "epoch": 0.6982168384621172, "grad_norm": 9.863501010248342, "learning_rate": 1.2636725954561479e-07, "logits/chosen": -2.75, "logits/rejected": -2.515625, "logps/chosen": -512.0, "logps/rejected": -920.0, "loss": 0.2023, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -3.109375, "rewards/margins": 4.28125, "rewards/rejected": -7.40625, "step": 9280 }, { "epoch": 0.6989692272966669, "grad_norm": 13.089623803156288, "learning_rate": 1.257969679177177e-07, "logits/chosen": -2.71875, "logits/rejected": -2.46875, "logps/chosen": -490.0, "logps/rejected": -888.0, "loss": 0.195, "rewards/accuracies": 0.9375, "rewards/chosen": -3.0, "rewards/margins": 3.90625, "rewards/rejected": -6.90625, "step": 9290 }, { "epoch": 0.6997216161312166, "grad_norm": 15.875787659799618, "learning_rate": 1.2522753312386347e-07, "logits/chosen": -2.8125, "logits/rejected": -2.53125, "logps/chosen": -516.0, "logps/rejected": -876.0, "loss": 0.186, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.25, "rewards/margins": 3.78125, "rewards/rejected": -7.03125, "step": 9300 }, { "epoch": 0.7004740049657663, "grad_norm": 13.855618228084984, "learning_rate": 1.2465895909238698e-07, "logits/chosen": -2.609375, "logits/rejected": -2.578125, "logps/chosen": -536.0, "logps/rejected": -852.0, "loss": 0.1941, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.296875, "rewards/margins": 3.484375, "rewards/rejected": -6.78125, "step": 9310 }, { "epoch": 0.701226393800316, "grad_norm": 13.304899177594217, "learning_rate": 1.24091249745685e-07, "logits/chosen": -2.625, "logits/rejected": -2.71875, "logps/chosen": -504.0, "logps/rejected": -856.0, "loss": 0.186, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.078125, "rewards/margins": 3.703125, "rewards/rejected": -6.78125, "step": 9320 }, { "epoch": 0.7019787826348657, "grad_norm": 13.860210569036854, "learning_rate": 1.2352440900018943e-07, "logits/chosen": -2.703125, "logits/rejected": -2.546875, "logps/chosen": -492.0, "logps/rejected": -880.0, "loss": 0.1826, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.09375, "rewards/margins": 3.859375, "rewards/rejected": -6.9375, "step": 9330 }, { "epoch": 0.7027311714694154, "grad_norm": 14.644449094769174, "learning_rate": 1.2295844076633973e-07, "logits/chosen": -2.765625, "logits/rejected": -2.78125, "logps/chosen": -506.0, "logps/rejected": -872.0, "loss": 0.2002, "rewards/accuracies": 0.9375, "rewards/chosen": -3.25, "rewards/margins": 3.8125, "rewards/rejected": -7.0625, "step": 9340 }, { "epoch": 0.7034835603039651, "grad_norm": 14.774616027253217, "learning_rate": 1.2239334894855622e-07, "logits/chosen": -2.71875, "logits/rejected": -2.578125, "logps/chosen": -504.0, "logps/rejected": -864.0, "loss": 0.2089, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.234375, "rewards/margins": 3.734375, "rewards/rejected": -6.96875, "step": 9350 }, { "epoch": 0.7042359491385148, "grad_norm": 11.142573647970615, "learning_rate": 1.2182913744521332e-07, "logits/chosen": -2.796875, "logits/rejected": -2.59375, "logps/chosen": -512.0, "logps/rejected": -848.0, "loss": 0.1953, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.28125, "rewards/margins": 3.53125, "rewards/rejected": -6.8125, "step": 9360 }, { "epoch": 0.7049883379730645, "grad_norm": 13.653051638419283, "learning_rate": 1.2126581014861227e-07, "logits/chosen": -2.609375, "logits/rejected": -2.640625, "logps/chosen": -536.0, "logps/rejected": -876.0, "loss": 0.2058, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.28125, "rewards/margins": 3.65625, "rewards/rejected": -6.9375, "step": 9370 }, { "epoch": 0.7057407268076141, "grad_norm": 8.15182523749014, "learning_rate": 1.207033709449545e-07, "logits/chosen": -2.59375, "logits/rejected": -2.765625, "logps/chosen": -528.0, "logps/rejected": -868.0, "loss": 0.2009, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.484375, "rewards/margins": 3.609375, "rewards/rejected": -7.09375, "step": 9380 }, { "epoch": 0.7064931156421639, "grad_norm": 10.089421348169376, "learning_rate": 1.2014182371431486e-07, "logits/chosen": -2.71875, "logits/rejected": -2.609375, "logps/chosen": -494.0, "logps/rejected": -852.0, "loss": 0.1905, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.046875, "rewards/margins": 3.84375, "rewards/rejected": -6.875, "step": 9390 }, { "epoch": 0.7072455044767135, "grad_norm": 10.305912150504387, "learning_rate": 1.1958117233061465e-07, "logits/chosen": -2.671875, "logits/rejected": -2.5, "logps/chosen": -506.0, "logps/rejected": -872.0, "loss": 0.1812, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.125, "rewards/margins": 3.859375, "rewards/rejected": -6.96875, "step": 9400 }, { "epoch": 0.7079978933112633, "grad_norm": 12.299345461335282, "learning_rate": 1.1902142066159529e-07, "logits/chosen": -2.71875, "logits/rejected": -2.65625, "logps/chosen": -500.0, "logps/rejected": -896.0, "loss": 0.1931, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.0, "rewards/margins": 4.0, "rewards/rejected": -7.0, "step": 9410 }, { "epoch": 0.7087502821458129, "grad_norm": 12.087321439254229, "learning_rate": 1.1846257256879116e-07, "logits/chosen": -2.671875, "logits/rejected": -2.59375, "logps/chosen": -508.0, "logps/rejected": -848.0, "loss": 0.2014, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.15625, "rewards/margins": 3.546875, "rewards/rejected": -6.6875, "step": 9420 }, { "epoch": 0.7095026709803627, "grad_norm": 12.139345616583777, "learning_rate": 1.1790463190750313e-07, "logits/chosen": -2.765625, "logits/rejected": -2.84375, "logps/chosen": -532.0, "logps/rejected": -900.0, "loss": 0.2125, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.421875, "rewards/margins": 3.96875, "rewards/rejected": -7.40625, "step": 9430 }, { "epoch": 0.7102550598149123, "grad_norm": 12.224712694545008, "learning_rate": 1.1734760252677228e-07, "logits/chosen": -2.53125, "logits/rejected": -2.59375, "logps/chosen": -502.0, "logps/rejected": -836.0, "loss": 0.1935, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.078125, "rewards/margins": 3.515625, "rewards/rejected": -6.59375, "step": 9440 }, { "epoch": 0.7110074486494621, "grad_norm": 14.940543247990009, "learning_rate": 1.1679148826935284e-07, "logits/chosen": -2.6875, "logits/rejected": -2.453125, "logps/chosen": -504.0, "logps/rejected": -812.0, "loss": 0.2126, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.25, "rewards/margins": 3.1875, "rewards/rejected": -6.4375, "step": 9450 }, { "epoch": 0.7117598374840117, "grad_norm": 17.42609930518133, "learning_rate": 1.1623629297168599e-07, "logits/chosen": -2.71875, "logits/rejected": -2.5625, "logps/chosen": -492.0, "logps/rejected": -860.0, "loss": 0.1963, "rewards/accuracies": 0.9375, "rewards/chosen": -3.109375, "rewards/margins": 3.75, "rewards/rejected": -6.84375, "step": 9460 }, { "epoch": 0.7125122263185615, "grad_norm": 16.865559190224058, "learning_rate": 1.1568202046387334e-07, "logits/chosen": -2.703125, "logits/rejected": -2.609375, "logps/chosen": -536.0, "logps/rejected": -844.0, "loss": 0.202, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.484375, "rewards/margins": 3.09375, "rewards/rejected": -6.5625, "step": 9470 }, { "epoch": 0.7132646151531111, "grad_norm": 15.71898492627018, "learning_rate": 1.1512867456965036e-07, "logits/chosen": -2.6875, "logits/rejected": -2.59375, "logps/chosen": -508.0, "logps/rejected": -876.0, "loss": 0.1917, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.078125, "rewards/margins": 3.890625, "rewards/rejected": -6.96875, "step": 9480 }, { "epoch": 0.7140170039876608, "grad_norm": 12.829342052248592, "learning_rate": 1.1457625910636042e-07, "logits/chosen": -2.75, "logits/rejected": -2.3125, "logps/chosen": -492.0, "logps/rejected": -844.0, "loss": 0.1859, "rewards/accuracies": 0.90625, "rewards/chosen": -3.09375, "rewards/margins": 3.640625, "rewards/rejected": -6.71875, "step": 9490 }, { "epoch": 0.7147693928222105, "grad_norm": 15.341007710223852, "learning_rate": 1.1402477788492796e-07, "logits/chosen": -2.8125, "logits/rejected": -2.703125, "logps/chosen": -496.0, "logps/rejected": -864.0, "loss": 0.2211, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.140625, "rewards/margins": 3.75, "rewards/rejected": -6.90625, "step": 9500 }, { "epoch": 0.7155217816567602, "grad_norm": 7.668693214191918, "learning_rate": 1.134742347098323e-07, "logits/chosen": -2.75, "logits/rejected": -2.8125, "logps/chosen": -456.0, "logps/rejected": -828.0, "loss": 0.1765, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -2.78125, "rewards/margins": 3.890625, "rewards/rejected": -6.6875, "step": 9510 }, { "epoch": 0.7162741704913099, "grad_norm": 15.247866020733747, "learning_rate": 1.1292463337908185e-07, "logits/chosen": -2.75, "logits/rejected": -2.703125, "logps/chosen": -520.0, "logps/rejected": -852.0, "loss": 0.1909, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.328125, "rewards/margins": 3.5, "rewards/rejected": -6.84375, "step": 9520 }, { "epoch": 0.7170265593258596, "grad_norm": 9.750347506083376, "learning_rate": 1.1237597768418714e-07, "logits/chosen": -2.671875, "logits/rejected": -2.65625, "logps/chosen": -512.0, "logps/rejected": -868.0, "loss": 0.1913, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.109375, "rewards/margins": 3.828125, "rewards/rejected": -6.9375, "step": 9530 }, { "epoch": 0.7177789481604093, "grad_norm": 13.176377196673503, "learning_rate": 1.1182827141013549e-07, "logits/chosen": -2.703125, "logits/rejected": -2.703125, "logps/chosen": -510.0, "logps/rejected": -828.0, "loss": 0.2031, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.265625, "rewards/margins": 3.40625, "rewards/rejected": -6.65625, "step": 9540 }, { "epoch": 0.718531336994959, "grad_norm": 14.001163273969555, "learning_rate": 1.112815183353642e-07, "logits/chosen": -2.671875, "logits/rejected": -2.734375, "logps/chosen": -544.0, "logps/rejected": -892.0, "loss": 0.1979, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.453125, "rewards/margins": 3.703125, "rewards/rejected": -7.15625, "step": 9550 }, { "epoch": 0.7192837258295087, "grad_norm": 10.035929560827112, "learning_rate": 1.1073572223173489e-07, "logits/chosen": -2.59375, "logits/rejected": -2.46875, "logps/chosen": -510.0, "logps/rejected": -832.0, "loss": 0.1747, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.1875, "rewards/margins": 3.3125, "rewards/rejected": -6.5, "step": 9560 }, { "epoch": 0.7200361146640584, "grad_norm": 12.451025303569383, "learning_rate": 1.1019088686450731e-07, "logits/chosen": -2.78125, "logits/rejected": -2.75, "logps/chosen": -516.0, "logps/rejected": -828.0, "loss": 0.1999, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.25, "rewards/margins": 3.25, "rewards/rejected": -6.5, "step": 9570 }, { "epoch": 0.7207885034986081, "grad_norm": 13.137214502633451, "learning_rate": 1.0964701599231341e-07, "logits/chosen": -2.796875, "logits/rejected": -2.75, "logps/chosen": -510.0, "logps/rejected": -904.0, "loss": 0.17, "rewards/accuracies": 0.9375, "rewards/chosen": -3.21875, "rewards/margins": 4.125, "rewards/rejected": -7.34375, "step": 9580 }, { "epoch": 0.7215408923331578, "grad_norm": 16.5871470802553, "learning_rate": 1.091041133671316e-07, "logits/chosen": -2.828125, "logits/rejected": -2.859375, "logps/chosen": -556.0, "logps/rejected": -900.0, "loss": 0.1858, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.75, "rewards/margins": 3.609375, "rewards/rejected": -7.375, "step": 9590 }, { "epoch": 0.7222932811677075, "grad_norm": 12.744751191337405, "learning_rate": 1.0856218273426049e-07, "logits/chosen": -2.8125, "logits/rejected": -2.734375, "logps/chosen": -510.0, "logps/rejected": -900.0, "loss": 0.1856, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.28125, "rewards/margins": 4.21875, "rewards/rejected": -7.46875, "step": 9600 }, { "epoch": 0.7230456700022572, "grad_norm": 11.231415611086979, "learning_rate": 1.0802122783229323e-07, "logits/chosen": -2.765625, "logits/rejected": -2.578125, "logps/chosen": -520.0, "logps/rejected": -900.0, "loss": 0.1613, "rewards/accuracies": 0.96875, "rewards/chosen": -3.375, "rewards/margins": 3.953125, "rewards/rejected": -7.3125, "step": 9610 }, { "epoch": 0.7237980588368068, "grad_norm": 14.300762388564625, "learning_rate": 1.0748125239309197e-07, "logits/chosen": -2.625, "logits/rejected": -2.359375, "logps/chosen": -544.0, "logps/rejected": -924.0, "loss": 0.1809, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.5, "rewards/margins": 4.03125, "rewards/rejected": -7.53125, "step": 9620 }, { "epoch": 0.7245504476713566, "grad_norm": 10.859346179379054, "learning_rate": 1.0694226014176167e-07, "logits/chosen": -2.75, "logits/rejected": -2.546875, "logps/chosen": -510.0, "logps/rejected": -872.0, "loss": 0.1689, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.3125, "rewards/margins": 3.734375, "rewards/rejected": -7.0625, "step": 9630 }, { "epoch": 0.7253028365059062, "grad_norm": 12.840881838284005, "learning_rate": 1.0640425479662465e-07, "logits/chosen": -2.828125, "logits/rejected": -2.6875, "logps/chosen": -496.0, "logps/rejected": -920.0, "loss": 0.1879, "rewards/accuracies": 0.96875, "rewards/chosen": -3.171875, "rewards/margins": 4.1875, "rewards/rejected": -7.375, "step": 9640 }, { "epoch": 0.726055225340456, "grad_norm": 11.073132518652582, "learning_rate": 1.0586724006919496e-07, "logits/chosen": -2.859375, "logits/rejected": -2.828125, "logps/chosen": -512.0, "logps/rejected": -884.0, "loss": 0.193, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.296875, "rewards/margins": 3.75, "rewards/rejected": -7.0625, "step": 9650 }, { "epoch": 0.7268076141750056, "grad_norm": 13.29822718030276, "learning_rate": 1.0533121966415257e-07, "logits/chosen": -2.71875, "logits/rejected": -2.984375, "logps/chosen": -504.0, "logps/rejected": -876.0, "loss": 0.184, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.203125, "rewards/margins": 3.890625, "rewards/rejected": -7.09375, "step": 9660 }, { "epoch": 0.7275600030095554, "grad_norm": 12.420915035949875, "learning_rate": 1.0479619727931827e-07, "logits/chosen": -2.765625, "logits/rejected": -2.796875, "logps/chosen": -508.0, "logps/rejected": -916.0, "loss": 0.1897, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.171875, "rewards/margins": 4.1875, "rewards/rejected": -7.375, "step": 9670 }, { "epoch": 0.728312391844105, "grad_norm": 10.987082630121515, "learning_rate": 1.0426217660562758e-07, "logits/chosen": -2.75, "logits/rejected": -2.65625, "logps/chosen": -520.0, "logps/rejected": -844.0, "loss": 0.1887, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.234375, "rewards/margins": 3.4375, "rewards/rejected": -6.65625, "step": 9680 }, { "epoch": 0.7290647806786548, "grad_norm": 9.528098729919652, "learning_rate": 1.0372916132710555e-07, "logits/chosen": -2.625, "logits/rejected": -2.78125, "logps/chosen": -556.0, "logps/rejected": -852.0, "loss": 0.1777, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.390625, "rewards/margins": 3.46875, "rewards/rejected": -6.84375, "step": 9690 }, { "epoch": 0.7298171695132044, "grad_norm": 14.265073461703265, "learning_rate": 1.031971551208416e-07, "logits/chosen": -2.625, "logits/rejected": -2.828125, "logps/chosen": -484.0, "logps/rejected": -836.0, "loss": 0.1993, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -3.0, "rewards/margins": 3.671875, "rewards/rejected": -6.6875, "step": 9700 }, { "epoch": 0.7305695583477542, "grad_norm": 8.704468908768735, "learning_rate": 1.026661616569637e-07, "logits/chosen": -2.71875, "logits/rejected": -2.65625, "logps/chosen": -508.0, "logps/rejected": -844.0, "loss": 0.1776, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.25, "rewards/margins": 3.421875, "rewards/rejected": -6.6875, "step": 9710 }, { "epoch": 0.7313219471823038, "grad_norm": 10.930273313573139, "learning_rate": 1.0213618459861321e-07, "logits/chosen": -2.671875, "logits/rejected": -2.59375, "logps/chosen": -502.0, "logps/rejected": -848.0, "loss": 0.1995, "rewards/accuracies": 0.9375, "rewards/chosen": -3.28125, "rewards/margins": 3.4375, "rewards/rejected": -6.71875, "step": 9720 }, { "epoch": 0.7320743360168535, "grad_norm": 12.320906412080113, "learning_rate": 1.0160722760192e-07, "logits/chosen": -2.671875, "logits/rejected": -2.90625, "logps/chosen": -500.0, "logps/rejected": -860.0, "loss": 0.1992, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.09375, "rewards/margins": 3.890625, "rewards/rejected": -7.0, "step": 9730 }, { "epoch": 0.7328267248514032, "grad_norm": 11.822385560087218, "learning_rate": 1.010792943159763e-07, "logits/chosen": -2.671875, "logits/rejected": -2.59375, "logps/chosen": -536.0, "logps/rejected": -864.0, "loss": 0.2007, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.296875, "rewards/margins": 3.390625, "rewards/rejected": -6.6875, "step": 9740 }, { "epoch": 0.7335791136859529, "grad_norm": 13.813129858671333, "learning_rate": 1.0055238838281275e-07, "logits/chosen": -2.796875, "logits/rejected": -2.703125, "logps/chosen": -502.0, "logps/rejected": -852.0, "loss": 0.1808, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.25, "rewards/margins": 3.375, "rewards/rejected": -6.625, "step": 9750 }, { "epoch": 0.7343315025205026, "grad_norm": 15.277163197291923, "learning_rate": 1.000265134373722e-07, "logits/chosen": -2.765625, "logits/rejected": -2.8125, "logps/chosen": -520.0, "logps/rejected": -832.0, "loss": 0.206, "rewards/accuracies": 0.90625, "rewards/chosen": -3.28125, "rewards/margins": 3.34375, "rewards/rejected": -6.625, "step": 9760 }, { "epoch": 0.7350838913550523, "grad_norm": 15.140577779829083, "learning_rate": 9.950167310748516e-08, "logits/chosen": -2.75, "logits/rejected": -2.65625, "logps/chosen": -524.0, "logps/rejected": -868.0, "loss": 0.2055, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.203125, "rewards/margins": 3.703125, "rewards/rejected": -6.90625, "step": 9770 }, { "epoch": 0.735836280189602, "grad_norm": 12.29918175483103, "learning_rate": 9.897787101384492e-08, "logits/chosen": -2.734375, "logits/rejected": -2.71875, "logps/chosen": -512.0, "logps/rejected": -880.0, "loss": 0.227, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.3125, "rewards/margins": 3.703125, "rewards/rejected": -7.03125, "step": 9780 }, { "epoch": 0.7365886690241517, "grad_norm": 11.332762109348012, "learning_rate": 9.845511076998195e-08, "logits/chosen": -2.578125, "logits/rejected": -2.609375, "logps/chosen": -488.0, "logps/rejected": -836.0, "loss": 0.2022, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.984375, "rewards/margins": 3.625, "rewards/rejected": -6.625, "step": 9790 }, { "epoch": 0.7373410578587014, "grad_norm": 10.123377625278309, "learning_rate": 9.79333959822397e-08, "logits/chosen": -2.8125, "logits/rejected": -2.453125, "logps/chosen": -504.0, "logps/rejected": -896.0, "loss": 0.2004, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -3.21875, "rewards/margins": 3.890625, "rewards/rejected": -7.09375, "step": 9800 }, { "epoch": 0.738093446693251, "grad_norm": 18.82478882680944, "learning_rate": 9.741273024974919e-08, "logits/chosen": -2.78125, "logits/rejected": -2.453125, "logps/chosen": -516.0, "logps/rejected": -904.0, "loss": 0.1994, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.3125, "rewards/margins": 3.90625, "rewards/rejected": -7.21875, "step": 9810 }, { "epoch": 0.7388458355278008, "grad_norm": 10.704018640780653, "learning_rate": 9.68931171644044e-08, "logits/chosen": -2.71875, "logits/rejected": -2.5, "logps/chosen": -524.0, "logps/rejected": -872.0, "loss": 0.2112, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.4375, "rewards/margins": 3.5625, "rewards/rejected": -7.0, "step": 9820 }, { "epoch": 0.7395982243623505, "grad_norm": 18.043860901712495, "learning_rate": 9.637456031083746e-08, "logits/chosen": -2.515625, "logits/rejected": -2.359375, "logps/chosen": -520.0, "logps/rejected": -848.0, "loss": 0.1954, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.140625, "rewards/margins": 3.4375, "rewards/rejected": -6.5625, "step": 9830 }, { "epoch": 0.7403506131969002, "grad_norm": 10.881022166446265, "learning_rate": 9.585706326639383e-08, "logits/chosen": -2.625, "logits/rejected": -2.65625, "logps/chosen": -510.0, "logps/rejected": -820.0, "loss": 0.1887, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.109375, "rewards/margins": 3.375, "rewards/rejected": -6.5, "step": 9840 }, { "epoch": 0.7411030020314499, "grad_norm": 17.471843008144283, "learning_rate": 9.534062960110803e-08, "logits/chosen": -2.6875, "logits/rejected": -2.546875, "logps/chosen": -508.0, "logps/rejected": -888.0, "loss": 0.1913, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.265625, "rewards/margins": 3.828125, "rewards/rejected": -7.09375, "step": 9850 }, { "epoch": 0.7418553908659995, "grad_norm": 11.264168548227694, "learning_rate": 9.482526287767836e-08, "logits/chosen": -2.71875, "logits/rejected": -2.53125, "logps/chosen": -520.0, "logps/rejected": -852.0, "loss": 0.1943, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.359375, "rewards/margins": 3.453125, "rewards/rejected": -6.8125, "step": 9860 }, { "epoch": 0.7426077797005493, "grad_norm": 16.14878796239615, "learning_rate": 9.431096665144268e-08, "logits/chosen": -2.609375, "logits/rejected": -2.546875, "logps/chosen": -544.0, "logps/rejected": -868.0, "loss": 0.1922, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.421875, "rewards/margins": 3.625, "rewards/rejected": -7.0625, "step": 9870 }, { "epoch": 0.7433601685350989, "grad_norm": 14.163864533293426, "learning_rate": 9.379774447035408e-08, "logits/chosen": -2.75, "logits/rejected": -2.328125, "logps/chosen": -528.0, "logps/rejected": -908.0, "loss": 0.2029, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.421875, "rewards/margins": 3.875, "rewards/rejected": -7.28125, "step": 9880 }, { "epoch": 0.7441125573696487, "grad_norm": 15.751524102132326, "learning_rate": 9.328559987495602e-08, "logits/chosen": -2.6875, "logits/rejected": -2.3125, "logps/chosen": -504.0, "logps/rejected": -896.0, "loss": 0.2063, "rewards/accuracies": 0.9375, "rewards/chosen": -3.21875, "rewards/margins": 3.78125, "rewards/rejected": -7.0, "step": 9890 }, { "epoch": 0.7448649462041983, "grad_norm": 10.18328554473717, "learning_rate": 9.277453639835795e-08, "logits/chosen": -2.78125, "logits/rejected": -2.890625, "logps/chosen": -524.0, "logps/rejected": -840.0, "loss": 0.192, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.25, "rewards/margins": 3.3125, "rewards/rejected": -6.5625, "step": 9900 }, { "epoch": 0.745617335038748, "grad_norm": 14.059345617949896, "learning_rate": 9.226455756621152e-08, "logits/chosen": -2.6875, "logits/rejected": -2.875, "logps/chosen": -548.0, "logps/rejected": -828.0, "loss": 0.1918, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.375, "rewards/margins": 3.234375, "rewards/rejected": -6.625, "step": 9910 }, { "epoch": 0.7463697238732977, "grad_norm": 12.698249481880408, "learning_rate": 9.175566689668506e-08, "logits/chosen": -2.625, "logits/rejected": -2.46875, "logps/chosen": -504.0, "logps/rejected": -852.0, "loss": 0.2175, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.109375, "rewards/margins": 3.65625, "rewards/rejected": -6.75, "step": 9920 }, { "epoch": 0.7471221127078475, "grad_norm": 11.939837394902945, "learning_rate": 9.124786790044076e-08, "logits/chosen": -2.703125, "logits/rejected": -2.75, "logps/chosen": -502.0, "logps/rejected": -896.0, "loss": 0.198, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.171875, "rewards/margins": 4.0625, "rewards/rejected": -7.25, "step": 9930 }, { "epoch": 0.7478745015423971, "grad_norm": 9.443464133059143, "learning_rate": 9.074116408060931e-08, "logits/chosen": -2.734375, "logits/rejected": -2.671875, "logps/chosen": -536.0, "logps/rejected": -892.0, "loss": 0.1862, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.390625, "rewards/margins": 3.75, "rewards/rejected": -7.15625, "step": 9940 }, { "epoch": 0.7486268903769469, "grad_norm": 12.764821701045845, "learning_rate": 9.023555893276613e-08, "logits/chosen": -2.609375, "logits/rejected": -2.59375, "logps/chosen": -528.0, "logps/rejected": -880.0, "loss": 0.1921, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.15625, "rewards/margins": 3.46875, "rewards/rejected": -6.625, "step": 9950 }, { "epoch": 0.7493792792114965, "grad_norm": 12.304571861813562, "learning_rate": 8.973105594490766e-08, "logits/chosen": -2.609375, "logits/rejected": -2.46875, "logps/chosen": -516.0, "logps/rejected": -844.0, "loss": 0.1907, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.296875, "rewards/margins": 3.484375, "rewards/rejected": -6.78125, "step": 9960 }, { "epoch": 0.7501316680460463, "grad_norm": 13.357328662964697, "learning_rate": 8.922765859742654e-08, "logits/chosen": -2.71875, "logits/rejected": -2.78125, "logps/chosen": -492.0, "logps/rejected": -856.0, "loss": 0.1874, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.0625, "rewards/margins": 3.828125, "rewards/rejected": -6.90625, "step": 9970 }, { "epoch": 0.7508840568805959, "grad_norm": 12.051703470557332, "learning_rate": 8.872537036308802e-08, "logits/chosen": -2.640625, "logits/rejected": -2.578125, "logps/chosen": -544.0, "logps/rejected": -912.0, "loss": 0.1641, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.359375, "rewards/margins": 3.859375, "rewards/rejected": -7.21875, "step": 9980 }, { "epoch": 0.7516364457151455, "grad_norm": 12.224504500520016, "learning_rate": 8.822419470700624e-08, "logits/chosen": -2.765625, "logits/rejected": -2.734375, "logps/chosen": -512.0, "logps/rejected": -892.0, "loss": 0.1964, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -3.3125, "rewards/margins": 3.859375, "rewards/rejected": -7.15625, "step": 9990 }, { "epoch": 0.7523888345496953, "grad_norm": 11.341058397561435, "learning_rate": 8.772413508661972e-08, "logits/chosen": -2.609375, "logits/rejected": -2.515625, "logps/chosen": -532.0, "logps/rejected": -880.0, "loss": 0.1841, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.375, "rewards/margins": 3.546875, "rewards/rejected": -6.9375, "step": 10000 }, { "epoch": 0.7523888345496953, "eval_logits/chosen": -2.671875, "eval_logits/rejected": -2.625, "eval_logps/chosen": -536.0, "eval_logps/rejected": -848.0, "eval_loss": 0.27069464325904846, "eval_rewards/accuracies": 0.8784698843955994, "eval_rewards/chosen": -3.421875, "eval_rewards/margins": 3.296875, "eval_rewards/rejected": -6.71875, "eval_runtime": 2788.7574, "eval_samples_per_second": 33.884, "eval_steps_per_second": 0.53, "step": 10000 }, { "epoch": 0.7531412233842449, "grad_norm": 14.856380738608333, "learning_rate": 8.722519495166799e-08, "logits/chosen": -2.765625, "logits/rejected": -2.671875, "logps/chosen": -504.0, "logps/rejected": -824.0, "loss": 0.1809, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.234375, "rewards/margins": 3.484375, "rewards/rejected": -6.71875, "step": 10010 }, { "epoch": 0.7538936122187947, "grad_norm": 18.276227615719822, "learning_rate": 8.672737774416761e-08, "logits/chosen": -2.796875, "logits/rejected": -2.53125, "logps/chosen": -540.0, "logps/rejected": -912.0, "loss": 0.1971, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -3.46875, "rewards/margins": 3.828125, "rewards/rejected": -7.3125, "step": 10020 }, { "epoch": 0.7546460010533443, "grad_norm": 13.680263335280905, "learning_rate": 8.623068689838836e-08, "logits/chosen": -2.78125, "logits/rejected": -2.6875, "logps/chosen": -532.0, "logps/rejected": -896.0, "loss": 0.197, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.375, "rewards/margins": 3.765625, "rewards/rejected": -7.125, "step": 10030 }, { "epoch": 0.7553983898878941, "grad_norm": 16.874915245677755, "learning_rate": 8.57351258408299e-08, "logits/chosen": -2.78125, "logits/rejected": -2.875, "logps/chosen": -510.0, "logps/rejected": -908.0, "loss": 0.1782, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.140625, "rewards/margins": 4.09375, "rewards/rejected": -7.21875, "step": 10040 }, { "epoch": 0.7561507787224437, "grad_norm": 13.920275046877688, "learning_rate": 8.52406979901975e-08, "logits/chosen": -2.734375, "logits/rejected": -2.65625, "logps/chosen": -516.0, "logps/rejected": -884.0, "loss": 0.1893, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.28125, "rewards/margins": 3.734375, "rewards/rejected": -7.0, "step": 10050 }, { "epoch": 0.7569031675569935, "grad_norm": 12.663211389384871, "learning_rate": 8.474740675737921e-08, "logits/chosen": -2.78125, "logits/rejected": -2.703125, "logps/chosen": -486.0, "logps/rejected": -888.0, "loss": 0.1841, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.09375, "rewards/margins": 3.96875, "rewards/rejected": -7.0625, "step": 10060 }, { "epoch": 0.7576555563915431, "grad_norm": 12.950087410444254, "learning_rate": 8.425525554542167e-08, "logits/chosen": -2.6875, "logits/rejected": -2.53125, "logps/chosen": -536.0, "logps/rejected": -904.0, "loss": 0.2212, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.34375, "rewards/margins": 3.765625, "rewards/rejected": -7.125, "step": 10070 }, { "epoch": 0.7584079452260929, "grad_norm": 12.416956814533648, "learning_rate": 8.376424774950691e-08, "logits/chosen": -2.796875, "logits/rejected": -2.625, "logps/chosen": -470.0, "logps/rejected": -856.0, "loss": 0.2057, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.0625, "rewards/margins": 3.65625, "rewards/rejected": -6.71875, "step": 10080 }, { "epoch": 0.7591603340606425, "grad_norm": 12.754173831254326, "learning_rate": 8.327438675692921e-08, "logits/chosen": -2.796875, "logits/rejected": -2.71875, "logps/chosen": -480.0, "logps/rejected": -848.0, "loss": 0.2029, "rewards/accuracies": 0.875, "rewards/chosen": -3.203125, "rewards/margins": 3.5, "rewards/rejected": -6.6875, "step": 10090 }, { "epoch": 0.7599127228951922, "grad_norm": 11.332805153321118, "learning_rate": 8.278567594707098e-08, "logits/chosen": -2.75, "logits/rejected": -2.8125, "logps/chosen": -528.0, "logps/rejected": -856.0, "loss": 0.1828, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.25, "rewards/margins": 3.546875, "rewards/rejected": -6.78125, "step": 10100 }, { "epoch": 0.7606651117297419, "grad_norm": 11.095856953984631, "learning_rate": 8.229811869138036e-08, "logits/chosen": -2.640625, "logits/rejected": -2.546875, "logps/chosen": -512.0, "logps/rejected": -928.0, "loss": 0.2036, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.171875, "rewards/margins": 4.1875, "rewards/rejected": -7.375, "step": 10110 }, { "epoch": 0.7614175005642916, "grad_norm": 12.615624907752109, "learning_rate": 8.181171835334733e-08, "logits/chosen": -2.71875, "logits/rejected": -2.796875, "logps/chosen": -498.0, "logps/rejected": -868.0, "loss": 0.1908, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.171875, "rewards/margins": 3.765625, "rewards/rejected": -6.9375, "step": 10120 }, { "epoch": 0.7621698893988413, "grad_norm": 14.642768623371223, "learning_rate": 8.132647828848052e-08, "logits/chosen": -2.796875, "logits/rejected": -2.484375, "logps/chosen": -488.0, "logps/rejected": -872.0, "loss": 0.1852, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.0625, "rewards/margins": 3.78125, "rewards/rejected": -6.84375, "step": 10130 }, { "epoch": 0.762922278233391, "grad_norm": 11.382126409244572, "learning_rate": 8.084240184428465e-08, "logits/chosen": -2.71875, "logits/rejected": -2.46875, "logps/chosen": -516.0, "logps/rejected": -888.0, "loss": 0.189, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.15625, "rewards/margins": 3.875, "rewards/rejected": -7.03125, "step": 10140 }, { "epoch": 0.7636746670679407, "grad_norm": 10.026471802124, "learning_rate": 8.035949236023668e-08, "logits/chosen": -2.796875, "logits/rejected": -2.5625, "logps/chosen": -492.0, "logps/rejected": -876.0, "loss": 0.184, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.09375, "rewards/margins": 3.84375, "rewards/rejected": -6.9375, "step": 10150 }, { "epoch": 0.7644270559024904, "grad_norm": 14.292885739292183, "learning_rate": 7.987775316776311e-08, "logits/chosen": -2.75, "logits/rejected": -2.796875, "logps/chosen": -486.0, "logps/rejected": -824.0, "loss": 0.1901, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.0625, "rewards/margins": 3.515625, "rewards/rejected": -6.59375, "step": 10160 }, { "epoch": 0.7651794447370401, "grad_norm": 10.57774158718183, "learning_rate": 7.939718759021729e-08, "logits/chosen": -2.65625, "logits/rejected": -2.546875, "logps/chosen": -516.0, "logps/rejected": -804.0, "loss": 0.1977, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.09375, "rewards/margins": 3.125, "rewards/rejected": -6.21875, "step": 10170 }, { "epoch": 0.7659318335715898, "grad_norm": 17.06097325195383, "learning_rate": 7.891779894285597e-08, "logits/chosen": -2.65625, "logits/rejected": -2.796875, "logps/chosen": -520.0, "logps/rejected": -848.0, "loss": 0.2036, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.25, "rewards/margins": 3.5, "rewards/rejected": -6.75, "step": 10180 }, { "epoch": 0.7666842224061395, "grad_norm": 11.9975288352918, "learning_rate": 7.843959053281663e-08, "logits/chosen": -2.71875, "logits/rejected": -2.796875, "logps/chosen": -512.0, "logps/rejected": -856.0, "loss": 0.1761, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.15625, "rewards/margins": 3.6875, "rewards/rejected": -6.84375, "step": 10190 }, { "epoch": 0.7674366112406892, "grad_norm": 14.347479004735309, "learning_rate": 7.796256565909487e-08, "logits/chosen": -2.59375, "logits/rejected": -2.46875, "logps/chosen": -532.0, "logps/rejected": -916.0, "loss": 0.1776, "rewards/accuracies": 0.90625, "rewards/chosen": -3.359375, "rewards/margins": 3.890625, "rewards/rejected": -7.25, "step": 10200 }, { "epoch": 0.7681890000752389, "grad_norm": 11.608197149879242, "learning_rate": 7.748672761252123e-08, "logits/chosen": -2.75, "logits/rejected": -2.65625, "logps/chosen": -510.0, "logps/rejected": -864.0, "loss": 0.1796, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.359375, "rewards/margins": 3.703125, "rewards/rejected": -7.0625, "step": 10210 }, { "epoch": 0.7689413889097886, "grad_norm": 10.759780622435853, "learning_rate": 7.701207967573911e-08, "logits/chosen": -2.53125, "logits/rejected": -2.578125, "logps/chosen": -524.0, "logps/rejected": -868.0, "loss": 0.1901, "rewards/accuracies": 0.9375, "rewards/chosen": -3.3125, "rewards/margins": 3.671875, "rewards/rejected": -6.96875, "step": 10220 }, { "epoch": 0.7696937777443382, "grad_norm": 12.761630455627909, "learning_rate": 7.653862512318146e-08, "logits/chosen": -2.90625, "logits/rejected": -2.734375, "logps/chosen": -504.0, "logps/rejected": -860.0, "loss": 0.2, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.375, "rewards/margins": 3.46875, "rewards/rejected": -6.84375, "step": 10230 }, { "epoch": 0.770446166578888, "grad_norm": 17.654097859458716, "learning_rate": 7.606636722104845e-08, "logits/chosen": -2.78125, "logits/rejected": -2.6875, "logps/chosen": -482.0, "logps/rejected": -856.0, "loss": 0.1731, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.078125, "rewards/margins": 3.78125, "rewards/rejected": -6.84375, "step": 10240 }, { "epoch": 0.7711985554134376, "grad_norm": 12.784090366731455, "learning_rate": 7.559530922728527e-08, "logits/chosen": -2.59375, "logits/rejected": -2.453125, "logps/chosen": -532.0, "logps/rejected": -924.0, "loss": 0.1832, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.390625, "rewards/margins": 3.984375, "rewards/rejected": -7.375, "step": 10250 }, { "epoch": 0.7719509442479874, "grad_norm": 13.127300886835316, "learning_rate": 7.512545439155904e-08, "logits/chosen": -2.640625, "logits/rejected": -2.5, "logps/chosen": -520.0, "logps/rejected": -892.0, "loss": 0.1911, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.25, "rewards/margins": 3.671875, "rewards/rejected": -6.90625, "step": 10260 }, { "epoch": 0.772703333082537, "grad_norm": 15.739599111909449, "learning_rate": 7.46568059552369e-08, "logits/chosen": -2.65625, "logits/rejected": -2.625, "logps/chosen": -544.0, "logps/rejected": -856.0, "loss": 0.184, "rewards/accuracies": 0.9375, "rewards/chosen": -3.484375, "rewards/margins": 3.375, "rewards/rejected": -6.875, "step": 10270 }, { "epoch": 0.7734557219170868, "grad_norm": 10.097136985364255, "learning_rate": 7.418936715136334e-08, "logits/chosen": -2.671875, "logits/rejected": -2.6875, "logps/chosen": -506.0, "logps/rejected": -892.0, "loss": 0.1767, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -3.171875, "rewards/margins": 4.03125, "rewards/rejected": -7.1875, "step": 10280 }, { "epoch": 0.7742081107516364, "grad_norm": 10.715377602793371, "learning_rate": 7.372314120463798e-08, "logits/chosen": -2.65625, "logits/rejected": -2.5625, "logps/chosen": -516.0, "logps/rejected": -900.0, "loss": 0.1818, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.328125, "rewards/margins": 3.96875, "rewards/rejected": -7.3125, "step": 10290 }, { "epoch": 0.7749604995861862, "grad_norm": 21.94417590294997, "learning_rate": 7.325813133139361e-08, "logits/chosen": -2.71875, "logits/rejected": -2.625, "logps/chosen": -540.0, "logps/rejected": -904.0, "loss": 0.1855, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.46875, "rewards/margins": 3.671875, "rewards/rejected": -7.125, "step": 10300 }, { "epoch": 0.7757128884207358, "grad_norm": 15.338850056524397, "learning_rate": 7.279434073957349e-08, "logits/chosen": -2.625, "logits/rejected": -2.578125, "logps/chosen": -544.0, "logps/rejected": -932.0, "loss": 0.1891, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.4375, "rewards/margins": 4.0625, "rewards/rejected": -7.5, "step": 10310 }, { "epoch": 0.7764652772552856, "grad_norm": 14.884269636533245, "learning_rate": 7.233177262870946e-08, "logits/chosen": -2.8125, "logits/rejected": -2.5, "logps/chosen": -536.0, "logps/rejected": -904.0, "loss": 0.1926, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.515625, "rewards/margins": 3.640625, "rewards/rejected": -7.15625, "step": 10320 }, { "epoch": 0.7772176660898352, "grad_norm": 11.490521713815857, "learning_rate": 7.187043018990016e-08, "logits/chosen": -2.609375, "logits/rejected": -2.65625, "logps/chosen": -556.0, "logps/rejected": -880.0, "loss": 0.1767, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.546875, "rewards/margins": 3.484375, "rewards/rejected": -7.03125, "step": 10330 }, { "epoch": 0.7779700549243849, "grad_norm": 10.494680356284237, "learning_rate": 7.141031660578839e-08, "logits/chosen": -2.703125, "logits/rejected": -2.78125, "logps/chosen": -536.0, "logps/rejected": -884.0, "loss": 0.1795, "rewards/accuracies": 0.90625, "rewards/chosen": -3.390625, "rewards/margins": 3.6875, "rewards/rejected": -7.0625, "step": 10340 }, { "epoch": 0.7787224437589346, "grad_norm": 12.121561793153582, "learning_rate": 7.095143505053982e-08, "logits/chosen": -2.6875, "logits/rejected": -2.5625, "logps/chosen": -540.0, "logps/rejected": -888.0, "loss": 0.1756, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.421875, "rewards/margins": 3.609375, "rewards/rejected": -7.03125, "step": 10350 }, { "epoch": 0.7794748325934843, "grad_norm": 12.02511664653878, "learning_rate": 7.049378868982065e-08, "logits/chosen": -2.703125, "logits/rejected": -2.875, "logps/chosen": -480.0, "logps/rejected": -868.0, "loss": 0.1801, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.015625, "rewards/margins": 3.9375, "rewards/rejected": -6.9375, "step": 10360 }, { "epoch": 0.780227221428034, "grad_norm": 12.880852335400098, "learning_rate": 7.003738068077564e-08, "logits/chosen": -2.75, "logits/rejected": -2.515625, "logps/chosen": -496.0, "logps/rejected": -872.0, "loss": 0.1789, "rewards/accuracies": 0.90625, "rewards/chosen": -3.34375, "rewards/margins": 3.609375, "rewards/rejected": -6.9375, "step": 10370 }, { "epoch": 0.7809796102625837, "grad_norm": 12.766197504312602, "learning_rate": 6.958221417200705e-08, "logits/chosen": -2.84375, "logits/rejected": -2.6875, "logps/chosen": -516.0, "logps/rejected": -888.0, "loss": 0.1883, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.140625, "rewards/margins": 3.875, "rewards/rejected": -7.03125, "step": 10380 }, { "epoch": 0.7817319990971334, "grad_norm": 14.69078303092729, "learning_rate": 6.912829230355208e-08, "logits/chosen": -2.8125, "logits/rejected": -2.8125, "logps/chosen": -532.0, "logps/rejected": -912.0, "loss": 0.1792, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.296875, "rewards/margins": 3.859375, "rewards/rejected": -7.15625, "step": 10390 }, { "epoch": 0.7824843879316831, "grad_norm": 13.735587853688621, "learning_rate": 6.867561820686187e-08, "logits/chosen": -2.671875, "logits/rejected": -2.765625, "logps/chosen": -528.0, "logps/rejected": -908.0, "loss": 0.1725, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.3125, "rewards/margins": 3.984375, "rewards/rejected": -7.28125, "step": 10400 }, { "epoch": 0.7832367767662328, "grad_norm": 17.1186513787871, "learning_rate": 6.822419500477947e-08, "logits/chosen": -2.65625, "logits/rejected": -2.65625, "logps/chosen": -516.0, "logps/rejected": -868.0, "loss": 0.1973, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.15625, "rewards/margins": 3.8125, "rewards/rejected": -6.96875, "step": 10410 }, { "epoch": 0.7839891656007825, "grad_norm": 15.054336342858894, "learning_rate": 6.777402581151825e-08, "logits/chosen": -2.75, "logits/rejected": -2.53125, "logps/chosen": -532.0, "logps/rejected": -892.0, "loss": 0.2161, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.5, "rewards/margins": 3.640625, "rewards/rejected": -7.125, "step": 10420 }, { "epoch": 0.7847415544353322, "grad_norm": 13.36551644016061, "learning_rate": 6.732511373264107e-08, "logits/chosen": -2.6875, "logits/rejected": -2.65625, "logps/chosen": -498.0, "logps/rejected": -864.0, "loss": 0.1975, "rewards/accuracies": 0.9375, "rewards/chosen": -3.0625, "rewards/margins": 3.875, "rewards/rejected": -6.9375, "step": 10430 }, { "epoch": 0.7854939432698819, "grad_norm": 12.597519646700718, "learning_rate": 6.687746186503796e-08, "logits/chosen": -2.765625, "logits/rejected": -2.65625, "logps/chosen": -520.0, "logps/rejected": -920.0, "loss": 0.2052, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.375, "rewards/margins": 3.90625, "rewards/rejected": -7.28125, "step": 10440 }, { "epoch": 0.7862463321044316, "grad_norm": 13.93083166006577, "learning_rate": 6.64310732969053e-08, "logits/chosen": -2.671875, "logits/rejected": -2.734375, "logps/chosen": -528.0, "logps/rejected": -844.0, "loss": 0.1912, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.40625, "rewards/margins": 3.421875, "rewards/rejected": -6.8125, "step": 10450 }, { "epoch": 0.7869987209389813, "grad_norm": 14.511751280150314, "learning_rate": 6.598595110772467e-08, "logits/chosen": -2.796875, "logits/rejected": -2.796875, "logps/chosen": -508.0, "logps/rejected": -860.0, "loss": 0.1675, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.25, "rewards/margins": 3.671875, "rewards/rejected": -6.9375, "step": 10460 }, { "epoch": 0.7877511097735309, "grad_norm": 10.938671080750341, "learning_rate": 6.554209836824081e-08, "logits/chosen": -2.59375, "logits/rejected": -2.546875, "logps/chosen": -520.0, "logps/rejected": -900.0, "loss": 0.1953, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.3125, "rewards/margins": 4.0, "rewards/rejected": -7.3125, "step": 10470 }, { "epoch": 0.7885034986080807, "grad_norm": 13.098025595517955, "learning_rate": 6.509951814044151e-08, "logits/chosen": -2.71875, "logits/rejected": -2.609375, "logps/chosen": -524.0, "logps/rejected": -852.0, "loss": 0.1866, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.34375, "rewards/margins": 3.4375, "rewards/rejected": -6.78125, "step": 10480 }, { "epoch": 0.7892558874426303, "grad_norm": 14.073670507101081, "learning_rate": 6.465821347753555e-08, "logits/chosen": -2.75, "logits/rejected": -2.71875, "logps/chosen": -512.0, "logps/rejected": -848.0, "loss": 0.1817, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.3125, "rewards/margins": 3.46875, "rewards/rejected": -6.78125, "step": 10490 }, { "epoch": 0.7900082762771801, "grad_norm": 13.750210169951577, "learning_rate": 6.421818742393217e-08, "logits/chosen": -2.703125, "logits/rejected": -2.546875, "logps/chosen": -536.0, "logps/rejected": -872.0, "loss": 0.1863, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.4375, "rewards/margins": 3.359375, "rewards/rejected": -6.78125, "step": 10500 }, { "epoch": 0.7907606651117297, "grad_norm": 11.504377324130047, "learning_rate": 6.377944301522004e-08, "logits/chosen": -2.671875, "logits/rejected": -2.5, "logps/chosen": -516.0, "logps/rejected": -872.0, "loss": 0.1781, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.1875, "rewards/margins": 3.765625, "rewards/rejected": -6.96875, "step": 10510 }, { "epoch": 0.7915130539462795, "grad_norm": 10.387686100440364, "learning_rate": 6.3341983278146e-08, "logits/chosen": -2.5625, "logits/rejected": -2.46875, "logps/chosen": -536.0, "logps/rejected": -896.0, "loss": 0.2049, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.546875, "rewards/margins": 3.6875, "rewards/rejected": -7.21875, "step": 10520 }, { "epoch": 0.7922654427808291, "grad_norm": 10.154206058495117, "learning_rate": 6.290581123059441e-08, "logits/chosen": -2.734375, "logits/rejected": -2.578125, "logps/chosen": -512.0, "logps/rejected": -876.0, "loss": 0.1978, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.265625, "rewards/margins": 3.875, "rewards/rejected": -7.125, "step": 10530 }, { "epoch": 0.7930178316153789, "grad_norm": 14.717109752633336, "learning_rate": 6.247092988156652e-08, "logits/chosen": -2.734375, "logits/rejected": -2.9375, "logps/chosen": -556.0, "logps/rejected": -856.0, "loss": 0.1882, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.5625, "rewards/margins": 3.375, "rewards/rejected": -6.9375, "step": 10540 }, { "epoch": 0.7937702204499285, "grad_norm": 14.350251747101877, "learning_rate": 6.203734223115922e-08, "logits/chosen": -2.703125, "logits/rejected": -2.53125, "logps/chosen": -544.0, "logps/rejected": -940.0, "loss": 0.1672, "rewards/accuracies": 0.9375, "rewards/chosen": -3.515625, "rewards/margins": 3.921875, "rewards/rejected": -7.4375, "step": 10550 }, { "epoch": 0.7945226092844783, "grad_norm": 10.537931877826397, "learning_rate": 6.160505127054475e-08, "logits/chosen": -2.59375, "logits/rejected": -2.53125, "logps/chosen": -498.0, "logps/rejected": -912.0, "loss": 0.2055, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.171875, "rewards/margins": 4.0, "rewards/rejected": -7.1875, "step": 10560 }, { "epoch": 0.7952749981190279, "grad_norm": 15.056006737511535, "learning_rate": 6.117405998194991e-08, "logits/chosen": -2.640625, "logits/rejected": -2.84375, "logps/chosen": -504.0, "logps/rejected": -884.0, "loss": 0.1842, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.171875, "rewards/margins": 3.890625, "rewards/rejected": -7.0625, "step": 10570 }, { "epoch": 0.7960273869535776, "grad_norm": 14.01409115507867, "learning_rate": 6.074437133863547e-08, "logits/chosen": -2.65625, "logits/rejected": -2.40625, "logps/chosen": -492.0, "logps/rejected": -884.0, "loss": 0.1998, "rewards/accuracies": 0.9375, "rewards/chosen": -3.125, "rewards/margins": 4.09375, "rewards/rejected": -7.21875, "step": 10580 }, { "epoch": 0.7967797757881273, "grad_norm": 11.303923492499267, "learning_rate": 6.031598830487586e-08, "logits/chosen": -2.71875, "logits/rejected": -2.71875, "logps/chosen": -520.0, "logps/rejected": -848.0, "loss": 0.1806, "rewards/accuracies": 0.875, "rewards/chosen": -3.328125, "rewards/margins": 3.40625, "rewards/rejected": -6.75, "step": 10590 }, { "epoch": 0.797532164622677, "grad_norm": 9.869482659181234, "learning_rate": 5.98889138359383e-08, "logits/chosen": -2.65625, "logits/rejected": -2.828125, "logps/chosen": -528.0, "logps/rejected": -900.0, "loss": 0.1984, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.34375, "rewards/margins": 3.984375, "rewards/rejected": -7.3125, "step": 10600 }, { "epoch": 0.7982845534572267, "grad_norm": 16.661856001910156, "learning_rate": 5.946315087806294e-08, "logits/chosen": -2.578125, "logits/rejected": -2.515625, "logps/chosen": -512.0, "logps/rejected": -852.0, "loss": 0.2089, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.40625, "rewards/margins": 3.34375, "rewards/rejected": -6.75, "step": 10610 }, { "epoch": 0.7990369422917764, "grad_norm": 12.284312647804917, "learning_rate": 5.903870236844211e-08, "logits/chosen": -2.640625, "logits/rejected": -2.640625, "logps/chosen": -536.0, "logps/rejected": -856.0, "loss": 0.2195, "rewards/accuracies": 0.90625, "rewards/chosen": -3.5, "rewards/margins": 3.328125, "rewards/rejected": -6.8125, "step": 10620 }, { "epoch": 0.7997893311263261, "grad_norm": 10.418550561939133, "learning_rate": 5.861557123520011e-08, "logits/chosen": -2.625, "logits/rejected": -2.515625, "logps/chosen": -504.0, "logps/rejected": -912.0, "loss": 0.1941, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.375, "rewards/margins": 4.03125, "rewards/rejected": -7.40625, "step": 10630 }, { "epoch": 0.8005417199608758, "grad_norm": 11.975233721711344, "learning_rate": 5.819376039737348e-08, "logits/chosen": -2.71875, "logits/rejected": -2.46875, "logps/chosen": -504.0, "logps/rejected": -900.0, "loss": 0.1666, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.140625, "rewards/margins": 4.15625, "rewards/rejected": -7.28125, "step": 10640 }, { "epoch": 0.8012941087954255, "grad_norm": 11.625808566780595, "learning_rate": 5.7773272764889966e-08, "logits/chosen": -2.875, "logits/rejected": -2.65625, "logps/chosen": -512.0, "logps/rejected": -872.0, "loss": 0.1879, "rewards/accuracies": 0.9375, "rewards/chosen": -3.328125, "rewards/margins": 3.59375, "rewards/rejected": -6.9375, "step": 10650 }, { "epoch": 0.8020464976299752, "grad_norm": 9.583351904730316, "learning_rate": 5.735411123854952e-08, "logits/chosen": -2.609375, "logits/rejected": -2.578125, "logps/chosen": -536.0, "logps/rejected": -884.0, "loss": 0.1865, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.421875, "rewards/margins": 3.59375, "rewards/rejected": -7.0, "step": 10660 }, { "epoch": 0.8027988864645249, "grad_norm": 8.403560077391695, "learning_rate": 5.693627871000337e-08, "logits/chosen": -2.703125, "logits/rejected": -2.625, "logps/chosen": -512.0, "logps/rejected": -856.0, "loss": 0.1947, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.21875, "rewards/margins": 3.75, "rewards/rejected": -6.96875, "step": 10670 }, { "epoch": 0.8035512752990746, "grad_norm": 11.100285135248782, "learning_rate": 5.651977806173452e-08, "logits/chosen": -2.671875, "logits/rejected": -2.65625, "logps/chosen": -548.0, "logps/rejected": -856.0, "loss": 0.1837, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.515625, "rewards/margins": 3.28125, "rewards/rejected": -6.78125, "step": 10680 }, { "epoch": 0.8043036641336243, "grad_norm": 14.2432951667194, "learning_rate": 5.610461216703796e-08, "logits/chosen": -2.65625, "logits/rejected": -2.59375, "logps/chosen": -524.0, "logps/rejected": -864.0, "loss": 0.1926, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -3.375, "rewards/margins": 3.421875, "rewards/rejected": -6.8125, "step": 10690 }, { "epoch": 0.805056052968174, "grad_norm": 11.28534163694424, "learning_rate": 5.569078389000048e-08, "logits/chosen": -2.703125, "logits/rejected": -2.59375, "logps/chosen": -510.0, "logps/rejected": -876.0, "loss": 0.1845, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.125, "rewards/margins": 3.6875, "rewards/rejected": -6.8125, "step": 10700 }, { "epoch": 0.8058084418027236, "grad_norm": 12.273446249157596, "learning_rate": 5.5278296085481125e-08, "logits/chosen": -2.75, "logits/rejected": -2.609375, "logps/chosen": -520.0, "logps/rejected": -884.0, "loss": 0.1785, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.265625, "rewards/margins": 3.796875, "rewards/rejected": -7.0625, "step": 10710 }, { "epoch": 0.8065608306372734, "grad_norm": 12.59536743876848, "learning_rate": 5.486715159909166e-08, "logits/chosen": -2.71875, "logits/rejected": -2.71875, "logps/chosen": -496.0, "logps/rejected": -844.0, "loss": 0.1782, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.140625, "rewards/margins": 3.578125, "rewards/rejected": -6.71875, "step": 10720 }, { "epoch": 0.807313219471823, "grad_norm": 12.16700630027233, "learning_rate": 5.4457353267176545e-08, "logits/chosen": -2.703125, "logits/rejected": -2.78125, "logps/chosen": -512.0, "logps/rejected": -884.0, "loss": 0.1927, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.171875, "rewards/margins": 3.96875, "rewards/rejected": -7.15625, "step": 10730 }, { "epoch": 0.8080656083063728, "grad_norm": 13.1672812783811, "learning_rate": 5.4048903916793676e-08, "logits/chosen": -2.78125, "logits/rejected": -2.734375, "logps/chosen": -512.0, "logps/rejected": -864.0, "loss": 0.2098, "rewards/accuracies": 0.9375, "rewards/chosen": -3.234375, "rewards/margins": 3.53125, "rewards/rejected": -6.78125, "step": 10740 }, { "epoch": 0.8088179971409224, "grad_norm": 11.158905160779634, "learning_rate": 5.3641806365694765e-08, "logits/chosen": -2.734375, "logits/rejected": -2.671875, "logps/chosen": -520.0, "logps/rejected": -872.0, "loss": 0.1848, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -3.28125, "rewards/margins": 3.71875, "rewards/rejected": -7.0, "step": 10750 }, { "epoch": 0.8095703859754722, "grad_norm": 12.271278815487282, "learning_rate": 5.323606342230591e-08, "logits/chosen": -2.75, "logits/rejected": -2.609375, "logps/chosen": -492.0, "logps/rejected": -884.0, "loss": 0.1972, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -3.109375, "rewards/margins": 4.0, "rewards/rejected": -7.125, "step": 10760 }, { "epoch": 0.8103227748100218, "grad_norm": 10.65084872284258, "learning_rate": 5.283167788570836e-08, "logits/chosen": -2.71875, "logits/rejected": -2.671875, "logps/chosen": -486.0, "logps/rejected": -844.0, "loss": 0.1893, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.0625, "rewards/margins": 3.703125, "rewards/rejected": -6.78125, "step": 10770 }, { "epoch": 0.8110751636445716, "grad_norm": 15.061683148135662, "learning_rate": 5.2428652545618954e-08, "logits/chosen": -2.75, "logits/rejected": -2.703125, "logps/chosen": -508.0, "logps/rejected": -896.0, "loss": 0.1933, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.140625, "rewards/margins": 3.9375, "rewards/rejected": -7.09375, "step": 10780 }, { "epoch": 0.8118275524791212, "grad_norm": 13.473316900405004, "learning_rate": 5.202699018237094e-08, "logits/chosen": -2.765625, "logits/rejected": -2.9375, "logps/chosen": -506.0, "logps/rejected": -852.0, "loss": 0.1949, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.21875, "rewards/margins": 3.671875, "rewards/rejected": -6.875, "step": 10790 }, { "epoch": 0.812579941313671, "grad_norm": 9.765829961139504, "learning_rate": 5.162669356689511e-08, "logits/chosen": -2.8125, "logits/rejected": -2.78125, "logps/chosen": -510.0, "logps/rejected": -844.0, "loss": 0.1823, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.25, "rewards/margins": 3.546875, "rewards/rejected": -6.8125, "step": 10800 }, { "epoch": 0.8133323301482206, "grad_norm": 12.482632594002045, "learning_rate": 5.122776546070015e-08, "logits/chosen": -2.71875, "logits/rejected": -2.75, "logps/chosen": -520.0, "logps/rejected": -864.0, "loss": 0.1918, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.296875, "rewards/margins": 3.53125, "rewards/rejected": -6.8125, "step": 10810 }, { "epoch": 0.8140847189827704, "grad_norm": 13.004654119961458, "learning_rate": 5.083020861585413e-08, "logits/chosen": -2.75, "logits/rejected": -2.609375, "logps/chosen": -482.0, "logps/rejected": -896.0, "loss": 0.1867, "rewards/accuracies": 0.90625, "rewards/chosen": -3.078125, "rewards/margins": 4.09375, "rewards/rejected": -7.1875, "step": 10820 }, { "epoch": 0.81483710781732, "grad_norm": 18.127057166551545, "learning_rate": 5.0434025774965e-08, "logits/chosen": -2.828125, "logits/rejected": -2.796875, "logps/chosen": -516.0, "logps/rejected": -860.0, "loss": 0.1693, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.296875, "rewards/margins": 3.703125, "rewards/rejected": -7.0, "step": 10830 }, { "epoch": 0.8155894966518696, "grad_norm": 12.60876386490739, "learning_rate": 5.003921967116201e-08, "logits/chosen": -2.8125, "logits/rejected": -2.828125, "logps/chosen": -506.0, "logps/rejected": -864.0, "loss": 0.1882, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -3.296875, "rewards/margins": 3.5625, "rewards/rejected": -6.84375, "step": 10840 }, { "epoch": 0.8163418854864194, "grad_norm": 12.484911940633994, "learning_rate": 4.9645793028076975e-08, "logits/chosen": -2.75, "logits/rejected": -2.671875, "logps/chosen": -502.0, "logps/rejected": -872.0, "loss": 0.2191, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -3.171875, "rewards/margins": 3.859375, "rewards/rejected": -7.0625, "step": 10850 }, { "epoch": 0.817094274320969, "grad_norm": 16.775182356009726, "learning_rate": 4.925374855982495e-08, "logits/chosen": -2.6875, "logits/rejected": -2.625, "logps/chosen": -528.0, "logps/rejected": -916.0, "loss": 0.2015, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -3.421875, "rewards/margins": 4.09375, "rewards/rejected": -7.5, "step": 10860 }, { "epoch": 0.8178466631555188, "grad_norm": 9.806278909630153, "learning_rate": 4.886308897098621e-08, "logits/chosen": -2.703125, "logits/rejected": -2.6875, "logps/chosen": -512.0, "logps/rejected": -900.0, "loss": 0.1742, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.1875, "rewards/margins": 4.0625, "rewards/rejected": -7.25, "step": 10870 }, { "epoch": 0.8185990519900684, "grad_norm": 15.02959545012159, "learning_rate": 4.847381695658692e-08, "logits/chosen": -2.546875, "logits/rejected": -2.78125, "logps/chosen": -536.0, "logps/rejected": -892.0, "loss": 0.1977, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.40625, "rewards/margins": 3.6875, "rewards/rejected": -7.09375, "step": 10880 }, { "epoch": 0.8193514408246182, "grad_norm": 14.445687304959678, "learning_rate": 4.80859352020809e-08, "logits/chosen": -2.6875, "logits/rejected": -2.5625, "logps/chosen": -564.0, "logps/rejected": -880.0, "loss": 0.1894, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.640625, "rewards/margins": 3.46875, "rewards/rejected": -7.09375, "step": 10890 }, { "epoch": 0.8201038296591678, "grad_norm": 15.548596864381523, "learning_rate": 4.769944638333123e-08, "logits/chosen": -2.53125, "logits/rejected": -2.578125, "logps/chosen": -528.0, "logps/rejected": -872.0, "loss": 0.1852, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.34375, "rewards/margins": 3.671875, "rewards/rejected": -7.03125, "step": 10900 }, { "epoch": 0.8208562184937176, "grad_norm": 11.92421298741924, "learning_rate": 4.731435316659141e-08, "logits/chosen": -2.796875, "logits/rejected": -2.53125, "logps/chosen": -504.0, "logps/rejected": -884.0, "loss": 0.1853, "rewards/accuracies": 0.9375, "rewards/chosen": -3.171875, "rewards/margins": 3.859375, "rewards/rejected": -7.03125, "step": 10910 }, { "epoch": 0.8216086073282672, "grad_norm": 12.443080033909837, "learning_rate": 4.693065820848724e-08, "logits/chosen": -2.8125, "logits/rejected": -2.5625, "logps/chosen": -496.0, "logps/rejected": -872.0, "loss": 0.2013, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.09375, "rewards/margins": 3.71875, "rewards/rejected": -6.8125, "step": 10920 }, { "epoch": 0.822360996162817, "grad_norm": 13.65323514030655, "learning_rate": 4.654836415599836e-08, "logits/chosen": -2.75, "logits/rejected": -2.75, "logps/chosen": -516.0, "logps/rejected": -872.0, "loss": 0.1777, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.328125, "rewards/margins": 3.703125, "rewards/rejected": -7.03125, "step": 10930 }, { "epoch": 0.8231133849973666, "grad_norm": 12.66244968454752, "learning_rate": 4.616747364644008e-08, "logits/chosen": -2.703125, "logits/rejected": -2.59375, "logps/chosen": -516.0, "logps/rejected": -888.0, "loss": 0.1942, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.265625, "rewards/margins": 3.75, "rewards/rejected": -7.0, "step": 10940 }, { "epoch": 0.8238657738319163, "grad_norm": 11.198669952546332, "learning_rate": 4.578798930744523e-08, "logits/chosen": -2.8125, "logits/rejected": -2.65625, "logps/chosen": -500.0, "logps/rejected": -852.0, "loss": 0.1971, "rewards/accuracies": 0.90625, "rewards/chosen": -3.15625, "rewards/margins": 3.5625, "rewards/rejected": -6.71875, "step": 10950 }, { "epoch": 0.824618162666466, "grad_norm": 13.672000032122849, "learning_rate": 4.5409913756945835e-08, "logits/chosen": -2.796875, "logits/rejected": -2.875, "logps/chosen": -494.0, "logps/rejected": -836.0, "loss": 0.1798, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.0625, "rewards/margins": 3.65625, "rewards/rejected": -6.71875, "step": 10960 }, { "epoch": 0.8253705515010157, "grad_norm": 19.549317875080593, "learning_rate": 4.5033249603155135e-08, "logits/chosen": -2.671875, "logits/rejected": -2.828125, "logps/chosen": -540.0, "logps/rejected": -836.0, "loss": 0.1939, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.421875, "rewards/margins": 3.265625, "rewards/rejected": -6.6875, "step": 10970 }, { "epoch": 0.8261229403355654, "grad_norm": 10.451080205851735, "learning_rate": 4.465799944454984e-08, "logits/chosen": -2.6875, "logits/rejected": -2.75, "logps/chosen": -528.0, "logps/rejected": -864.0, "loss": 0.1966, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.234375, "rewards/margins": 3.5625, "rewards/rejected": -6.8125, "step": 10980 }, { "epoch": 0.8268753291701151, "grad_norm": 8.22398363338978, "learning_rate": 4.428416586985184e-08, "logits/chosen": -2.78125, "logits/rejected": -2.578125, "logps/chosen": -524.0, "logps/rejected": -896.0, "loss": 0.1828, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.28125, "rewards/margins": 3.890625, "rewards/rejected": -7.1875, "step": 10990 }, { "epoch": 0.8276277180046648, "grad_norm": 13.1082664124389, "learning_rate": 4.3911751458010434e-08, "logits/chosen": -2.75, "logits/rejected": -2.828125, "logps/chosen": -504.0, "logps/rejected": -844.0, "loss": 0.1723, "rewards/accuracies": 0.96875, "rewards/chosen": -3.21875, "rewards/margins": 3.5625, "rewards/rejected": -6.78125, "step": 11000 }, { "epoch": 0.8283801068392145, "grad_norm": 12.202634364448846, "learning_rate": 4.354075877818475e-08, "logits/chosen": -2.734375, "logits/rejected": -2.6875, "logps/chosen": -540.0, "logps/rejected": -876.0, "loss": 0.2039, "rewards/accuracies": 0.9375, "rewards/chosen": -3.40625, "rewards/margins": 3.734375, "rewards/rejected": -7.15625, "step": 11010 }, { "epoch": 0.8291324956737642, "grad_norm": 11.820334469174025, "learning_rate": 4.3171190389725746e-08, "logits/chosen": -2.71875, "logits/rejected": -2.671875, "logps/chosen": -470.0, "logps/rejected": -860.0, "loss": 0.1629, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -2.859375, "rewards/margins": 4.09375, "rewards/rejected": -6.96875, "step": 11020 }, { "epoch": 0.8298848845083139, "grad_norm": 13.483538537421131, "learning_rate": 4.280304884215885e-08, "logits/chosen": -2.78125, "logits/rejected": -2.671875, "logps/chosen": -536.0, "logps/rejected": -908.0, "loss": 0.1879, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.359375, "rewards/margins": 3.90625, "rewards/rejected": -7.25, "step": 11030 }, { "epoch": 0.8306372733428636, "grad_norm": 15.406013702649151, "learning_rate": 4.2436336675166076e-08, "logits/chosen": -2.625, "logits/rejected": -2.375, "logps/chosen": -520.0, "logps/rejected": -888.0, "loss": 0.2105, "rewards/accuracies": 0.9375, "rewards/chosen": -3.328125, "rewards/margins": 3.765625, "rewards/rejected": -7.09375, "step": 11040 }, { "epoch": 0.8313896621774133, "grad_norm": 14.013931975292765, "learning_rate": 4.207105641856859e-08, "logits/chosen": -2.6875, "logits/rejected": -2.4375, "logps/chosen": -498.0, "logps/rejected": -848.0, "loss": 0.1896, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.1875, "rewards/margins": 3.546875, "rewards/rejected": -6.71875, "step": 11050 }, { "epoch": 0.832142051011963, "grad_norm": 14.393197502425686, "learning_rate": 4.17072105923095e-08, "logits/chosen": -2.625, "logits/rejected": -2.65625, "logps/chosen": -502.0, "logps/rejected": -800.0, "loss": 0.1825, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.1875, "rewards/margins": 3.25, "rewards/rejected": -6.4375, "step": 11060 }, { "epoch": 0.8328944398465127, "grad_norm": 16.532629127975618, "learning_rate": 4.134480170643606e-08, "logits/chosen": -2.65625, "logits/rejected": -2.359375, "logps/chosen": -524.0, "logps/rejected": -868.0, "loss": 0.2056, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -3.40625, "rewards/margins": 3.484375, "rewards/rejected": -6.90625, "step": 11070 }, { "epoch": 0.8336468286810623, "grad_norm": 10.957829281243484, "learning_rate": 4.0983832261082624e-08, "logits/chosen": -2.734375, "logits/rejected": -2.609375, "logps/chosen": -532.0, "logps/rejected": -880.0, "loss": 0.184, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.34375, "rewards/margins": 3.640625, "rewards/rejected": -7.0, "step": 11080 }, { "epoch": 0.8343992175156121, "grad_norm": 12.330984985759155, "learning_rate": 4.062430474645353e-08, "logits/chosen": -2.8125, "logits/rejected": -2.859375, "logps/chosen": -490.0, "logps/rejected": -880.0, "loss": 0.1721, "rewards/accuracies": 0.90625, "rewards/chosen": -3.09375, "rewards/margins": 4.09375, "rewards/rejected": -7.1875, "step": 11090 }, { "epoch": 0.8351516063501617, "grad_norm": 12.892580172863383, "learning_rate": 4.0266221642805355e-08, "logits/chosen": -2.765625, "logits/rejected": -2.703125, "logps/chosen": -512.0, "logps/rejected": -880.0, "loss": 0.1837, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -3.28125, "rewards/margins": 3.78125, "rewards/rejected": -7.0625, "step": 11100 }, { "epoch": 0.8359039951847115, "grad_norm": 15.94972340549044, "learning_rate": 3.990958542043052e-08, "logits/chosen": -2.796875, "logits/rejected": -2.609375, "logps/chosen": -516.0, "logps/rejected": -828.0, "loss": 0.1918, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.375, "rewards/margins": 3.15625, "rewards/rejected": -6.53125, "step": 11110 }, { "epoch": 0.8366563840192611, "grad_norm": 11.951613111272898, "learning_rate": 3.9554398539639766e-08, "logits/chosen": -2.75, "logits/rejected": -2.6875, "logps/chosen": -524.0, "logps/rejected": -844.0, "loss": 0.1845, "rewards/accuracies": 0.90625, "rewards/chosen": -3.375, "rewards/margins": 3.421875, "rewards/rejected": -6.8125, "step": 11120 }, { "epoch": 0.8374087728538109, "grad_norm": 11.373473576296734, "learning_rate": 3.92006634507453e-08, "logits/chosen": -2.6875, "logits/rejected": -2.5625, "logps/chosen": -528.0, "logps/rejected": -884.0, "loss": 0.1787, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.3125, "rewards/margins": 3.78125, "rewards/rejected": -7.09375, "step": 11130 }, { "epoch": 0.8381611616883605, "grad_norm": 10.164055970498385, "learning_rate": 3.884838259404402e-08, "logits/chosen": -2.59375, "logits/rejected": -2.5625, "logps/chosen": -488.0, "logps/rejected": -876.0, "loss": 0.1965, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -2.96875, "rewards/margins": 3.9375, "rewards/rejected": -6.90625, "step": 11140 }, { "epoch": 0.8389135505229103, "grad_norm": 11.679519926208636, "learning_rate": 3.8497558399800454e-08, "logits/chosen": -2.546875, "logits/rejected": -2.359375, "logps/chosen": -508.0, "logps/rejected": -884.0, "loss": 0.1879, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.109375, "rewards/margins": 3.875, "rewards/rejected": -7.0, "step": 11150 }, { "epoch": 0.8396659393574599, "grad_norm": 8.470191389148653, "learning_rate": 3.814819328823027e-08, "logits/chosen": -2.625, "logits/rejected": -2.609375, "logps/chosen": -552.0, "logps/rejected": -848.0, "loss": 0.1986, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.53125, "rewards/margins": 3.1875, "rewards/rejected": -6.71875, "step": 11160 }, { "epoch": 0.8404183281920097, "grad_norm": 15.20126611570185, "learning_rate": 3.780028966948326e-08, "logits/chosen": -2.671875, "logits/rejected": -2.75, "logps/chosen": -528.0, "logps/rejected": -848.0, "loss": 0.2135, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -3.390625, "rewards/margins": 3.359375, "rewards/rejected": -6.75, "step": 11170 }, { "epoch": 0.8411707170265593, "grad_norm": 12.682952794640746, "learning_rate": 3.7453849943626964e-08, "logits/chosen": -2.671875, "logits/rejected": -2.71875, "logps/chosen": -520.0, "logps/rejected": -816.0, "loss": 0.1897, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.28125, "rewards/margins": 3.125, "rewards/rejected": -6.40625, "step": 11180 }, { "epoch": 0.841923105861109, "grad_norm": 12.298035121631226, "learning_rate": 3.710887650063005e-08, "logits/chosen": -2.71875, "logits/rejected": -2.625, "logps/chosen": -520.0, "logps/rejected": -816.0, "loss": 0.1965, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.34375, "rewards/margins": 3.265625, "rewards/rejected": -6.59375, "step": 11190 }, { "epoch": 0.8426754946956587, "grad_norm": 16.767167108058175, "learning_rate": 3.676537172034563e-08, "logits/chosen": -2.71875, "logits/rejected": -2.40625, "logps/chosen": -520.0, "logps/rejected": -908.0, "loss": 0.2055, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.140625, "rewards/margins": 4.09375, "rewards/rejected": -7.21875, "step": 11200 }, { "epoch": 0.8434278835302084, "grad_norm": 15.989918161039656, "learning_rate": 3.642333797249536e-08, "logits/chosen": -2.59375, "logits/rejected": -2.640625, "logps/chosen": -524.0, "logps/rejected": -864.0, "loss": 0.1947, "rewards/accuracies": 0.90625, "rewards/chosen": -3.296875, "rewards/margins": 3.59375, "rewards/rejected": -6.90625, "step": 11210 }, { "epoch": 0.8441802723647581, "grad_norm": 8.96562162079509, "learning_rate": 3.608277761665243e-08, "logits/chosen": -2.796875, "logits/rejected": -2.546875, "logps/chosen": -500.0, "logps/rejected": -868.0, "loss": 0.194, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.3125, "rewards/margins": 3.5, "rewards/rejected": -6.8125, "step": 11220 }, { "epoch": 0.8449326611993078, "grad_norm": 16.486921764946686, "learning_rate": 3.574369300222568e-08, "logits/chosen": -2.78125, "logits/rejected": -2.78125, "logps/chosen": -492.0, "logps/rejected": -852.0, "loss": 0.1804, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.109375, "rewards/margins": 3.703125, "rewards/rejected": -6.8125, "step": 11230 }, { "epoch": 0.8456850500338575, "grad_norm": 13.480896383421298, "learning_rate": 3.540608646844348e-08, "logits/chosen": -2.71875, "logits/rejected": -2.484375, "logps/chosen": -508.0, "logps/rejected": -876.0, "loss": 0.1874, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.25, "rewards/margins": 3.515625, "rewards/rejected": -6.78125, "step": 11240 }, { "epoch": 0.8464374388684072, "grad_norm": 15.026350841703996, "learning_rate": 3.506996034433723e-08, "logits/chosen": -2.75, "logits/rejected": -2.65625, "logps/chosen": -516.0, "logps/rejected": -852.0, "loss": 0.1777, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.171875, "rewards/margins": 3.703125, "rewards/rejected": -6.875, "step": 11250 }, { "epoch": 0.8471898277029569, "grad_norm": 13.361805867630897, "learning_rate": 3.473531694872556e-08, "logits/chosen": -2.796875, "logits/rejected": -2.484375, "logps/chosen": -520.0, "logps/rejected": -920.0, "loss": 0.1845, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.359375, "rewards/margins": 3.921875, "rewards/rejected": -7.28125, "step": 11260 }, { "epoch": 0.8479422165375066, "grad_norm": 13.258328942335758, "learning_rate": 3.440215859019838e-08, "logits/chosen": -2.75, "logits/rejected": -2.734375, "logps/chosen": -488.0, "logps/rejected": -860.0, "loss": 0.1731, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -3.09375, "rewards/margins": 3.875, "rewards/rejected": -6.96875, "step": 11270 }, { "epoch": 0.8486946053720563, "grad_norm": 12.52345540569602, "learning_rate": 3.4070487567100516e-08, "logits/chosen": -2.6875, "logits/rejected": -2.578125, "logps/chosen": -524.0, "logps/rejected": -844.0, "loss": 0.1718, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.34375, "rewards/margins": 3.4375, "rewards/rejected": -6.78125, "step": 11280 }, { "epoch": 0.849446994206606, "grad_norm": 14.9338250052211, "learning_rate": 3.374030616751661e-08, "logits/chosen": -2.578125, "logits/rejected": -2.515625, "logps/chosen": -528.0, "logps/rejected": -856.0, "loss": 0.184, "rewards/accuracies": 0.875, "rewards/chosen": -3.296875, "rewards/margins": 3.53125, "rewards/rejected": -6.8125, "step": 11290 }, { "epoch": 0.8501993830411557, "grad_norm": 13.163938580793278, "learning_rate": 3.3411616669254627e-08, "logits/chosen": -2.6875, "logits/rejected": -2.515625, "logps/chosen": -494.0, "logps/rejected": -828.0, "loss": 0.202, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.125, "rewards/margins": 3.4375, "rewards/rejected": -6.5625, "step": 11300 }, { "epoch": 0.8509517718757054, "grad_norm": 10.730937461958687, "learning_rate": 3.308442133983036e-08, "logits/chosen": -2.734375, "logits/rejected": -2.796875, "logps/chosen": -532.0, "logps/rejected": -852.0, "loss": 0.1934, "rewards/accuracies": 0.90625, "rewards/chosen": -3.1875, "rewards/margins": 3.65625, "rewards/rejected": -6.84375, "step": 11310 }, { "epoch": 0.851704160710255, "grad_norm": 13.858982845312156, "learning_rate": 3.275872243645214e-08, "logits/chosen": -2.625, "logits/rejected": -2.78125, "logps/chosen": -536.0, "logps/rejected": -820.0, "loss": 0.1991, "rewards/accuracies": 0.9375, "rewards/chosen": -3.265625, "rewards/margins": 3.375, "rewards/rejected": -6.625, "step": 11320 }, { "epoch": 0.8524565495448048, "grad_norm": 15.68658557746299, "learning_rate": 3.243452220600473e-08, "logits/chosen": -2.59375, "logits/rejected": -2.46875, "logps/chosen": -524.0, "logps/rejected": -888.0, "loss": 0.1849, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.328125, "rewards/margins": 3.625, "rewards/rejected": -6.96875, "step": 11330 }, { "epoch": 0.8532089383793544, "grad_norm": 12.749958758787077, "learning_rate": 3.2111822885034054e-08, "logits/chosen": -2.625, "logits/rejected": -2.390625, "logps/chosen": -524.0, "logps/rejected": -880.0, "loss": 0.1849, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.453125, "rewards/margins": 3.609375, "rewards/rejected": -7.0625, "step": 11340 }, { "epoch": 0.8539613272139042, "grad_norm": 15.77050295329717, "learning_rate": 3.1790626699731955e-08, "logits/chosen": -2.734375, "logits/rejected": -2.625, "logps/chosen": -516.0, "logps/rejected": -880.0, "loss": 0.1997, "rewards/accuracies": 0.9375, "rewards/chosen": -3.21875, "rewards/margins": 3.75, "rewards/rejected": -6.9375, "step": 11350 }, { "epoch": 0.8547137160484538, "grad_norm": 15.822742629251751, "learning_rate": 3.1470935865920505e-08, "logits/chosen": -2.734375, "logits/rejected": -2.59375, "logps/chosen": -520.0, "logps/rejected": -876.0, "loss": 0.1894, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -3.296875, "rewards/margins": 3.71875, "rewards/rejected": -7.0, "step": 11360 }, { "epoch": 0.8554661048830036, "grad_norm": 15.398753137616787, "learning_rate": 3.1152752589036904e-08, "logits/chosen": -2.6875, "logits/rejected": -2.5625, "logps/chosen": -512.0, "logps/rejected": -900.0, "loss": 0.1919, "rewards/accuracies": 0.90625, "rewards/chosen": -3.28125, "rewards/margins": 3.953125, "rewards/rejected": -7.21875, "step": 11370 }, { "epoch": 0.8562184937175532, "grad_norm": 13.318235275082994, "learning_rate": 3.083607906411831e-08, "logits/chosen": -2.875, "logits/rejected": -2.53125, "logps/chosen": -528.0, "logps/rejected": -852.0, "loss": 0.1766, "rewards/accuracies": 0.90625, "rewards/chosen": -3.59375, "rewards/margins": 3.140625, "rewards/rejected": -6.71875, "step": 11380 }, { "epoch": 0.856970882552103, "grad_norm": 13.863790336875159, "learning_rate": 3.052091747578644e-08, "logits/chosen": -2.8125, "logits/rejected": -2.828125, "logps/chosen": -540.0, "logps/rejected": -884.0, "loss": 0.1863, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.34375, "rewards/margins": 3.578125, "rewards/rejected": -6.9375, "step": 11390 }, { "epoch": 0.8577232713866526, "grad_norm": 12.040284881876008, "learning_rate": 3.020726999823298e-08, "logits/chosen": -2.78125, "logits/rejected": -2.65625, "logps/chosen": -500.0, "logps/rejected": -852.0, "loss": 0.1904, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -3.21875, "rewards/margins": 3.53125, "rewards/rejected": -6.75, "step": 11400 }, { "epoch": 0.8584756602212024, "grad_norm": 9.03811211298267, "learning_rate": 2.989513879520394e-08, "logits/chosen": -2.6875, "logits/rejected": -2.515625, "logps/chosen": -516.0, "logps/rejected": -880.0, "loss": 0.1949, "rewards/accuracies": 0.90625, "rewards/chosen": -3.109375, "rewards/margins": 3.859375, "rewards/rejected": -7.0, "step": 11410 }, { "epoch": 0.859228049055752, "grad_norm": 11.586878557148363, "learning_rate": 2.9584526019985373e-08, "logits/chosen": -2.703125, "logits/rejected": -2.359375, "logps/chosen": -496.0, "logps/rejected": -872.0, "loss": 0.1948, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.125, "rewards/margins": 3.75, "rewards/rejected": -6.875, "step": 11420 }, { "epoch": 0.8599804378903018, "grad_norm": 11.28292110465477, "learning_rate": 2.927543381538805e-08, "logits/chosen": -2.625, "logits/rejected": -2.5, "logps/chosen": -520.0, "logps/rejected": -900.0, "loss": 0.1789, "rewards/accuracies": 0.90625, "rewards/chosen": -3.359375, "rewards/margins": 3.796875, "rewards/rejected": -7.15625, "step": 11430 }, { "epoch": 0.8607328267248514, "grad_norm": 10.045316344926698, "learning_rate": 2.8967864313732826e-08, "logits/chosen": -2.703125, "logits/rejected": -2.75, "logps/chosen": -520.0, "logps/rejected": -864.0, "loss": 0.1721, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.34375, "rewards/margins": 3.671875, "rewards/rejected": -7.0, "step": 11440 }, { "epoch": 0.861485215559401, "grad_norm": 11.195051607374667, "learning_rate": 2.866181963683617e-08, "logits/chosen": -2.625, "logits/rejected": -2.640625, "logps/chosen": -532.0, "logps/rejected": -864.0, "loss": 0.1895, "rewards/accuracies": 0.9375, "rewards/chosen": -3.359375, "rewards/margins": 3.59375, "rewards/rejected": -6.9375, "step": 11450 }, { "epoch": 0.8622376043939508, "grad_norm": 10.195058977366074, "learning_rate": 2.8357301895994946e-08, "logits/chosen": -2.703125, "logits/rejected": -2.859375, "logps/chosen": -510.0, "logps/rejected": -844.0, "loss": 0.1688, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.3125, "rewards/margins": 3.421875, "rewards/rejected": -6.75, "step": 11460 }, { "epoch": 0.8629899932285005, "grad_norm": 10.696884501738472, "learning_rate": 2.8054313191972574e-08, "logits/chosen": -2.765625, "logits/rejected": -2.671875, "logps/chosen": -524.0, "logps/rejected": -892.0, "loss": 0.1781, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.21875, "rewards/margins": 3.796875, "rewards/rejected": -7.0, "step": 11470 }, { "epoch": 0.8637423820630502, "grad_norm": 11.713528714821328, "learning_rate": 2.775285561498397e-08, "logits/chosen": -2.625, "logits/rejected": -2.6875, "logps/chosen": -532.0, "logps/rejected": -832.0, "loss": 0.1916, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.34375, "rewards/margins": 3.25, "rewards/rejected": -6.59375, "step": 11480 }, { "epoch": 0.8644947708975999, "grad_norm": 13.461491174458803, "learning_rate": 2.7452931244681314e-08, "logits/chosen": -2.75, "logits/rejected": -2.78125, "logps/chosen": -540.0, "logps/rejected": -896.0, "loss": 0.1942, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.40625, "rewards/margins": 3.890625, "rewards/rejected": -7.28125, "step": 11490 }, { "epoch": 0.8652471597321496, "grad_norm": 16.314530467129643, "learning_rate": 2.7154542150139876e-08, "logits/chosen": -2.8125, "logits/rejected": -2.84375, "logps/chosen": -516.0, "logps/rejected": -800.0, "loss": 0.2069, "rewards/accuracies": 0.90625, "rewards/chosen": -3.421875, "rewards/margins": 2.953125, "rewards/rejected": -6.375, "step": 11500 }, { "epoch": 0.8659995485666993, "grad_norm": 16.287179928659214, "learning_rate": 2.6857690389843478e-08, "logits/chosen": -2.734375, "logits/rejected": -2.8125, "logps/chosen": -510.0, "logps/rejected": -876.0, "loss": 0.1774, "rewards/accuracies": 0.9375, "rewards/chosen": -3.234375, "rewards/margins": 3.6875, "rewards/rejected": -6.90625, "step": 11510 }, { "epoch": 0.866751937401249, "grad_norm": 8.504122206338133, "learning_rate": 2.656237801167033e-08, "logits/chosen": -2.78125, "logits/rejected": -2.71875, "logps/chosen": -504.0, "logps/rejected": -880.0, "loss": 0.1803, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.125, "rewards/margins": 3.859375, "rewards/rejected": -7.0, "step": 11520 }, { "epoch": 0.8675043262357987, "grad_norm": 16.207348949205983, "learning_rate": 2.62686070528792e-08, "logits/chosen": -2.8125, "logits/rejected": -2.671875, "logps/chosen": -508.0, "logps/rejected": -872.0, "loss": 0.1893, "rewards/accuracies": 0.90625, "rewards/chosen": -3.234375, "rewards/margins": 3.578125, "rewards/rejected": -6.8125, "step": 11530 }, { "epoch": 0.8682567150703484, "grad_norm": 13.87076989191311, "learning_rate": 2.5976379540094907e-08, "logits/chosen": -2.734375, "logits/rejected": -2.734375, "logps/chosen": -516.0, "logps/rejected": -892.0, "loss": 0.182, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.203125, "rewards/margins": 3.890625, "rewards/rejected": -7.09375, "step": 11540 }, { "epoch": 0.869009103904898, "grad_norm": 10.443191571392008, "learning_rate": 2.5685697489294665e-08, "logits/chosen": -2.65625, "logits/rejected": -2.625, "logps/chosen": -516.0, "logps/rejected": -828.0, "loss": 0.1744, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.0625, "rewards/margins": 3.359375, "rewards/rejected": -6.40625, "step": 11550 }, { "epoch": 0.8697614927394477, "grad_norm": 15.394340912754013, "learning_rate": 2.539656290579409e-08, "logits/chosen": -2.734375, "logits/rejected": -2.671875, "logps/chosen": -520.0, "logps/rejected": -864.0, "loss": 0.1815, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.265625, "rewards/margins": 3.5625, "rewards/rejected": -6.84375, "step": 11560 }, { "epoch": 0.8705138815739975, "grad_norm": 18.897630674853026, "learning_rate": 2.510897778423324e-08, "logits/chosen": -2.703125, "logits/rejected": -2.65625, "logps/chosen": -524.0, "logps/rejected": -844.0, "loss": 0.2142, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.375, "rewards/margins": 3.375, "rewards/rejected": -6.75, "step": 11570 }, { "epoch": 0.8712662704085471, "grad_norm": 11.142949232919456, "learning_rate": 2.482294410856317e-08, "logits/chosen": -2.75, "logits/rejected": -2.59375, "logps/chosen": -500.0, "logps/rejected": -864.0, "loss": 0.186, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.15625, "rewards/margins": 3.71875, "rewards/rejected": -6.875, "step": 11580 }, { "epoch": 0.8720186592430968, "grad_norm": 15.119341291018834, "learning_rate": 2.4538463852031897e-08, "logits/chosen": -2.65625, "logits/rejected": -2.78125, "logps/chosen": -532.0, "logps/rejected": -872.0, "loss": 0.195, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.453125, "rewards/margins": 3.609375, "rewards/rejected": -7.0625, "step": 11590 }, { "epoch": 0.8727710480776465, "grad_norm": 13.645098322422426, "learning_rate": 2.425553897717092e-08, "logits/chosen": -2.71875, "logits/rejected": -2.71875, "logps/chosen": -524.0, "logps/rejected": -868.0, "loss": 0.1872, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.25, "rewards/margins": 3.78125, "rewards/rejected": -7.03125, "step": 11600 }, { "epoch": 0.8735234369121962, "grad_norm": 11.781800856002343, "learning_rate": 2.3974171435781787e-08, "logits/chosen": -2.671875, "logits/rejected": -2.671875, "logps/chosen": -500.0, "logps/rejected": -856.0, "loss": 0.1961, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -3.109375, "rewards/margins": 3.75, "rewards/rejected": -6.875, "step": 11610 }, { "epoch": 0.8742758257467459, "grad_norm": 11.566208292937905, "learning_rate": 2.3694363168922454e-08, "logits/chosen": -2.6875, "logits/rejected": -2.671875, "logps/chosen": -494.0, "logps/rejected": -876.0, "loss": 0.1905, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.125, "rewards/margins": 3.765625, "rewards/rejected": -6.875, "step": 11620 }, { "epoch": 0.8750282145812956, "grad_norm": 12.951568789159218, "learning_rate": 2.3416116106894062e-08, "logits/chosen": -2.78125, "logits/rejected": -2.8125, "logps/chosen": -494.0, "logps/rejected": -864.0, "loss": 0.191, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.21875, "rewards/margins": 3.90625, "rewards/rejected": -7.125, "step": 11630 }, { "epoch": 0.8757806034158453, "grad_norm": 12.610195198557632, "learning_rate": 2.3139432169227507e-08, "logits/chosen": -2.734375, "logits/rejected": -2.671875, "logps/chosen": -502.0, "logps/rejected": -872.0, "loss": 0.182, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.3125, "rewards/margins": 3.734375, "rewards/rejected": -7.03125, "step": 11640 }, { "epoch": 0.876532992250395, "grad_norm": 11.362359382358516, "learning_rate": 2.2864313264670058e-08, "logits/chosen": -2.640625, "logits/rejected": -2.59375, "logps/chosen": -516.0, "logps/rejected": -912.0, "loss": 0.1988, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.171875, "rewards/margins": 4.0625, "rewards/rejected": -7.25, "step": 11650 }, { "epoch": 0.8772853810849447, "grad_norm": 10.203450290586224, "learning_rate": 2.2590761291172655e-08, "logits/chosen": -2.84375, "logits/rejected": -2.40625, "logps/chosen": -520.0, "logps/rejected": -900.0, "loss": 0.1907, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.3125, "rewards/margins": 3.84375, "rewards/rejected": -7.15625, "step": 11660 }, { "epoch": 0.8780377699194944, "grad_norm": 9.961540587984306, "learning_rate": 2.2318778135876292e-08, "logits/chosen": -2.671875, "logits/rejected": -2.5625, "logps/chosen": -524.0, "logps/rejected": -864.0, "loss": 0.1718, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.203125, "rewards/margins": 3.578125, "rewards/rejected": -6.8125, "step": 11670 }, { "epoch": 0.8787901587540441, "grad_norm": 14.74267233075504, "learning_rate": 2.2048365675099378e-08, "logits/chosen": -2.65625, "logits/rejected": -2.6875, "logps/chosen": -520.0, "logps/rejected": -912.0, "loss": 0.2155, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.40625, "rewards/margins": 3.5625, "rewards/rejected": -6.96875, "step": 11680 }, { "epoch": 0.8795425475885937, "grad_norm": 16.277287895922615, "learning_rate": 2.1779525774324514e-08, "logits/chosen": -2.609375, "logits/rejected": -2.75, "logps/chosen": -520.0, "logps/rejected": -868.0, "loss": 0.1813, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.296875, "rewards/margins": 3.703125, "rewards/rejected": -7.0, "step": 11690 }, { "epoch": 0.8802949364231435, "grad_norm": 12.13322425851075, "learning_rate": 2.1512260288185786e-08, "logits/chosen": -2.65625, "logits/rejected": -2.5, "logps/chosen": -544.0, "logps/rejected": -856.0, "loss": 0.181, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.4375, "rewards/margins": 3.296875, "rewards/rejected": -6.75, "step": 11700 }, { "epoch": 0.8810473252576931, "grad_norm": 10.170841346175349, "learning_rate": 2.124657106045602e-08, "logits/chosen": -2.75, "logits/rejected": -2.65625, "logps/chosen": -498.0, "logps/rejected": -836.0, "loss": 0.1697, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.25, "rewards/margins": 3.5, "rewards/rejected": -6.75, "step": 11710 }, { "epoch": 0.8817997140922429, "grad_norm": 12.68943712405746, "learning_rate": 2.0982459924033857e-08, "logits/chosen": -2.75, "logits/rejected": -2.546875, "logps/chosen": -520.0, "logps/rejected": -888.0, "loss": 0.1938, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.25, "rewards/margins": 3.734375, "rewards/rejected": -6.96875, "step": 11720 }, { "epoch": 0.8825521029267925, "grad_norm": 14.961323513071827, "learning_rate": 2.071992870093131e-08, "logits/chosen": -2.6875, "logits/rejected": -2.75, "logps/chosen": -520.0, "logps/rejected": -888.0, "loss": 0.1757, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.265625, "rewards/margins": 3.71875, "rewards/rejected": -6.96875, "step": 11730 }, { "epoch": 0.8833044917613423, "grad_norm": 12.809711222351913, "learning_rate": 2.0458979202261057e-08, "logits/chosen": -2.6875, "logits/rejected": -2.609375, "logps/chosen": -504.0, "logps/rejected": -868.0, "loss": 0.1785, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.296875, "rewards/margins": 3.5625, "rewards/rejected": -6.875, "step": 11740 }, { "epoch": 0.8840568805958919, "grad_norm": 15.678559888549533, "learning_rate": 2.0199613228224e-08, "logits/chosen": -2.671875, "logits/rejected": -2.546875, "logps/chosen": -520.0, "logps/rejected": -860.0, "loss": 0.1963, "rewards/accuracies": 0.9375, "rewards/chosen": -3.296875, "rewards/margins": 3.671875, "rewards/rejected": -6.96875, "step": 11750 }, { "epoch": 0.8848092694304417, "grad_norm": 11.994524476409111, "learning_rate": 1.9941832568096978e-08, "logits/chosen": -2.59375, "logits/rejected": -2.671875, "logps/chosen": -520.0, "logps/rejected": -856.0, "loss": 0.1735, "rewards/accuracies": 0.90625, "rewards/chosen": -3.34375, "rewards/margins": 3.5, "rewards/rejected": -6.84375, "step": 11760 }, { "epoch": 0.8855616582649913, "grad_norm": 13.045687858169329, "learning_rate": 1.9685639000220105e-08, "logits/chosen": -2.6875, "logits/rejected": -2.453125, "logps/chosen": -536.0, "logps/rejected": -940.0, "loss": 0.189, "rewards/accuracies": 0.9375, "rewards/chosen": -3.484375, "rewards/margins": 3.921875, "rewards/rejected": -7.40625, "step": 11770 }, { "epoch": 0.8863140470995411, "grad_norm": 15.925591372265597, "learning_rate": 1.9431034291984755e-08, "logits/chosen": -2.71875, "logits/rejected": -2.84375, "logps/chosen": -528.0, "logps/rejected": -880.0, "loss": 0.1764, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.34375, "rewards/margins": 3.765625, "rewards/rejected": -7.09375, "step": 11780 }, { "epoch": 0.8870664359340907, "grad_norm": 10.771152684925749, "learning_rate": 1.9178020199821426e-08, "logits/chosen": -2.671875, "logits/rejected": -2.5625, "logps/chosen": -536.0, "logps/rejected": -896.0, "loss": 0.1938, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.375, "rewards/margins": 3.609375, "rewards/rejected": -6.96875, "step": 11790 }, { "epoch": 0.8878188247686404, "grad_norm": 15.975713143397098, "learning_rate": 1.892659846918737e-08, "logits/chosen": -2.6875, "logits/rejected": -2.65625, "logps/chosen": -516.0, "logps/rejected": -864.0, "loss": 0.1886, "rewards/accuracies": 0.90625, "rewards/chosen": -3.234375, "rewards/margins": 3.5625, "rewards/rejected": -6.8125, "step": 11800 }, { "epoch": 0.8885712136031901, "grad_norm": 13.400655555817842, "learning_rate": 1.8676770834554655e-08, "logits/chosen": -2.609375, "logits/rejected": -2.625, "logps/chosen": -544.0, "logps/rejected": -888.0, "loss": 0.1744, "rewards/accuracies": 0.9375, "rewards/chosen": -3.46875, "rewards/margins": 3.578125, "rewards/rejected": -7.0625, "step": 11810 }, { "epoch": 0.8893236024377398, "grad_norm": 13.900497994962649, "learning_rate": 1.842853901939842e-08, "logits/chosen": -2.796875, "logits/rejected": -2.609375, "logps/chosen": -508.0, "logps/rejected": -892.0, "loss": 0.177, "rewards/accuracies": 0.9375, "rewards/chosen": -3.265625, "rewards/margins": 3.9375, "rewards/rejected": -7.21875, "step": 11820 }, { "epoch": 0.8900759912722895, "grad_norm": 18.898106502853572, "learning_rate": 1.8181904736184528e-08, "logits/chosen": -2.703125, "logits/rejected": -2.625, "logps/chosen": -536.0, "logps/rejected": -908.0, "loss": 0.1914, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.4375, "rewards/margins": 3.90625, "rewards/rejected": -7.34375, "step": 11830 }, { "epoch": 0.8908283801068392, "grad_norm": 11.95613505863579, "learning_rate": 1.793686968635824e-08, "logits/chosen": -2.65625, "logits/rejected": -2.71875, "logps/chosen": -544.0, "logps/rejected": -856.0, "loss": 0.162, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.40625, "rewards/margins": 3.546875, "rewards/rejected": -6.9375, "step": 11840 }, { "epoch": 0.8915807689413889, "grad_norm": 11.583705941012417, "learning_rate": 1.7693435560332148e-08, "logits/chosen": -2.828125, "logits/rejected": -2.796875, "logps/chosen": -480.0, "logps/rejected": -832.0, "loss": 0.1652, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.078125, "rewards/margins": 3.625, "rewards/rejected": -6.6875, "step": 11850 }, { "epoch": 0.8923331577759386, "grad_norm": 13.184353442124387, "learning_rate": 1.7451604037474616e-08, "logits/chosen": -2.765625, "logits/rejected": -2.703125, "logps/chosen": -528.0, "logps/rejected": -892.0, "loss": 0.1851, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.40625, "rewards/margins": 3.796875, "rewards/rejected": -7.21875, "step": 11860 }, { "epoch": 0.8930855466104883, "grad_norm": 11.403254998592201, "learning_rate": 1.72113767860983e-08, "logits/chosen": -2.734375, "logits/rejected": -2.515625, "logps/chosen": -504.0, "logps/rejected": -864.0, "loss": 0.2002, "rewards/accuracies": 0.9375, "rewards/chosen": -3.125, "rewards/margins": 3.640625, "rewards/rejected": -6.78125, "step": 11870 }, { "epoch": 0.893837935445038, "grad_norm": 11.296088860160049, "learning_rate": 1.6972755463448423e-08, "logits/chosen": -2.859375, "logits/rejected": -2.921875, "logps/chosen": -544.0, "logps/rejected": -848.0, "loss": 0.1946, "rewards/accuracies": 0.90625, "rewards/chosen": -3.421875, "rewards/margins": 3.421875, "rewards/rejected": -6.84375, "step": 11880 }, { "epoch": 0.8945903242795877, "grad_norm": 12.80379600181592, "learning_rate": 1.6735741715691448e-08, "logits/chosen": -2.71875, "logits/rejected": -2.796875, "logps/chosen": -532.0, "logps/rejected": -864.0, "loss": 0.2002, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.375, "rewards/margins": 3.546875, "rewards/rejected": -6.9375, "step": 11890 }, { "epoch": 0.8953427131141374, "grad_norm": 15.706521170760107, "learning_rate": 1.650033717790389e-08, "logits/chosen": -2.703125, "logits/rejected": -2.625, "logps/chosen": -508.0, "logps/rejected": -860.0, "loss": 0.1874, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.21875, "rewards/margins": 3.6875, "rewards/rejected": -6.90625, "step": 11900 }, { "epoch": 0.8960951019486871, "grad_norm": 13.129139805168508, "learning_rate": 1.626654347406073e-08, "logits/chosen": -2.8125, "logits/rejected": -2.546875, "logps/chosen": -524.0, "logps/rejected": -908.0, "loss": 0.1794, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.40625, "rewards/margins": 3.6875, "rewards/rejected": -7.09375, "step": 11910 }, { "epoch": 0.8968474907832368, "grad_norm": 16.936049698791354, "learning_rate": 1.6034362217024396e-08, "logits/chosen": -2.625, "logits/rejected": -2.734375, "logps/chosen": -510.0, "logps/rejected": -848.0, "loss": 0.2086, "rewards/accuracies": 0.9375, "rewards/chosen": -3.28125, "rewards/margins": 3.59375, "rewards/rejected": -6.875, "step": 11920 }, { "epoch": 0.8975998796177864, "grad_norm": 14.305980182513187, "learning_rate": 1.580379500853357e-08, "logits/chosen": -2.6875, "logits/rejected": -2.765625, "logps/chosen": -520.0, "logps/rejected": -896.0, "loss": 0.1935, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.25, "rewards/margins": 3.984375, "rewards/rejected": -7.21875, "step": 11930 }, { "epoch": 0.8983522684523362, "grad_norm": 12.612906729376396, "learning_rate": 1.5574843439192213e-08, "logits/chosen": -2.65625, "logits/rejected": -2.59375, "logps/chosen": -486.0, "logps/rejected": -828.0, "loss": 0.1843, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.15625, "rewards/margins": 3.53125, "rewards/rejected": -6.6875, "step": 11940 }, { "epoch": 0.8991046572868858, "grad_norm": 10.767892561420428, "learning_rate": 1.534750908845858e-08, "logits/chosen": -2.671875, "logits/rejected": -2.46875, "logps/chosen": -506.0, "logps/rejected": -884.0, "loss": 0.1872, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.1875, "rewards/margins": 3.890625, "rewards/rejected": -7.0625, "step": 11950 }, { "epoch": 0.8998570461214356, "grad_norm": 13.112255128219287, "learning_rate": 1.512179352463419e-08, "logits/chosen": -2.796875, "logits/rejected": -2.6875, "logps/chosen": -498.0, "logps/rejected": -836.0, "loss": 0.1699, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.1875, "rewards/margins": 3.421875, "rewards/rejected": -6.625, "step": 11960 }, { "epoch": 0.9006094349559852, "grad_norm": 12.164267090100886, "learning_rate": 1.4897698304853213e-08, "logits/chosen": -2.65625, "logits/rejected": -2.671875, "logps/chosen": -544.0, "logps/rejected": -916.0, "loss": 0.1747, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.421875, "rewards/margins": 3.875, "rewards/rejected": -7.3125, "step": 11970 }, { "epoch": 0.901361823790535, "grad_norm": 16.2020086215253, "learning_rate": 1.4675224975071565e-08, "logits/chosen": -2.6875, "logits/rejected": -2.65625, "logps/chosen": -498.0, "logps/rejected": -876.0, "loss": 0.201, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.265625, "rewards/margins": 3.765625, "rewards/rejected": -7.03125, "step": 11980 }, { "epoch": 0.9021142126250846, "grad_norm": 17.214148699608025, "learning_rate": 1.445437507005623e-08, "logits/chosen": -2.765625, "logits/rejected": -2.484375, "logps/chosen": -512.0, "logps/rejected": -888.0, "loss": 0.2122, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.25, "rewards/margins": 3.765625, "rewards/rejected": -7.03125, "step": 11990 }, { "epoch": 0.9028666014596344, "grad_norm": 12.65762604990627, "learning_rate": 1.4235150113374977e-08, "logits/chosen": -2.6875, "logits/rejected": -2.828125, "logps/chosen": -498.0, "logps/rejected": -868.0, "loss": 0.1794, "rewards/accuracies": 0.9375, "rewards/chosen": -3.3125, "rewards/margins": 3.9375, "rewards/rejected": -7.25, "step": 12000 }, { "epoch": 0.903618990294184, "grad_norm": 9.750859513069612, "learning_rate": 1.4017551617385298e-08, "logits/chosen": -2.671875, "logits/rejected": -2.625, "logps/chosen": -520.0, "logps/rejected": -852.0, "loss": 0.1705, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.265625, "rewards/margins": 3.484375, "rewards/rejected": -6.75, "step": 12010 }, { "epoch": 0.9043713791287338, "grad_norm": 11.822701846296571, "learning_rate": 1.3801581083224544e-08, "logits/chosen": -2.71875, "logits/rejected": -2.703125, "logps/chosen": -536.0, "logps/rejected": -856.0, "loss": 0.1861, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.34375, "rewards/margins": 3.484375, "rewards/rejected": -6.84375, "step": 12020 }, { "epoch": 0.9051237679632834, "grad_norm": 12.61041649891951, "learning_rate": 1.3587240000799166e-08, "logits/chosen": -2.703125, "logits/rejected": -2.5625, "logps/chosen": -520.0, "logps/rejected": -880.0, "loss": 0.175, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.359375, "rewards/margins": 3.640625, "rewards/rejected": -7.0, "step": 12030 }, { "epoch": 0.9058761567978331, "grad_norm": 10.772900198007626, "learning_rate": 1.3374529848774685e-08, "logits/chosen": -2.59375, "logits/rejected": -2.65625, "logps/chosen": -536.0, "logps/rejected": -904.0, "loss": 0.1879, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.421875, "rewards/margins": 3.78125, "rewards/rejected": -7.21875, "step": 12040 }, { "epoch": 0.9066285456323828, "grad_norm": 15.423287199354897, "learning_rate": 1.3163452094565348e-08, "logits/chosen": -2.765625, "logits/rejected": -2.640625, "logps/chosen": -572.0, "logps/rejected": -860.0, "loss": 0.1732, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.65625, "rewards/margins": 3.109375, "rewards/rejected": -6.75, "step": 12050 }, { "epoch": 0.9073809344669325, "grad_norm": 13.570586905445042, "learning_rate": 1.2954008194324046e-08, "logits/chosen": -2.6875, "logits/rejected": -2.65625, "logps/chosen": -510.0, "logps/rejected": -864.0, "loss": 0.181, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.234375, "rewards/margins": 3.78125, "rewards/rejected": -7.03125, "step": 12060 }, { "epoch": 0.9081333233014822, "grad_norm": 11.490067154206665, "learning_rate": 1.2746199592932272e-08, "logits/chosen": -2.75, "logits/rejected": -2.640625, "logps/chosen": -544.0, "logps/rejected": -932.0, "loss": 0.1872, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.578125, "rewards/margins": 3.921875, "rewards/rejected": -7.5, "step": 12070 }, { "epoch": 0.9088857121360319, "grad_norm": 13.97211418524147, "learning_rate": 1.254002772399021e-08, "logits/chosen": -2.640625, "logits/rejected": -2.6875, "logps/chosen": -494.0, "logps/rejected": -852.0, "loss": 0.1993, "rewards/accuracies": 0.90625, "rewards/chosen": -3.15625, "rewards/margins": 3.6875, "rewards/rejected": -6.84375, "step": 12080 }, { "epoch": 0.9096381009705816, "grad_norm": 16.460708679223075, "learning_rate": 1.2335494009806712e-08, "logits/chosen": -2.625, "logits/rejected": -2.78125, "logps/chosen": -548.0, "logps/rejected": -856.0, "loss": 0.199, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.46875, "rewards/margins": 3.515625, "rewards/rejected": -6.96875, "step": 12090 }, { "epoch": 0.9103904898051313, "grad_norm": 11.592938945546745, "learning_rate": 1.2132599861389591e-08, "logits/chosen": -2.765625, "logits/rejected": -2.625, "logps/chosen": -528.0, "logps/rejected": -884.0, "loss": 0.1854, "rewards/accuracies": 0.9375, "rewards/chosen": -3.296875, "rewards/margins": 3.640625, "rewards/rejected": -6.9375, "step": 12100 }, { "epoch": 0.911142878639681, "grad_norm": 8.879502815879967, "learning_rate": 1.1931346678435872e-08, "logits/chosen": -2.828125, "logits/rejected": -2.53125, "logps/chosen": -504.0, "logps/rejected": -904.0, "loss": 0.1769, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.203125, "rewards/margins": 3.984375, "rewards/rejected": -7.1875, "step": 12110 }, { "epoch": 0.9118952674742307, "grad_norm": 15.454584313955285, "learning_rate": 1.1731735849322077e-08, "logits/chosen": -2.6875, "logits/rejected": -2.625, "logps/chosen": -516.0, "logps/rejected": -844.0, "loss": 0.1793, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.1875, "rewards/margins": 3.46875, "rewards/rejected": -6.65625, "step": 12120 }, { "epoch": 0.9126476563087804, "grad_norm": 18.272152209541336, "learning_rate": 1.1533768751094798e-08, "logits/chosen": -2.703125, "logits/rejected": -2.6875, "logps/chosen": -544.0, "logps/rejected": -884.0, "loss": 0.1942, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.40625, "rewards/margins": 3.625, "rewards/rejected": -7.03125, "step": 12130 }, { "epoch": 0.9134000451433301, "grad_norm": 14.629740578242856, "learning_rate": 1.1337446749461021e-08, "logits/chosen": -2.625, "logits/rejected": -2.515625, "logps/chosen": -520.0, "logps/rejected": -888.0, "loss": 0.1659, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -3.234375, "rewards/margins": 3.6875, "rewards/rejected": -6.9375, "step": 12140 }, { "epoch": 0.9141524339778798, "grad_norm": 15.03054484908946, "learning_rate": 1.1142771198778683e-08, "logits/chosen": -2.75, "logits/rejected": -2.75, "logps/chosen": -510.0, "logps/rejected": -840.0, "loss": 0.2027, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.296875, "rewards/margins": 3.359375, "rewards/rejected": -6.65625, "step": 12150 }, { "epoch": 0.9149048228124295, "grad_norm": 16.276004544211936, "learning_rate": 1.0949743442047632e-08, "logits/chosen": -2.609375, "logits/rejected": -2.296875, "logps/chosen": -512.0, "logps/rejected": -916.0, "loss": 0.1844, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.15625, "rewards/margins": 4.15625, "rewards/rejected": -7.3125, "step": 12160 }, { "epoch": 0.9156572116469791, "grad_norm": 11.883745763868845, "learning_rate": 1.0758364810899977e-08, "logits/chosen": -2.765625, "logits/rejected": -2.609375, "logps/chosen": -508.0, "logps/rejected": -848.0, "loss": 0.2091, "rewards/accuracies": 0.875, "rewards/chosen": -3.359375, "rewards/margins": 3.328125, "rewards/rejected": -6.6875, "step": 12170 }, { "epoch": 0.9164096004815289, "grad_norm": 13.041104567211635, "learning_rate": 1.056863662559121e-08, "logits/chosen": -2.59375, "logits/rejected": -2.375, "logps/chosen": -528.0, "logps/rejected": -880.0, "loss": 0.1919, "rewards/accuracies": 0.90625, "rewards/chosen": -3.3125, "rewards/margins": 3.6875, "rewards/rejected": -7.0, "step": 12180 }, { "epoch": 0.9171619893160785, "grad_norm": 13.785804876318807, "learning_rate": 1.0380560194990784e-08, "logits/chosen": -2.609375, "logits/rejected": -2.640625, "logps/chosen": -524.0, "logps/rejected": -864.0, "loss": 0.1843, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.21875, "rewards/margins": 3.796875, "rewards/rejected": -7.03125, "step": 12190 }, { "epoch": 0.9179143781506283, "grad_norm": 10.406219168833275, "learning_rate": 1.0194136816573411e-08, "logits/chosen": -2.65625, "logits/rejected": -2.6875, "logps/chosen": -544.0, "logps/rejected": -844.0, "loss": 0.1808, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.4375, "rewards/margins": 3.28125, "rewards/rejected": -6.71875, "step": 12200 }, { "epoch": 0.9186667669851779, "grad_norm": 18.739192214900353, "learning_rate": 1.0009367776409982e-08, "logits/chosen": -2.640625, "logits/rejected": -2.6875, "logps/chosen": -516.0, "logps/rejected": -908.0, "loss": 0.1956, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.265625, "rewards/margins": 3.984375, "rewards/rejected": -7.25, "step": 12210 }, { "epoch": 0.9194191558197277, "grad_norm": 11.545719481610336, "learning_rate": 9.826254349158513e-09, "logits/chosen": -2.59375, "logits/rejected": -2.65625, "logps/chosen": -510.0, "logps/rejected": -852.0, "loss": 0.1908, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.25, "rewards/margins": 3.59375, "rewards/rejected": -6.84375, "step": 12220 }, { "epoch": 0.9201715446542773, "grad_norm": 13.95012410013233, "learning_rate": 9.644797798055743e-09, "logits/chosen": -2.734375, "logits/rejected": -2.578125, "logps/chosen": -508.0, "logps/rejected": -860.0, "loss": 0.1903, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.328125, "rewards/margins": 3.5, "rewards/rejected": -6.84375, "step": 12230 }, { "epoch": 0.9209239334888271, "grad_norm": 10.624451620714355, "learning_rate": 9.464999374907994e-09, "logits/chosen": -2.71875, "logits/rejected": -2.703125, "logps/chosen": -468.0, "logps/rejected": -868.0, "loss": 0.19, "rewards/accuracies": 0.9375, "rewards/chosen": -3.03125, "rewards/margins": 3.90625, "rewards/rejected": -6.9375, "step": 12240 }, { "epoch": 0.9216763223233767, "grad_norm": 17.354026234473952, "learning_rate": 9.286860320082801e-09, "logits/chosen": -2.71875, "logits/rejected": -2.625, "logps/chosen": -524.0, "logps/rejected": -872.0, "loss": 0.1902, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -3.390625, "rewards/margins": 3.5625, "rewards/rejected": -6.96875, "step": 12250 }, { "epoch": 0.9224287111579265, "grad_norm": 9.538317179449727, "learning_rate": 9.110381862500349e-09, "logits/chosen": -2.703125, "logits/rejected": -2.59375, "logps/chosen": -520.0, "logps/rejected": -904.0, "loss": 0.1633, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.25, "rewards/margins": 3.75, "rewards/rejected": -7.0, "step": 12260 }, { "epoch": 0.9231810999924761, "grad_norm": 15.25358227095578, "learning_rate": 8.935565219624852e-09, "logits/chosen": -2.5625, "logits/rejected": -2.46875, "logps/chosen": -516.0, "logps/rejected": -900.0, "loss": 0.1825, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.21875, "rewards/margins": 3.96875, "rewards/rejected": -7.1875, "step": 12270 }, { "epoch": 0.9239334888270259, "grad_norm": 14.562125314525952, "learning_rate": 8.762411597456249e-09, "logits/chosen": -2.703125, "logits/rejected": -2.53125, "logps/chosen": -532.0, "logps/rejected": -916.0, "loss": 0.1962, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.5, "rewards/margins": 3.796875, "rewards/rejected": -7.28125, "step": 12280 }, { "epoch": 0.9246858776615755, "grad_norm": 17.288104148861272, "learning_rate": 8.590922190521904e-09, "logits/chosen": -2.578125, "logits/rejected": -2.5625, "logps/chosen": -536.0, "logps/rejected": -892.0, "loss": 0.1981, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.40625, "rewards/margins": 3.84375, "rewards/rejected": -7.25, "step": 12290 }, { "epoch": 0.9254382664961252, "grad_norm": 9.03682938844224, "learning_rate": 8.421098181868285e-09, "logits/chosen": -2.625, "logits/rejected": -2.6875, "logps/chosen": -500.0, "logps/rejected": -816.0, "loss": 0.1858, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.09375, "rewards/margins": 3.421875, "rewards/rejected": -6.5, "step": 12300 }, { "epoch": 0.9261906553306749, "grad_norm": 12.79304128035952, "learning_rate": 8.25294074305291e-09, "logits/chosen": -2.765625, "logits/rejected": -2.59375, "logps/chosen": -512.0, "logps/rejected": -872.0, "loss": 0.1704, "rewards/accuracies": 0.9375, "rewards/chosen": -3.359375, "rewards/margins": 3.671875, "rewards/rejected": -7.03125, "step": 12310 }, { "epoch": 0.9269430441652246, "grad_norm": 13.56668460146615, "learning_rate": 8.086451034136187e-09, "logits/chosen": -2.75, "logits/rejected": -2.4375, "logps/chosen": -504.0, "logps/rejected": -880.0, "loss": 0.1806, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.15625, "rewards/margins": 3.84375, "rewards/rejected": -7.0, "step": 12320 }, { "epoch": 0.9276954329997743, "grad_norm": 10.870922185697514, "learning_rate": 7.921630203673341e-09, "logits/chosen": -2.8125, "logits/rejected": -2.890625, "logps/chosen": -506.0, "logps/rejected": -864.0, "loss": 0.1845, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.109375, "rewards/margins": 3.84375, "rewards/rejected": -6.96875, "step": 12330 }, { "epoch": 0.928447821834324, "grad_norm": 16.42292931119488, "learning_rate": 7.758479388706718e-09, "logits/chosen": -2.6875, "logits/rejected": -2.546875, "logps/chosen": -520.0, "logps/rejected": -912.0, "loss": 0.1928, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.25, "rewards/margins": 3.96875, "rewards/rejected": -7.21875, "step": 12340 }, { "epoch": 0.9292002106688737, "grad_norm": 13.764787045811413, "learning_rate": 7.596999714757718e-09, "logits/chosen": -2.703125, "logits/rejected": -2.703125, "logps/chosen": -506.0, "logps/rejected": -876.0, "loss": 0.1748, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -3.09375, "rewards/margins": 3.921875, "rewards/rejected": -7.03125, "step": 12350 }, { "epoch": 0.9299525995034233, "grad_norm": 12.376886959684262, "learning_rate": 7.4371922958191e-09, "logits/chosen": -2.625, "logits/rejected": -2.359375, "logps/chosen": -512.0, "logps/rejected": -864.0, "loss": 0.1737, "rewards/accuracies": 0.9375, "rewards/chosen": -3.25, "rewards/margins": 3.65625, "rewards/rejected": -6.90625, "step": 12360 }, { "epoch": 0.9307049883379731, "grad_norm": 11.09009236302399, "learning_rate": 7.279058234347352e-09, "logits/chosen": -2.71875, "logits/rejected": -2.625, "logps/chosen": -488.0, "logps/rejected": -892.0, "loss": 0.1785, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -2.9375, "rewards/margins": 4.125, "rewards/rejected": -7.0625, "step": 12370 }, { "epoch": 0.9314573771725227, "grad_norm": 9.621016154247043, "learning_rate": 7.122598621255027e-09, "logits/chosen": -2.609375, "logits/rejected": -2.734375, "logps/chosen": -540.0, "logps/rejected": -856.0, "loss": 0.1768, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.46875, "rewards/margins": 3.390625, "rewards/rejected": -6.84375, "step": 12380 }, { "epoch": 0.9322097660070725, "grad_norm": 12.35154989810525, "learning_rate": 6.967814535903283e-09, "logits/chosen": -2.53125, "logits/rejected": -2.59375, "logps/chosen": -544.0, "logps/rejected": -868.0, "loss": 0.191, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.359375, "rewards/margins": 3.484375, "rewards/rejected": -6.84375, "step": 12390 }, { "epoch": 0.9329621548416221, "grad_norm": 14.309357542466383, "learning_rate": 6.81470704609427e-09, "logits/chosen": -2.671875, "logits/rejected": -2.6875, "logps/chosen": -510.0, "logps/rejected": -928.0, "loss": 0.1774, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -3.140625, "rewards/margins": 4.375, "rewards/rejected": -7.5, "step": 12400 }, { "epoch": 0.9337145436761718, "grad_norm": 12.617796635797982, "learning_rate": 6.6632772080639775e-09, "logits/chosen": -2.609375, "logits/rejected": -2.640625, "logps/chosen": -524.0, "logps/rejected": -868.0, "loss": 0.1759, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.3125, "rewards/margins": 3.65625, "rewards/rejected": -6.96875, "step": 12410 }, { "epoch": 0.9344669325107215, "grad_norm": 13.493100138022534, "learning_rate": 6.513526066474873e-09, "logits/chosen": -2.703125, "logits/rejected": -2.4375, "logps/chosen": -524.0, "logps/rejected": -868.0, "loss": 0.1734, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.296875, "rewards/margins": 3.640625, "rewards/rejected": -6.9375, "step": 12420 }, { "epoch": 0.9352193213452712, "grad_norm": 15.906818208925205, "learning_rate": 6.365454654408547e-09, "logits/chosen": -2.71875, "logits/rejected": -2.5625, "logps/chosen": -524.0, "logps/rejected": -868.0, "loss": 0.1858, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.421875, "rewards/margins": 3.609375, "rewards/rejected": -7.03125, "step": 12430 }, { "epoch": 0.935971710179821, "grad_norm": 12.12995809442107, "learning_rate": 6.219063993358864e-09, "logits/chosen": -2.640625, "logits/rejected": -2.46875, "logps/chosen": -544.0, "logps/rejected": -912.0, "loss": 0.1843, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.375, "rewards/margins": 3.828125, "rewards/rejected": -7.1875, "step": 12440 }, { "epoch": 0.9367240990143706, "grad_norm": 14.434996647206258, "learning_rate": 6.0743550932247086e-09, "logits/chosen": -2.65625, "logits/rejected": -2.640625, "logps/chosen": -528.0, "logps/rejected": -860.0, "loss": 0.1756, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.390625, "rewards/margins": 3.546875, "rewards/rejected": -6.9375, "step": 12450 }, { "epoch": 0.9374764878489203, "grad_norm": 12.539603427354898, "learning_rate": 5.931328952302972e-09, "logits/chosen": -2.640625, "logits/rejected": -2.59375, "logps/chosen": -552.0, "logps/rejected": -876.0, "loss": 0.1887, "rewards/accuracies": 0.875, "rewards/chosen": -3.546875, "rewards/margins": 3.375, "rewards/rejected": -6.90625, "step": 12460 }, { "epoch": 0.93822887668347, "grad_norm": 14.59214657769156, "learning_rate": 5.7899865572819116e-09, "logits/chosen": -2.6875, "logits/rejected": -2.75, "logps/chosen": -508.0, "logps/rejected": -892.0, "loss": 0.1831, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.1875, "rewards/margins": 3.890625, "rewards/rejected": -7.09375, "step": 12470 }, { "epoch": 0.9389812655180197, "grad_norm": 13.84568932472763, "learning_rate": 5.650328883234134e-09, "logits/chosen": -2.640625, "logits/rejected": -2.671875, "logps/chosen": -504.0, "logps/rejected": -832.0, "loss": 0.189, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.25, "rewards/margins": 3.46875, "rewards/rejected": -6.71875, "step": 12480 }, { "epoch": 0.9397336543525694, "grad_norm": 8.601323493423363, "learning_rate": 5.512356893609987e-09, "logits/chosen": -2.640625, "logits/rejected": -2.5, "logps/chosen": -532.0, "logps/rejected": -904.0, "loss": 0.1732, "rewards/accuracies": 0.9375, "rewards/chosen": -3.34375, "rewards/margins": 3.78125, "rewards/rejected": -7.125, "step": 12490 }, { "epoch": 0.9404860431871191, "grad_norm": 12.050053209714699, "learning_rate": 5.376071540230787e-09, "logits/chosen": -2.65625, "logits/rejected": -2.6875, "logps/chosen": -520.0, "logps/rejected": -868.0, "loss": 0.1887, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.375, "rewards/margins": 3.625, "rewards/rejected": -7.03125, "step": 12500 }, { "epoch": 0.9412384320216688, "grad_norm": 13.679604300278124, "learning_rate": 5.24147376328235e-09, "logits/chosen": -2.609375, "logits/rejected": -2.4375, "logps/chosen": -490.0, "logps/rejected": -852.0, "loss": 0.1792, "rewards/accuracies": 0.96875, "rewards/chosen": -2.953125, "rewards/margins": 3.75, "rewards/rejected": -6.71875, "step": 12510 }, { "epoch": 0.9419908208562185, "grad_norm": 14.778175441828465, "learning_rate": 5.108564491308504e-09, "logits/chosen": -2.609375, "logits/rejected": -2.59375, "logps/chosen": -504.0, "logps/rejected": -864.0, "loss": 0.1804, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.125, "rewards/margins": 3.703125, "rewards/rejected": -6.8125, "step": 12520 }, { "epoch": 0.9427432096907682, "grad_norm": 14.521302186875847, "learning_rate": 4.9773446412046675e-09, "logits/chosen": -2.515625, "logits/rejected": -2.359375, "logps/chosen": -528.0, "logps/rejected": -892.0, "loss": 0.1666, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.3125, "rewards/margins": 3.765625, "rewards/rejected": -7.0625, "step": 12530 }, { "epoch": 0.9434955985253178, "grad_norm": 13.376705800818476, "learning_rate": 4.8478151182114735e-09, "logits/chosen": -2.640625, "logits/rejected": -2.515625, "logps/chosen": -528.0, "logps/rejected": -888.0, "loss": 0.1995, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.15625, "rewards/margins": 3.9375, "rewards/rejected": -7.09375, "step": 12540 }, { "epoch": 0.9442479873598676, "grad_norm": 10.014617355802963, "learning_rate": 4.719976815908605e-09, "logits/chosen": -2.671875, "logits/rejected": -2.609375, "logps/chosen": -540.0, "logps/rejected": -916.0, "loss": 0.2009, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.421875, "rewards/margins": 3.984375, "rewards/rejected": -7.40625, "step": 12550 }, { "epoch": 0.9450003761944172, "grad_norm": 12.22514577144868, "learning_rate": 4.59383061620855e-09, "logits/chosen": -2.671875, "logits/rejected": -2.53125, "logps/chosen": -516.0, "logps/rejected": -896.0, "loss": 0.18, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.21875, "rewards/margins": 3.84375, "rewards/rejected": -7.0625, "step": 12560 }, { "epoch": 0.945752765028967, "grad_norm": 12.238840481167603, "learning_rate": 4.469377389350659e-09, "logits/chosen": -2.640625, "logits/rejected": -2.71875, "logps/chosen": -524.0, "logps/rejected": -860.0, "loss": 0.1628, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.203125, "rewards/margins": 3.75, "rewards/rejected": -6.9375, "step": 12570 }, { "epoch": 0.9465051538635166, "grad_norm": 15.138861157543811, "learning_rate": 4.3466179938949635e-09, "logits/chosen": -2.703125, "logits/rejected": -2.734375, "logps/chosen": -486.0, "logps/rejected": -844.0, "loss": 0.175, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.046875, "rewards/margins": 3.796875, "rewards/rejected": -6.84375, "step": 12580 }, { "epoch": 0.9472575426980664, "grad_norm": 13.813710751386672, "learning_rate": 4.225553276716309e-09, "logits/chosen": -2.578125, "logits/rejected": -2.609375, "logps/chosen": -516.0, "logps/rejected": -888.0, "loss": 0.1925, "rewards/accuracies": 0.9375, "rewards/chosen": -3.21875, "rewards/margins": 3.90625, "rewards/rejected": -7.125, "step": 12590 }, { "epoch": 0.948009931532616, "grad_norm": 14.234790938733639, "learning_rate": 4.106184072998647e-09, "logits/chosen": -2.828125, "logits/rejected": -2.75, "logps/chosen": -524.0, "logps/rejected": -872.0, "loss": 0.1823, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.390625, "rewards/margins": 3.734375, "rewards/rejected": -7.125, "step": 12600 }, { "epoch": 0.9487623203671658, "grad_norm": 14.462216169206338, "learning_rate": 3.988511206229062e-09, "logits/chosen": -2.75, "logits/rejected": -2.703125, "logps/chosen": -524.0, "logps/rejected": -864.0, "loss": 0.1836, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.4375, "rewards/margins": 3.46875, "rewards/rejected": -6.90625, "step": 12610 }, { "epoch": 0.9495147092017154, "grad_norm": 12.4993661318265, "learning_rate": 3.872535488192247e-09, "logits/chosen": -2.625, "logits/rejected": -2.578125, "logps/chosen": -528.0, "logps/rejected": -904.0, "loss": 0.1907, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -3.25, "rewards/margins": 3.9375, "rewards/rejected": -7.1875, "step": 12620 }, { "epoch": 0.9502670980362652, "grad_norm": 17.561798408064295, "learning_rate": 3.7582577189648194e-09, "logits/chosen": -2.59375, "logits/rejected": -2.59375, "logps/chosen": -508.0, "logps/rejected": -844.0, "loss": 0.1815, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.3125, "rewards/margins": 3.453125, "rewards/rejected": -6.78125, "step": 12630 }, { "epoch": 0.9510194868708148, "grad_norm": 19.126014511486506, "learning_rate": 3.64567868690982e-09, "logits/chosen": -2.640625, "logits/rejected": -2.46875, "logps/chosen": -564.0, "logps/rejected": -896.0, "loss": 0.2058, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.625, "rewards/margins": 3.515625, "rewards/rejected": -7.125, "step": 12640 }, { "epoch": 0.9517718757053645, "grad_norm": 16.50507197978361, "learning_rate": 3.53479916867136e-09, "logits/chosen": -2.796875, "logits/rejected": -2.734375, "logps/chosen": -506.0, "logps/rejected": -880.0, "loss": 0.179, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.28125, "rewards/margins": 3.796875, "rewards/rejected": -7.09375, "step": 12650 }, { "epoch": 0.9525242645399142, "grad_norm": 13.825306204344907, "learning_rate": 3.4256199291691214e-09, "logits/chosen": -2.703125, "logits/rejected": -2.609375, "logps/chosen": -532.0, "logps/rejected": -932.0, "loss": 0.188, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.375, "rewards/margins": 4.0625, "rewards/rejected": -7.46875, "step": 12660 }, { "epoch": 0.9532766533744639, "grad_norm": 10.115839585043656, "learning_rate": 3.318141721593143e-09, "logits/chosen": -2.703125, "logits/rejected": -2.828125, "logps/chosen": -512.0, "logps/rejected": -864.0, "loss": 0.1822, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.3125, "rewards/margins": 3.671875, "rewards/rejected": -6.96875, "step": 12670 }, { "epoch": 0.9540290422090136, "grad_norm": 13.149949744890375, "learning_rate": 3.212365287398655e-09, "logits/chosen": -2.578125, "logits/rejected": -2.46875, "logps/chosen": -536.0, "logps/rejected": -892.0, "loss": 0.2137, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.53125, "rewards/margins": 3.46875, "rewards/rejected": -7.0, "step": 12680 }, { "epoch": 0.9547814310435633, "grad_norm": 14.024550609833657, "learning_rate": 3.1082913563009737e-09, "logits/chosen": -2.671875, "logits/rejected": -2.46875, "logps/chosen": -512.0, "logps/rejected": -920.0, "loss": 0.1782, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.1875, "rewards/margins": 3.984375, "rewards/rejected": -7.15625, "step": 12690 }, { "epoch": 0.955533819878113, "grad_norm": 12.491818250915493, "learning_rate": 3.00592064627031e-09, "logits/chosen": -2.734375, "logits/rejected": -2.5625, "logps/chosen": -512.0, "logps/rejected": -904.0, "loss": 0.2026, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.296875, "rewards/margins": 4.0, "rewards/rejected": -7.3125, "step": 12700 }, { "epoch": 0.9562862087126627, "grad_norm": 13.92084718441631, "learning_rate": 2.905253863527107e-09, "logits/chosen": -2.65625, "logits/rejected": -2.734375, "logps/chosen": -520.0, "logps/rejected": -880.0, "loss": 0.1757, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.359375, "rewards/margins": 3.765625, "rewards/rejected": -7.125, "step": 12710 }, { "epoch": 0.9570385975472124, "grad_norm": 11.787892981973815, "learning_rate": 2.8062917025368504e-09, "logits/chosen": -2.796875, "logits/rejected": -2.84375, "logps/chosen": -520.0, "logps/rejected": -868.0, "loss": 0.1841, "rewards/accuracies": 0.90625, "rewards/chosen": -3.3125, "rewards/margins": 3.609375, "rewards/rejected": -6.9375, "step": 12720 }, { "epoch": 0.9577909863817621, "grad_norm": 11.812039246565192, "learning_rate": 2.7090348460055146e-09, "logits/chosen": -2.671875, "logits/rejected": -2.578125, "logps/chosen": -524.0, "logps/rejected": -912.0, "loss": 0.199, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.25, "rewards/margins": 3.96875, "rewards/rejected": -7.21875, "step": 12730 }, { "epoch": 0.9585433752163118, "grad_norm": 9.617177032808774, "learning_rate": 2.6134839648747075e-09, "logits/chosen": -2.765625, "logits/rejected": -2.875, "logps/chosen": -528.0, "logps/rejected": -840.0, "loss": 0.1802, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.359375, "rewards/margins": 3.25, "rewards/rejected": -6.625, "step": 12740 }, { "epoch": 0.9592957640508615, "grad_norm": 11.580752274389772, "learning_rate": 2.519639718317146e-09, "logits/chosen": -2.625, "logits/rejected": -2.375, "logps/chosen": -540.0, "logps/rejected": -936.0, "loss": 0.1879, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.4375, "rewards/margins": 3.828125, "rewards/rejected": -7.25, "step": 12750 }, { "epoch": 0.9600481528854112, "grad_norm": 13.099601395954553, "learning_rate": 2.4275027537320745e-09, "logits/chosen": -2.53125, "logits/rejected": -2.375, "logps/chosen": -532.0, "logps/rejected": -884.0, "loss": 0.1946, "rewards/accuracies": 0.9375, "rewards/chosen": -3.421875, "rewards/margins": 3.6875, "rewards/rejected": -7.125, "step": 12760 }, { "epoch": 0.9608005417199609, "grad_norm": 16.226493013723722, "learning_rate": 2.3370737067406886e-09, "logits/chosen": -2.828125, "logits/rejected": -2.71875, "logps/chosen": -498.0, "logps/rejected": -868.0, "loss": 0.1717, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.140625, "rewards/margins": 3.859375, "rewards/rejected": -7.0, "step": 12770 }, { "epoch": 0.9615529305545105, "grad_norm": 10.728190445235656, "learning_rate": 2.2483532011819408e-09, "logits/chosen": -2.640625, "logits/rejected": -2.5, "logps/chosen": -490.0, "logps/rejected": -912.0, "loss": 0.1858, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -2.9375, "rewards/margins": 4.34375, "rewards/rejected": -7.28125, "step": 12780 }, { "epoch": 0.9623053193890603, "grad_norm": 13.571215538424822, "learning_rate": 2.161341849108156e-09, "logits/chosen": -2.6875, "logits/rejected": -2.609375, "logps/chosen": -520.0, "logps/rejected": -860.0, "loss": 0.1742, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.3125, "rewards/margins": 3.625, "rewards/rejected": -6.9375, "step": 12790 }, { "epoch": 0.9630577082236099, "grad_norm": 13.331183064568759, "learning_rate": 2.076040250780675e-09, "logits/chosen": -2.625, "logits/rejected": -2.859375, "logps/chosen": -512.0, "logps/rejected": -828.0, "loss": 0.1712, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.1875, "rewards/margins": 3.453125, "rewards/rejected": -6.625, "step": 12800 }, { "epoch": 0.9638100970581597, "grad_norm": 10.264290394956205, "learning_rate": 1.9924489946659963e-09, "logits/chosen": -2.65625, "logits/rejected": -2.65625, "logps/chosen": -524.0, "logps/rejected": -876.0, "loss": 0.1711, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.375, "rewards/margins": 3.71875, "rewards/rejected": -7.09375, "step": 12810 }, { "epoch": 0.9645624858927093, "grad_norm": 11.093898150942696, "learning_rate": 1.910568657431416e-09, "logits/chosen": -2.796875, "logits/rejected": -2.53125, "logps/chosen": -524.0, "logps/rejected": -868.0, "loss": 0.2083, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.40625, "rewards/margins": 3.71875, "rewards/rejected": -7.125, "step": 12820 }, { "epoch": 0.9653148747272591, "grad_norm": 11.630528040327542, "learning_rate": 1.830399803941285e-09, "logits/chosen": -2.65625, "logits/rejected": -2.546875, "logps/chosen": -516.0, "logps/rejected": -844.0, "loss": 0.1756, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.4375, "rewards/margins": 3.4375, "rewards/rejected": -6.875, "step": 12830 }, { "epoch": 0.9660672635618087, "grad_norm": 7.615478243974393, "learning_rate": 1.7519429872529523e-09, "logits/chosen": -2.78125, "logits/rejected": -2.65625, "logps/chosen": -484.0, "logps/rejected": -856.0, "loss": 0.182, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.125, "rewards/margins": 3.75, "rewards/rejected": -6.875, "step": 12840 }, { "epoch": 0.9668196523963585, "grad_norm": 10.730954655137392, "learning_rate": 1.6751987486130493e-09, "logits/chosen": -2.734375, "logits/rejected": -2.46875, "logps/chosen": -504.0, "logps/rejected": -876.0, "loss": 0.1811, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.21875, "rewards/margins": 3.75, "rewards/rejected": -6.96875, "step": 12850 }, { "epoch": 0.9675720412309081, "grad_norm": 13.648418763379283, "learning_rate": 1.6001676174537127e-09, "logits/chosen": -2.6875, "logits/rejected": -2.625, "logps/chosen": -490.0, "logps/rejected": -864.0, "loss": 0.1765, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.1875, "rewards/margins": 3.625, "rewards/rejected": -6.8125, "step": 12860 }, { "epoch": 0.9683244300654579, "grad_norm": 12.521116330054422, "learning_rate": 1.526850111388922e-09, "logits/chosen": -2.65625, "logits/rejected": -2.65625, "logps/chosen": -528.0, "logps/rejected": -884.0, "loss": 0.1997, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.328125, "rewards/margins": 3.703125, "rewards/rejected": -7.03125, "step": 12870 }, { "epoch": 0.9690768189000075, "grad_norm": 8.283644353778895, "learning_rate": 1.4552467362109744e-09, "logits/chosen": -2.59375, "logits/rejected": -2.515625, "logps/chosen": -516.0, "logps/rejected": -884.0, "loss": 0.1801, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.1875, "rewards/margins": 3.84375, "rewards/rejected": -7.03125, "step": 12880 }, { "epoch": 0.9698292077345572, "grad_norm": 16.292608918793285, "learning_rate": 1.3853579858869313e-09, "logits/chosen": -2.65625, "logits/rejected": -2.546875, "logps/chosen": -520.0, "logps/rejected": -864.0, "loss": 0.1827, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.25, "rewards/margins": 3.609375, "rewards/rejected": -6.875, "step": 12890 }, { "epoch": 0.9705815965691069, "grad_norm": 14.414351565276329, "learning_rate": 1.3171843425553163e-09, "logits/chosen": -2.78125, "logits/rejected": -2.640625, "logps/chosen": -506.0, "logps/rejected": -884.0, "loss": 0.189, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.328125, "rewards/margins": 3.875, "rewards/rejected": -7.1875, "step": 12900 }, { "epoch": 0.9713339854036566, "grad_norm": 12.036567824182415, "learning_rate": 1.250726276522618e-09, "logits/chosen": -2.65625, "logits/rejected": -2.703125, "logps/chosen": -524.0, "logps/rejected": -852.0, "loss": 0.1847, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.25, "rewards/margins": 3.65625, "rewards/rejected": -6.90625, "step": 12910 }, { "epoch": 0.9720863742382063, "grad_norm": 13.286632591665281, "learning_rate": 1.1859842462602077e-09, "logits/chosen": -2.71875, "logits/rejected": -2.4375, "logps/chosen": -536.0, "logps/rejected": -860.0, "loss": 0.1868, "rewards/accuracies": 0.90625, "rewards/chosen": -3.421875, "rewards/margins": 3.390625, "rewards/rejected": -6.8125, "step": 12920 }, { "epoch": 0.972838763072756, "grad_norm": 13.426089606523835, "learning_rate": 1.1229586984011496e-09, "logits/chosen": -2.4375, "logits/rejected": -2.765625, "logps/chosen": -548.0, "logps/rejected": -856.0, "loss": 0.1859, "rewards/accuracies": 0.90625, "rewards/chosen": -3.40625, "rewards/margins": 3.46875, "rewards/rejected": -6.875, "step": 12930 }, { "epoch": 0.9735911519073057, "grad_norm": 13.28726043168659, "learning_rate": 1.0616500677369799e-09, "logits/chosen": -2.609375, "logits/rejected": -2.46875, "logps/chosen": -544.0, "logps/rejected": -916.0, "loss": 0.1713, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.46875, "rewards/margins": 3.78125, "rewards/rejected": -7.25, "step": 12940 }, { "epoch": 0.9743435407418554, "grad_norm": 12.332103558420394, "learning_rate": 1.0020587772149314e-09, "logits/chosen": -2.78125, "logits/rejected": -2.734375, "logps/chosen": -508.0, "logps/rejected": -852.0, "loss": 0.1928, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.25, "rewards/margins": 3.515625, "rewards/rejected": -6.78125, "step": 12950 }, { "epoch": 0.9750959295764051, "grad_norm": 14.289328310714357, "learning_rate": 9.441852379347969e-10, "logits/chosen": -2.6875, "logits/rejected": -2.484375, "logps/chosen": -516.0, "logps/rejected": -888.0, "loss": 0.159, "rewards/accuracies": 0.9375, "rewards/chosen": -3.203125, "rewards/margins": 3.84375, "rewards/rejected": -7.0625, "step": 12960 }, { "epoch": 0.9758483184109548, "grad_norm": 10.566003193474794, "learning_rate": 8.880298491462934e-10, "logits/chosen": -2.671875, "logits/rejected": -2.390625, "logps/chosen": -540.0, "logps/rejected": -912.0, "loss": 0.1761, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.484375, "rewards/margins": 3.84375, "rewards/rejected": -7.3125, "step": 12970 }, { "epoch": 0.9766007072455045, "grad_norm": 12.820550959056012, "learning_rate": 8.335929982460909e-10, "logits/chosen": -2.71875, "logits/rejected": -2.796875, "logps/chosen": -512.0, "logps/rejected": -840.0, "loss": 0.1938, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.15625, "rewards/margins": 3.515625, "rewards/rejected": -6.6875, "step": 12980 }, { "epoch": 0.9773530960800542, "grad_norm": 13.148624794946386, "learning_rate": 7.808750607753711e-10, "logits/chosen": -2.671875, "logits/rejected": -2.71875, "logps/chosen": -532.0, "logps/rejected": -876.0, "loss": 0.2004, "rewards/accuracies": 0.90625, "rewards/chosen": -3.328125, "rewards/margins": 3.765625, "rewards/rejected": -7.09375, "step": 12990 }, { "epoch": 0.9781054849146039, "grad_norm": 17.393847162676682, "learning_rate": 7.29876400417051e-10, "logits/chosen": -2.625, "logits/rejected": -2.515625, "logps/chosen": -508.0, "logps/rejected": -844.0, "loss": 0.1775, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.1875, "rewards/margins": 3.578125, "rewards/rejected": -6.75, "step": 13000 }, { "epoch": 0.9788578737491536, "grad_norm": 11.43509024553458, "learning_rate": 6.805973689933408e-10, "logits/chosen": -2.71875, "logits/rejected": -2.671875, "logps/chosen": -516.0, "logps/rejected": -856.0, "loss": 0.2009, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -3.421875, "rewards/margins": 3.515625, "rewards/rejected": -6.9375, "step": 13010 }, { "epoch": 0.9796102625837032, "grad_norm": 15.795226589492664, "learning_rate": 6.330383064633849e-10, "logits/chosen": -2.84375, "logits/rejected": -2.859375, "logps/chosen": -516.0, "logps/rejected": -852.0, "loss": 0.2017, "rewards/accuracies": 0.9375, "rewards/chosen": -3.265625, "rewards/margins": 3.609375, "rewards/rejected": -6.875, "step": 13020 }, { "epoch": 0.980362651418253, "grad_norm": 15.459891781604293, "learning_rate": 5.871995409207908e-10, "logits/chosen": -2.703125, "logits/rejected": -2.6875, "logps/chosen": -508.0, "logps/rejected": -880.0, "loss": 0.205, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.234375, "rewards/margins": 3.875, "rewards/rejected": -7.09375, "step": 13030 }, { "epoch": 0.9811150402528026, "grad_norm": 9.566122049645333, "learning_rate": 5.430813885914653e-10, "logits/chosen": -2.546875, "logits/rejected": -2.59375, "logps/chosen": -506.0, "logps/rejected": -860.0, "loss": 0.192, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.171875, "rewards/margins": 3.90625, "rewards/rejected": -7.0625, "step": 13040 }, { "epoch": 0.9818674290873524, "grad_norm": 11.64589660773263, "learning_rate": 5.006841538313933e-10, "logits/chosen": -2.8125, "logits/rejected": -2.671875, "logps/chosen": -524.0, "logps/rejected": -876.0, "loss": 0.189, "rewards/accuracies": 0.9375, "rewards/chosen": -3.34375, "rewards/margins": 3.828125, "rewards/rejected": -7.1875, "step": 13050 }, { "epoch": 0.982619817921902, "grad_norm": 11.005645067640108, "learning_rate": 4.600081291245006e-10, "logits/chosen": -2.625, "logits/rejected": -2.671875, "logps/chosen": -512.0, "logps/rejected": -860.0, "loss": 0.1938, "rewards/accuracies": 0.875, "rewards/chosen": -3.265625, "rewards/margins": 3.515625, "rewards/rejected": -6.78125, "step": 13060 }, { "epoch": 0.9833722067564518, "grad_norm": 13.784951696355165, "learning_rate": 4.2105359508071147e-10, "logits/chosen": -2.640625, "logits/rejected": -2.34375, "logps/chosen": -556.0, "logps/rejected": -912.0, "loss": 0.1832, "rewards/accuracies": 0.9375, "rewards/chosen": -3.578125, "rewards/margins": 3.734375, "rewards/rejected": -7.3125, "step": 13070 }, { "epoch": 0.9841245955910014, "grad_norm": 12.837081104638157, "learning_rate": 3.8382082043400544e-10, "logits/chosen": -2.59375, "logits/rejected": -2.453125, "logps/chosen": -536.0, "logps/rejected": -904.0, "loss": 0.2005, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.34375, "rewards/margins": 3.75, "rewards/rejected": -7.09375, "step": 13080 }, { "epoch": 0.9848769844255512, "grad_norm": 13.401390941001852, "learning_rate": 3.4831006204044666e-10, "logits/chosen": -2.78125, "logits/rejected": -2.640625, "logps/chosen": -506.0, "logps/rejected": -908.0, "loss": 0.1857, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.34375, "rewards/margins": 3.921875, "rewards/rejected": -7.25, "step": 13090 }, { "epoch": 0.9856293732601008, "grad_norm": 12.355627168169, "learning_rate": 3.14521564876602e-10, "logits/chosen": -2.578125, "logits/rejected": -2.59375, "logps/chosen": -536.0, "logps/rejected": -888.0, "loss": 0.1748, "rewards/accuracies": 0.9375, "rewards/chosen": -3.34375, "rewards/margins": 3.734375, "rewards/rejected": -7.0625, "step": 13100 }, { "epoch": 0.9863817620946506, "grad_norm": 11.279318380850045, "learning_rate": 2.8245556203768116e-10, "logits/chosen": -2.765625, "logits/rejected": -2.6875, "logps/chosen": -524.0, "logps/rejected": -872.0, "loss": 0.1844, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.40625, "rewards/margins": 3.5625, "rewards/rejected": -6.96875, "step": 13110 }, { "epoch": 0.9871341509292002, "grad_norm": 13.240660052413096, "learning_rate": 2.5211227473603826e-10, "logits/chosen": -2.609375, "logits/rejected": -2.65625, "logps/chosen": -548.0, "logps/rejected": -896.0, "loss": 0.2163, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.4375, "rewards/margins": 3.640625, "rewards/rejected": -7.09375, "step": 13120 }, { "epoch": 0.98788653976375, "grad_norm": 12.392685279239085, "learning_rate": 2.2349191229956156e-10, "logits/chosen": -2.671875, "logits/rejected": -2.453125, "logps/chosen": -524.0, "logps/rejected": -840.0, "loss": 0.1903, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.40625, "rewards/margins": 3.359375, "rewards/rejected": -6.78125, "step": 13130 }, { "epoch": 0.9886389285982996, "grad_norm": 12.393748800452036, "learning_rate": 1.9659467217031377e-10, "logits/chosen": -2.703125, "logits/rejected": -2.6875, "logps/chosen": -528.0, "logps/rejected": -880.0, "loss": 0.1977, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.359375, "rewards/margins": 3.75, "rewards/rejected": -7.125, "step": 13140 }, { "epoch": 0.9893913174328492, "grad_norm": 13.07989897426148, "learning_rate": 1.7142073990308868e-10, "logits/chosen": -2.734375, "logits/rejected": -2.78125, "logps/chosen": -512.0, "logps/rejected": -904.0, "loss": 0.169, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.265625, "rewards/margins": 3.984375, "rewards/rejected": -7.25, "step": 13150 }, { "epoch": 0.990143706267399, "grad_norm": 18.42974433552264, "learning_rate": 1.4797028916424536e-10, "logits/chosen": -2.6875, "logits/rejected": -2.6875, "logps/chosen": -528.0, "logps/rejected": -880.0, "loss": 0.1987, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -3.34375, "rewards/margins": 3.53125, "rewards/rejected": -6.875, "step": 13160 }, { "epoch": 0.9908960951019486, "grad_norm": 14.17832118045331, "learning_rate": 1.2624348173034814e-10, "logits/chosen": -2.625, "logits/rejected": -2.53125, "logps/chosen": -512.0, "logps/rejected": -872.0, "loss": 0.1974, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -3.234375, "rewards/margins": 3.671875, "rewards/rejected": -6.90625, "step": 13170 }, { "epoch": 0.9916484839364984, "grad_norm": 17.19096683462614, "learning_rate": 1.0624046748716753e-10, "logits/chosen": -2.734375, "logits/rejected": -2.71875, "logps/chosen": -512.0, "logps/rejected": -916.0, "loss": 0.156, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -3.1875, "rewards/margins": 4.09375, "rewards/rejected": -7.28125, "step": 13180 }, { "epoch": 0.992400872771048, "grad_norm": 10.147038263763307, "learning_rate": 8.796138442868084e-11, "logits/chosen": -2.625, "logits/rejected": -2.59375, "logps/chosen": -520.0, "logps/rejected": -876.0, "loss": 0.1965, "rewards/accuracies": 0.90625, "rewards/chosen": -3.15625, "rewards/margins": 3.890625, "rewards/rejected": -7.03125, "step": 13190 }, { "epoch": 0.9931532616055978, "grad_norm": 10.227512417024357, "learning_rate": 7.140635865593437e-11, "logits/chosen": -2.65625, "logits/rejected": -2.6875, "logps/chosen": -520.0, "logps/rejected": -872.0, "loss": 0.1808, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.46875, "rewards/margins": 3.640625, "rewards/rejected": -7.125, "step": 13200 }, { "epoch": 0.9939056504401474, "grad_norm": 11.702241382399492, "learning_rate": 5.65755043764049e-11, "logits/chosen": -2.828125, "logits/rejected": -2.6875, "logps/chosen": -544.0, "logps/rejected": -904.0, "loss": 0.1848, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.421875, "rewards/margins": 3.828125, "rewards/rejected": -7.25, "step": 13210 }, { "epoch": 0.9946580392746972, "grad_norm": 13.380192668937356, "learning_rate": 4.346892390302836e-11, "logits/chosen": -2.640625, "logits/rejected": -2.828125, "logps/chosen": -552.0, "logps/rejected": -884.0, "loss": 0.1985, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.546875, "rewards/margins": 3.40625, "rewards/rejected": -6.96875, "step": 13220 }, { "epoch": 0.9954104281092468, "grad_norm": 9.745105666979178, "learning_rate": 3.208670765364463e-11, "logits/chosen": -2.59375, "logits/rejected": -2.640625, "logps/chosen": -520.0, "logps/rejected": -860.0, "loss": 0.1782, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -3.328125, "rewards/margins": 3.46875, "rewards/rejected": -6.8125, "step": 13230 }, { "epoch": 0.9961628169437966, "grad_norm": 11.283849545501477, "learning_rate": 2.2428934150192646e-11, "logits/chosen": -2.734375, "logits/rejected": -2.546875, "logps/chosen": -516.0, "logps/rejected": -884.0, "loss": 0.1755, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -3.3125, "rewards/margins": 3.6875, "rewards/rejected": -7.0, "step": 13240 }, { "epoch": 0.9969152057783462, "grad_norm": 13.425786071866433, "learning_rate": 1.4495670018405125e-11, "logits/chosen": -2.71875, "logits/rejected": -2.71875, "logps/chosen": -536.0, "logps/rejected": -888.0, "loss": 0.1925, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -3.65625, "rewards/margins": 3.625, "rewards/rejected": -7.28125, "step": 13250 }, { "epoch": 0.9976675946128959, "grad_norm": 11.628744872190834, "learning_rate": 8.286969987086888e-12, "logits/chosen": -2.53125, "logits/rejected": -2.6875, "logps/chosen": -532.0, "logps/rejected": -900.0, "loss": 0.1898, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.375, "rewards/margins": 3.703125, "rewards/rejected": -7.0625, "step": 13260 }, { "epoch": 0.9984199834474456, "grad_norm": 15.21065240784059, "learning_rate": 3.802876887948336e-12, "logits/chosen": -2.609375, "logits/rejected": -2.59375, "logps/chosen": -524.0, "logps/rejected": -848.0, "loss": 0.1841, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -3.421875, "rewards/margins": 3.265625, "rewards/rejected": -6.6875, "step": 13270 }, { "epoch": 0.9991723722819953, "grad_norm": 14.99393716460416, "learning_rate": 1.0434216552168695e-12, "logits/chosen": -2.5625, "logits/rejected": -2.390625, "logps/chosen": -524.0, "logps/rejected": -896.0, "loss": 0.1736, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -3.28125, "rewards/margins": 3.9375, "rewards/rejected": -7.1875, "step": 13280 }, { "epoch": 0.999924761116545, "grad_norm": 10.9952179604844, "learning_rate": 8.623325414847116e-15, "logits/chosen": -2.75, "logits/rejected": -2.671875, "logps/chosen": -528.0, "logps/rejected": -852.0, "loss": 0.1786, "rewards/accuracies": 0.90625, "rewards/chosen": -3.34375, "rewards/margins": 3.453125, "rewards/rejected": -6.78125, "step": 13290 }, { "epoch": 1.0, "step": 13291, "total_flos": 0.0, "train_loss": 0.2561736330384371, "train_runtime": 125157.49, "train_samples_per_second": 13.592, "train_steps_per_second": 0.106 } ], "logging_steps": 10, "max_steps": 13291, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 10000, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }