| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 2.0, |
| "eval_steps": 100, |
| "global_step": 3112, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.0, |
| "learning_rate": 5.353319057815846e-10, |
| "logits/chosen": -2.322030782699585, |
| "logits/rejected": -2.360077381134033, |
| "logps/chosen": -413.0701599121094, |
| "logps/rejected": -503.9693603515625, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.01, |
| "learning_rate": 5.353319057815845e-09, |
| "logits/chosen": -2.3369531631469727, |
| "logits/rejected": -2.352255344390869, |
| "logps/chosen": -334.3316650390625, |
| "logps/rejected": -329.3804016113281, |
| "loss": 0.6949, |
| "rewards/accuracies": 0.5416666865348816, |
| "rewards/chosen": 0.0067011509090662, |
| "rewards/margins": 0.011725478805601597, |
| "rewards/rejected": -0.005024327430874109, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.01, |
| "learning_rate": 1.070663811563169e-08, |
| "logits/chosen": -2.307915210723877, |
| "logits/rejected": -2.3184759616851807, |
| "logps/chosen": -383.9498291015625, |
| "logps/rejected": -349.3071594238281, |
| "loss": 0.6939, |
| "rewards/accuracies": 0.4000000059604645, |
| "rewards/chosen": -0.020408455282449722, |
| "rewards/margins": -0.008849766105413437, |
| "rewards/rejected": -0.011558687314391136, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.02, |
| "learning_rate": 1.6059957173447538e-08, |
| "logits/chosen": -2.3540706634521484, |
| "logits/rejected": -2.3323521614074707, |
| "logps/chosen": -382.1279602050781, |
| "logps/rejected": -429.32147216796875, |
| "loss": 0.6794, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.009235721081495285, |
| "rewards/margins": 0.022110218182206154, |
| "rewards/rejected": -0.03134594112634659, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.03, |
| "learning_rate": 2.141327623126338e-08, |
| "logits/chosen": -2.281588315963745, |
| "logits/rejected": -2.2917075157165527, |
| "logps/chosen": -329.18243408203125, |
| "logps/rejected": -268.74761962890625, |
| "loss": 0.6573, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": 0.008301397785544395, |
| "rewards/margins": 0.07960359007120132, |
| "rewards/rejected": -0.07130218297243118, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.03, |
| "learning_rate": 2.676659528907923e-08, |
| "logits/chosen": -2.2929608821868896, |
| "logits/rejected": -2.3293919563293457, |
| "logps/chosen": -348.1233215332031, |
| "logps/rejected": -352.0605163574219, |
| "loss": 0.6243, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 0.02957112155854702, |
| "rewards/margins": 0.16750425100326538, |
| "rewards/rejected": -0.1379331350326538, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.04, |
| "learning_rate": 3.2119914346895076e-08, |
| "logits/chosen": -2.2743587493896484, |
| "logits/rejected": -2.271085262298584, |
| "logps/chosen": -369.2187805175781, |
| "logps/rejected": -364.81781005859375, |
| "loss": 0.5746, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.016968611627817154, |
| "rewards/margins": 0.25147581100463867, |
| "rewards/rejected": -0.23450717329978943, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.04, |
| "learning_rate": 3.747323340471092e-08, |
| "logits/chosen": -2.3003923892974854, |
| "logits/rejected": -2.3059000968933105, |
| "logps/chosen": -358.39813232421875, |
| "logps/rejected": -377.95391845703125, |
| "loss": 0.5103, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.06770019978284836, |
| "rewards/margins": 0.510492205619812, |
| "rewards/rejected": -0.44279199838638306, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.05, |
| "learning_rate": 4.282655246252676e-08, |
| "logits/chosen": -2.2736544609069824, |
| "logits/rejected": -2.2347025871276855, |
| "logps/chosen": -350.6732177734375, |
| "logps/rejected": -310.04315185546875, |
| "loss": 0.4697, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 0.11150838434696198, |
| "rewards/margins": 0.5623964071273804, |
| "rewards/rejected": -0.45088809728622437, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.06, |
| "learning_rate": 4.817987152034261e-08, |
| "logits/chosen": -2.2885279655456543, |
| "logits/rejected": -2.2864298820495605, |
| "logps/chosen": -376.4400939941406, |
| "logps/rejected": -356.20245361328125, |
| "loss": 0.4095, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.13773031532764435, |
| "rewards/margins": 0.7758801579475403, |
| "rewards/rejected": -0.6381498575210571, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.06, |
| "learning_rate": 5.353319057815846e-08, |
| "logits/chosen": -2.2083301544189453, |
| "logits/rejected": -2.191849946975708, |
| "logps/chosen": -388.324462890625, |
| "logps/rejected": -376.165771484375, |
| "loss": 0.3876, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": 0.19459189474582672, |
| "rewards/margins": 0.9206008911132812, |
| "rewards/rejected": -0.7260090112686157, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.06, |
| "eval_logits/chosen": -2.2498624324798584, |
| "eval_logits/rejected": -2.221994400024414, |
| "eval_logps/chosen": -328.4210510253906, |
| "eval_logps/rejected": -324.17828369140625, |
| "eval_loss": 0.3689849376678467, |
| "eval_rewards/accuracies": 0.8984375, |
| "eval_rewards/chosen": 0.0942004844546318, |
| "eval_rewards/margins": 0.8769543170928955, |
| "eval_rewards/rejected": -0.7827538251876831, |
| "eval_runtime": 76.5553, |
| "eval_samples_per_second": 13.062, |
| "eval_steps_per_second": 0.418, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.07, |
| "learning_rate": 5.88865096359743e-08, |
| "logits/chosen": -2.2286324501037598, |
| "logits/rejected": -2.2091312408447266, |
| "logps/chosen": -372.5191650390625, |
| "logps/rejected": -399.0936584472656, |
| "loss": 0.3497, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.1948019564151764, |
| "rewards/margins": 1.1746506690979004, |
| "rewards/rejected": -0.9798487424850464, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.08, |
| "learning_rate": 6.423982869379015e-08, |
| "logits/chosen": -2.1091272830963135, |
| "logits/rejected": -2.118820905685425, |
| "logps/chosen": -361.56353759765625, |
| "logps/rejected": -404.4062805175781, |
| "loss": 0.3081, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.11251312494277954, |
| "rewards/margins": 1.413537621498108, |
| "rewards/rejected": -1.3010244369506836, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.08, |
| "learning_rate": 6.959314775160599e-08, |
| "logits/chosen": -2.1156742572784424, |
| "logits/rejected": -2.092258930206299, |
| "logps/chosen": -406.53851318359375, |
| "logps/rejected": -382.41717529296875, |
| "loss": 0.2846, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": 0.18506963551044464, |
| "rewards/margins": 1.6774908304214478, |
| "rewards/rejected": -1.492421269416809, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.09, |
| "learning_rate": 7.494646680942184e-08, |
| "logits/chosen": -2.092482089996338, |
| "logits/rejected": -2.0639469623565674, |
| "logps/chosen": -364.37774658203125, |
| "logps/rejected": -419.9878845214844, |
| "loss": 0.2908, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.16728416085243225, |
| "rewards/margins": 1.9277279376983643, |
| "rewards/rejected": -1.760443925857544, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.1, |
| "learning_rate": 8.029978586723767e-08, |
| "logits/chosen": -2.035637378692627, |
| "logits/rejected": -2.007483959197998, |
| "logps/chosen": -364.07745361328125, |
| "logps/rejected": -378.3775939941406, |
| "loss": 0.2519, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.009305549785494804, |
| "rewards/margins": 1.7874408960342407, |
| "rewards/rejected": -1.7967464923858643, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.1, |
| "learning_rate": 8.565310492505352e-08, |
| "logits/chosen": -1.9844402074813843, |
| "logits/rejected": -1.9815524816513062, |
| "logps/chosen": -384.30645751953125, |
| "logps/rejected": -454.4593811035156, |
| "loss": 0.2155, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.009780190885066986, |
| "rewards/margins": 2.3053088188171387, |
| "rewards/rejected": -2.315088987350464, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.11, |
| "learning_rate": 9.100642398286937e-08, |
| "logits/chosen": -1.8912779092788696, |
| "logits/rejected": -1.8273910284042358, |
| "logps/chosen": -378.23199462890625, |
| "logps/rejected": -344.0122985839844, |
| "loss": 0.2089, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.13415472209453583, |
| "rewards/margins": 2.2053399085998535, |
| "rewards/rejected": -2.3394949436187744, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.12, |
| "learning_rate": 9.635974304068522e-08, |
| "logits/chosen": -1.8499170541763306, |
| "logits/rejected": -1.8102867603302002, |
| "logps/chosen": -337.0228271484375, |
| "logps/rejected": -421.8775329589844, |
| "loss": 0.1974, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.24989835917949677, |
| "rewards/margins": 2.7448952198028564, |
| "rewards/rejected": -2.99479341506958, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.12, |
| "learning_rate": 1.0171306209850107e-07, |
| "logits/chosen": -1.8178815841674805, |
| "logits/rejected": -1.8075618743896484, |
| "logps/chosen": -420.0069885253906, |
| "logps/rejected": -451.9561462402344, |
| "loss": 0.1646, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.2596689462661743, |
| "rewards/margins": 2.7779135704040527, |
| "rewards/rejected": -3.0375826358795166, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.13, |
| "learning_rate": 1.0706638115631692e-07, |
| "logits/chosen": -1.8774926662445068, |
| "logits/rejected": -1.7711089849472046, |
| "logps/chosen": -368.1988830566406, |
| "logps/rejected": -364.1625671386719, |
| "loss": 0.1791, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.20116904377937317, |
| "rewards/margins": 3.0269274711608887, |
| "rewards/rejected": -3.2280964851379395, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.13, |
| "eval_logits/chosen": -1.9170525074005127, |
| "eval_logits/rejected": -1.8318464756011963, |
| "eval_logps/chosen": -330.80859375, |
| "eval_logps/rejected": -343.30322265625, |
| "eval_loss": 0.19462376832962036, |
| "eval_rewards/accuracies": 0.90625, |
| "eval_rewards/chosen": -0.14455503225326538, |
| "eval_rewards/margins": 2.5506958961486816, |
| "eval_rewards/rejected": -2.695250988006592, |
| "eval_runtime": 76.4342, |
| "eval_samples_per_second": 13.083, |
| "eval_steps_per_second": 0.419, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.13, |
| "learning_rate": 1.1241970021413276e-07, |
| "logits/chosen": -1.870164155960083, |
| "logits/rejected": -1.7529096603393555, |
| "logps/chosen": -394.24310302734375, |
| "logps/rejected": -359.798828125, |
| "loss": 0.1689, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.23141837120056152, |
| "rewards/margins": 2.955934524536133, |
| "rewards/rejected": -3.1873526573181152, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.14, |
| "learning_rate": 1.177730192719486e-07, |
| "logits/chosen": -1.7515084743499756, |
| "logits/rejected": -1.6350901126861572, |
| "logps/chosen": -399.573486328125, |
| "logps/rejected": -395.66864013671875, |
| "loss": 0.1476, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.43568697571754456, |
| "rewards/margins": 3.202346086502075, |
| "rewards/rejected": -3.638033390045166, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.15, |
| "learning_rate": 1.2312633832976445e-07, |
| "logits/chosen": -1.7598545551300049, |
| "logits/rejected": -1.6209675073623657, |
| "logps/chosen": -397.47747802734375, |
| "logps/rejected": -407.18780517578125, |
| "loss": 0.1534, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.2008744478225708, |
| "rewards/margins": 3.397059917449951, |
| "rewards/rejected": -3.5979347229003906, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.15, |
| "learning_rate": 1.284796573875803e-07, |
| "logits/chosen": -1.7311824560165405, |
| "logits/rejected": -1.6193158626556396, |
| "logps/chosen": -379.2996520996094, |
| "logps/rejected": -434.6549377441406, |
| "loss": 0.136, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.4003673493862152, |
| "rewards/margins": 3.5797741413116455, |
| "rewards/rejected": -3.9801411628723145, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.16, |
| "learning_rate": 1.3383297644539615e-07, |
| "logits/chosen": -1.7749998569488525, |
| "logits/rejected": -1.6336084604263306, |
| "logps/chosen": -384.36016845703125, |
| "logps/rejected": -412.198974609375, |
| "loss": 0.1424, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.44941583275794983, |
| "rewards/margins": 4.18085241317749, |
| "rewards/rejected": -4.630268096923828, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.17, |
| "learning_rate": 1.3918629550321198e-07, |
| "logits/chosen": -1.6577208042144775, |
| "logits/rejected": -1.4876978397369385, |
| "logps/chosen": -368.1956481933594, |
| "logps/rejected": -311.98223876953125, |
| "loss": 0.1379, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.5439226031303406, |
| "rewards/margins": 3.506497621536255, |
| "rewards/rejected": -4.050419807434082, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.17, |
| "learning_rate": 1.4453961456102785e-07, |
| "logits/chosen": -1.6528087854385376, |
| "logits/rejected": -1.5587131977081299, |
| "logps/chosen": -357.02655029296875, |
| "logps/rejected": -449.92718505859375, |
| "loss": 0.1273, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.7018794417381287, |
| "rewards/margins": 4.135760307312012, |
| "rewards/rejected": -4.837639808654785, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.18, |
| "learning_rate": 1.4989293361884367e-07, |
| "logits/chosen": -1.6728260517120361, |
| "logits/rejected": -1.4406911134719849, |
| "logps/chosen": -383.44635009765625, |
| "logps/rejected": -405.33428955078125, |
| "loss": 0.1286, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.7234546542167664, |
| "rewards/margins": 4.43070125579834, |
| "rewards/rejected": -5.154156684875488, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.19, |
| "learning_rate": 1.5524625267665952e-07, |
| "logits/chosen": -1.5811737775802612, |
| "logits/rejected": -1.4415075778961182, |
| "logps/chosen": -380.16302490234375, |
| "logps/rejected": -385.4619140625, |
| "loss": 0.1184, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.6038004159927368, |
| "rewards/margins": 4.522972106933594, |
| "rewards/rejected": -5.126772403717041, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.19, |
| "learning_rate": 1.6059957173447535e-07, |
| "logits/chosen": -1.5964694023132324, |
| "logits/rejected": -1.5073761940002441, |
| "logps/chosen": -348.82110595703125, |
| "logps/rejected": -396.21234130859375, |
| "loss": 0.1218, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.7199587821960449, |
| "rewards/margins": 4.322809219360352, |
| "rewards/rejected": -5.042768478393555, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.19, |
| "eval_logits/chosen": -1.7016242742538452, |
| "eval_logits/rejected": -1.5659476518630981, |
| "eval_logps/chosen": -336.16741943359375, |
| "eval_logps/rejected": -363.2919006347656, |
| "eval_loss": 0.1464938521385193, |
| "eval_rewards/accuracies": 0.921875, |
| "eval_rewards/chosen": -0.6804376840591431, |
| "eval_rewards/margins": 4.013677597045898, |
| "eval_rewards/rejected": -4.69411563873291, |
| "eval_runtime": 76.4822, |
| "eval_samples_per_second": 13.075, |
| "eval_steps_per_second": 0.418, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.2, |
| "learning_rate": 1.6595289079229122e-07, |
| "logits/chosen": -1.5078377723693848, |
| "logits/rejected": -1.4056973457336426, |
| "logps/chosen": -412.2068786621094, |
| "logps/rejected": -462.7405700683594, |
| "loss": 0.1094, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.718084454536438, |
| "rewards/margins": 5.0283284187316895, |
| "rewards/rejected": -5.746413230895996, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.21, |
| "learning_rate": 1.7130620985010704e-07, |
| "logits/chosen": -1.586196780204773, |
| "logits/rejected": -1.4498493671417236, |
| "logps/chosen": -351.50775146484375, |
| "logps/rejected": -403.0026550292969, |
| "loss": 0.1016, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.6627265214920044, |
| "rewards/margins": 4.463755130767822, |
| "rewards/rejected": -5.126482009887695, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.21, |
| "learning_rate": 1.766595289079229e-07, |
| "logits/chosen": -1.5722558498382568, |
| "logits/rejected": -1.340435266494751, |
| "logps/chosen": -337.4762878417969, |
| "logps/rejected": -419.3818359375, |
| "loss": 0.1221, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.869371235370636, |
| "rewards/margins": 5.570626258850098, |
| "rewards/rejected": -6.439997673034668, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.22, |
| "learning_rate": 1.8201284796573874e-07, |
| "logits/chosen": -1.563241958618164, |
| "logits/rejected": -1.43202805519104, |
| "logps/chosen": -332.5054016113281, |
| "logps/rejected": -401.9011535644531, |
| "loss": 0.1141, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.7410942912101746, |
| "rewards/margins": 5.449254512786865, |
| "rewards/rejected": -6.1903486251831055, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.22, |
| "learning_rate": 1.873661670235546e-07, |
| "logits/chosen": -1.509913682937622, |
| "logits/rejected": -1.352683186531067, |
| "logps/chosen": -428.9737243652344, |
| "logps/rejected": -455.664306640625, |
| "loss": 0.1092, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.7532263398170471, |
| "rewards/margins": 5.451329708099365, |
| "rewards/rejected": -6.204555988311768, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.23, |
| "learning_rate": 1.9271948608137044e-07, |
| "logits/chosen": -1.6219438314437866, |
| "logits/rejected": -1.3939545154571533, |
| "logps/chosen": -389.11297607421875, |
| "logps/rejected": -411.80035400390625, |
| "loss": 0.1218, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.0884325504302979, |
| "rewards/margins": 5.317915439605713, |
| "rewards/rejected": -6.406347751617432, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.24, |
| "learning_rate": 1.980728051391863e-07, |
| "logits/chosen": -1.465427041053772, |
| "logits/rejected": -1.3383185863494873, |
| "logps/chosen": -373.2828369140625, |
| "logps/rejected": -433.7022399902344, |
| "loss": 0.1098, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.5387589931488037, |
| "rewards/margins": 5.308978080749512, |
| "rewards/rejected": -6.8477373123168945, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.24, |
| "learning_rate": 2.0342612419700214e-07, |
| "logits/chosen": -1.4776766300201416, |
| "logits/rejected": -1.390604019165039, |
| "logps/chosen": -384.6776428222656, |
| "logps/rejected": -409.8622131347656, |
| "loss": 0.1068, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.1275439262390137, |
| "rewards/margins": 5.649918079376221, |
| "rewards/rejected": -6.777462005615234, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.25, |
| "learning_rate": 2.0877944325481796e-07, |
| "logits/chosen": -1.496004343032837, |
| "logits/rejected": -1.2643308639526367, |
| "logps/chosen": -319.6006774902344, |
| "logps/rejected": -435.0767517089844, |
| "loss": 0.0963, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.9759249687194824, |
| "rewards/margins": 5.894278526306152, |
| "rewards/rejected": -6.870204925537109, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.26, |
| "learning_rate": 2.1413276231263384e-07, |
| "logits/chosen": -1.373808741569519, |
| "logits/rejected": -1.2059423923492432, |
| "logps/chosen": -355.2186584472656, |
| "logps/rejected": -375.163330078125, |
| "loss": 0.1065, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.117621660232544, |
| "rewards/margins": 5.132685661315918, |
| "rewards/rejected": -6.250307559967041, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.26, |
| "eval_logits/chosen": -1.5739349126815796, |
| "eval_logits/rejected": -1.3864831924438477, |
| "eval_logps/chosen": -342.44061279296875, |
| "eval_logps/rejected": -380.4227600097656, |
| "eval_loss": 0.1241927221417427, |
| "eval_rewards/accuracies": 0.921875, |
| "eval_rewards/chosen": -1.3077539205551147, |
| "eval_rewards/margins": 5.099446773529053, |
| "eval_rewards/rejected": -6.407200336456299, |
| "eval_runtime": 76.309, |
| "eval_samples_per_second": 13.105, |
| "eval_steps_per_second": 0.419, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.26, |
| "learning_rate": 2.1948608137044966e-07, |
| "logits/chosen": -1.453360915184021, |
| "logits/rejected": -1.215315341949463, |
| "logps/chosen": -309.32196044921875, |
| "logps/rejected": -383.0122985839844, |
| "loss": 0.0932, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.0941340923309326, |
| "rewards/margins": 5.864490985870361, |
| "rewards/rejected": -6.958624839782715, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.27, |
| "learning_rate": 2.248394004282655e-07, |
| "logits/chosen": -1.6208438873291016, |
| "logits/rejected": -1.3579902648925781, |
| "logps/chosen": -420.99951171875, |
| "logps/rejected": -458.4291076660156, |
| "loss": 0.0972, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.9325690269470215, |
| "rewards/margins": 6.371849536895752, |
| "rewards/rejected": -7.304419040679932, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.28, |
| "learning_rate": 2.3019271948608136e-07, |
| "logits/chosen": -1.5590957403182983, |
| "logits/rejected": -1.3185454607009888, |
| "logps/chosen": -433.14007568359375, |
| "logps/rejected": -423.8736877441406, |
| "loss": 0.0955, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.888096034526825, |
| "rewards/margins": 6.284546852111816, |
| "rewards/rejected": -7.172643184661865, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.28, |
| "learning_rate": 2.355460385438972e-07, |
| "logits/chosen": -1.5379421710968018, |
| "logits/rejected": -1.287793755531311, |
| "logps/chosen": -384.4920959472656, |
| "logps/rejected": -459.51495361328125, |
| "loss": 0.0925, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.9522300958633423, |
| "rewards/margins": 6.38286828994751, |
| "rewards/rejected": -7.3350982666015625, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.29, |
| "learning_rate": 2.4089935760171303e-07, |
| "logits/chosen": -1.539567232131958, |
| "logits/rejected": -1.2881194353103638, |
| "logps/chosen": -360.4696960449219, |
| "logps/rejected": -421.70953369140625, |
| "loss": 0.0983, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.032198190689087, |
| "rewards/margins": 6.628798484802246, |
| "rewards/rejected": -7.660996437072754, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.3, |
| "learning_rate": 2.462526766595289e-07, |
| "logits/chosen": -1.5270180702209473, |
| "logits/rejected": -1.213521122932434, |
| "logps/chosen": -341.48370361328125, |
| "logps/rejected": -416.1814880371094, |
| "loss": 0.0862, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.2647520303726196, |
| "rewards/margins": 7.327714443206787, |
| "rewards/rejected": -8.592466354370117, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.3, |
| "learning_rate": 2.5160599571734473e-07, |
| "logits/chosen": -1.539104700088501, |
| "logits/rejected": -1.304837942123413, |
| "logps/chosen": -412.41192626953125, |
| "logps/rejected": -530.3709106445312, |
| "loss": 0.0685, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.0689477920532227, |
| "rewards/margins": 7.198890686035156, |
| "rewards/rejected": -8.267837524414062, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.31, |
| "learning_rate": 2.569593147751606e-07, |
| "logits/chosen": -1.4452335834503174, |
| "logits/rejected": -1.177215576171875, |
| "logps/chosen": -356.6268005371094, |
| "logps/rejected": -386.1933288574219, |
| "loss": 0.0906, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.5285716652870178, |
| "rewards/margins": 7.398108005523682, |
| "rewards/rejected": -7.9266791343688965, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.31, |
| "learning_rate": 2.6231263383297643e-07, |
| "logits/chosen": -1.299133062362671, |
| "logits/rejected": -1.2396031618118286, |
| "logps/chosen": -354.5841979980469, |
| "logps/rejected": -451.559326171875, |
| "loss": 0.0801, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.7138751745223999, |
| "rewards/margins": 6.743910789489746, |
| "rewards/rejected": -7.457786560058594, |
| "step": 490 |
| }, |
| { |
| "epoch": 0.32, |
| "learning_rate": 2.676659528907923e-07, |
| "logits/chosen": -1.2933090925216675, |
| "logits/rejected": -1.0351635217666626, |
| "logps/chosen": -403.50091552734375, |
| "logps/rejected": -444.50811767578125, |
| "loss": 0.0804, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -1.2215214967727661, |
| "rewards/margins": 7.684378623962402, |
| "rewards/rejected": -8.905900955200195, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.32, |
| "eval_logits/chosen": -1.5204213857650757, |
| "eval_logits/rejected": -1.313443899154663, |
| "eval_logps/chosen": -341.24200439453125, |
| "eval_logps/rejected": -387.53155517578125, |
| "eval_loss": 0.11293376982212067, |
| "eval_rewards/accuracies": 0.9296875, |
| "eval_rewards/chosen": -1.1878938674926758, |
| "eval_rewards/margins": 5.9301862716674805, |
| "eval_rewards/rejected": -7.118079662322998, |
| "eval_runtime": 76.7239, |
| "eval_samples_per_second": 13.034, |
| "eval_steps_per_second": 0.417, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.33, |
| "learning_rate": 2.7301927194860813e-07, |
| "logits/chosen": -1.4917323589324951, |
| "logits/rejected": -1.2333643436431885, |
| "logps/chosen": -391.20941162109375, |
| "logps/rejected": -462.3330993652344, |
| "loss": 0.0677, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.9355085492134094, |
| "rewards/margins": 8.421406745910645, |
| "rewards/rejected": -9.356914520263672, |
| "step": 510 |
| }, |
| { |
| "epoch": 0.33, |
| "learning_rate": 2.7837259100642395e-07, |
| "logits/chosen": -1.3520570993423462, |
| "logits/rejected": -1.1625728607177734, |
| "logps/chosen": -392.84210205078125, |
| "logps/rejected": -441.2123107910156, |
| "loss": 0.084, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.286471962928772, |
| "rewards/margins": 7.407790184020996, |
| "rewards/rejected": -8.69426155090332, |
| "step": 520 |
| }, |
| { |
| "epoch": 0.34, |
| "learning_rate": 2.8372591006423977e-07, |
| "logits/chosen": -1.2561280727386475, |
| "logits/rejected": -1.0563820600509644, |
| "logps/chosen": -355.0276794433594, |
| "logps/rejected": -434.96099853515625, |
| "loss": 0.0754, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.644857406616211, |
| "rewards/margins": 7.365194797515869, |
| "rewards/rejected": -9.010051727294922, |
| "step": 530 |
| }, |
| { |
| "epoch": 0.35, |
| "learning_rate": 2.890792291220557e-07, |
| "logits/chosen": -1.3786545991897583, |
| "logits/rejected": -1.1711790561676025, |
| "logps/chosen": -343.2646179199219, |
| "logps/rejected": -414.624267578125, |
| "loss": 0.0644, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.0238958597183228, |
| "rewards/margins": 7.4112067222595215, |
| "rewards/rejected": -8.435102462768555, |
| "step": 540 |
| }, |
| { |
| "epoch": 0.35, |
| "learning_rate": 2.944325481798715e-07, |
| "logits/chosen": -1.483944296836853, |
| "logits/rejected": -1.2009809017181396, |
| "logps/chosen": -397.4231872558594, |
| "logps/rejected": -453.69903564453125, |
| "loss": 0.0742, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.842347264289856, |
| "rewards/margins": 7.386562347412109, |
| "rewards/rejected": -8.228910446166992, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.36, |
| "learning_rate": 2.9978586723768735e-07, |
| "logits/chosen": -1.4178438186645508, |
| "logits/rejected": -1.243758201599121, |
| "logps/chosen": -412.52105712890625, |
| "logps/rejected": -464.28741455078125, |
| "loss": 0.0944, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.449329137802124, |
| "rewards/margins": 8.025833129882812, |
| "rewards/rejected": -9.475163459777832, |
| "step": 560 |
| }, |
| { |
| "epoch": 0.37, |
| "learning_rate": 3.051391862955032e-07, |
| "logits/chosen": -1.3710880279541016, |
| "logits/rejected": -1.218766450881958, |
| "logps/chosen": -344.7862548828125, |
| "logps/rejected": -453.5824279785156, |
| "loss": 0.0435, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.8545820713043213, |
| "rewards/margins": 7.798386573791504, |
| "rewards/rejected": -9.652968406677246, |
| "step": 570 |
| }, |
| { |
| "epoch": 0.37, |
| "learning_rate": 3.1049250535331905e-07, |
| "logits/chosen": -1.461828589439392, |
| "logits/rejected": -1.0271979570388794, |
| "logps/chosen": -387.4298095703125, |
| "logps/rejected": -435.98663330078125, |
| "loss": 0.0867, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.7421836853027344, |
| "rewards/margins": 7.6707444190979, |
| "rewards/rejected": -9.412927627563477, |
| "step": 580 |
| }, |
| { |
| "epoch": 0.38, |
| "learning_rate": 3.1584582441113487e-07, |
| "logits/chosen": -1.3038650751113892, |
| "logits/rejected": -1.07460618019104, |
| "logps/chosen": -324.758544921875, |
| "logps/rejected": -503.78387451171875, |
| "loss": 0.0716, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -2.3957910537719727, |
| "rewards/margins": 8.855663299560547, |
| "rewards/rejected": -11.25145435333252, |
| "step": 590 |
| }, |
| { |
| "epoch": 0.39, |
| "learning_rate": 3.211991434689507e-07, |
| "logits/chosen": -1.3503267765045166, |
| "logits/rejected": -1.0638809204101562, |
| "logps/chosen": -374.38348388671875, |
| "logps/rejected": -516.7820434570312, |
| "loss": 0.0767, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.1524558067321777, |
| "rewards/margins": 8.407071113586426, |
| "rewards/rejected": -10.559527397155762, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.39, |
| "eval_logits/chosen": -1.4297420978546143, |
| "eval_logits/rejected": -1.150868535041809, |
| "eval_logps/chosen": -353.238037109375, |
| "eval_logps/rejected": -407.5989990234375, |
| "eval_loss": 0.13097576797008514, |
| "eval_rewards/accuracies": 0.8984375, |
| "eval_rewards/chosen": -2.387495994567871, |
| "eval_rewards/margins": 6.737332820892334, |
| "eval_rewards/rejected": -9.124829292297363, |
| "eval_runtime": 76.6752, |
| "eval_samples_per_second": 13.042, |
| "eval_steps_per_second": 0.417, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.39, |
| "learning_rate": 3.265524625267666e-07, |
| "logits/chosen": -1.1903326511383057, |
| "logits/rejected": -0.9525307416915894, |
| "logps/chosen": -419.54608154296875, |
| "logps/rejected": -444.11383056640625, |
| "loss": 0.0831, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.369199752807617, |
| "rewards/margins": 8.518608093261719, |
| "rewards/rejected": -10.887807846069336, |
| "step": 610 |
| }, |
| { |
| "epoch": 0.4, |
| "learning_rate": 3.3190578158458244e-07, |
| "logits/chosen": -1.3438398838043213, |
| "logits/rejected": -1.143065333366394, |
| "logps/chosen": -369.13970947265625, |
| "logps/rejected": -439.51715087890625, |
| "loss": 0.0779, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.6147950887680054, |
| "rewards/margins": 7.42797327041626, |
| "rewards/rejected": -9.042768478393555, |
| "step": 620 |
| }, |
| { |
| "epoch": 0.4, |
| "learning_rate": 3.3725910064239827e-07, |
| "logits/chosen": -1.4843438863754272, |
| "logits/rejected": -1.0890620946884155, |
| "logps/chosen": -409.73583984375, |
| "logps/rejected": -517.4986572265625, |
| "loss": 0.0703, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -1.1902521848678589, |
| "rewards/margins": 9.822367668151855, |
| "rewards/rejected": -11.01262092590332, |
| "step": 630 |
| }, |
| { |
| "epoch": 0.41, |
| "learning_rate": 3.426124197002141e-07, |
| "logits/chosen": -1.3501672744750977, |
| "logits/rejected": -1.0996173620224, |
| "logps/chosen": -360.51531982421875, |
| "logps/rejected": -489.69061279296875, |
| "loss": 0.0968, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.6506973505020142, |
| "rewards/margins": 8.347593307495117, |
| "rewards/rejected": -9.998289108276367, |
| "step": 640 |
| }, |
| { |
| "epoch": 0.42, |
| "learning_rate": 3.4796573875802996e-07, |
| "logits/chosen": -1.2105796337127686, |
| "logits/rejected": -0.9770170450210571, |
| "logps/chosen": -335.6686096191406, |
| "logps/rejected": -449.52447509765625, |
| "loss": 0.0659, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.2099552154541016, |
| "rewards/margins": 7.764392852783203, |
| "rewards/rejected": -9.974349021911621, |
| "step": 650 |
| }, |
| { |
| "epoch": 0.42, |
| "learning_rate": 3.533190578158458e-07, |
| "logits/chosen": -1.4429116249084473, |
| "logits/rejected": -1.06112539768219, |
| "logps/chosen": -420.8074645996094, |
| "logps/rejected": -442.88568115234375, |
| "loss": 0.0967, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.681353211402893, |
| "rewards/margins": 7.297033786773682, |
| "rewards/rejected": -8.978387832641602, |
| "step": 660 |
| }, |
| { |
| "epoch": 0.43, |
| "learning_rate": 3.5867237687366166e-07, |
| "logits/chosen": -1.4578073024749756, |
| "logits/rejected": -1.1529022455215454, |
| "logps/chosen": -362.67230224609375, |
| "logps/rejected": -455.08367919921875, |
| "loss": 0.0694, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.878594994544983, |
| "rewards/margins": 8.333452224731445, |
| "rewards/rejected": -10.212045669555664, |
| "step": 670 |
| }, |
| { |
| "epoch": 0.44, |
| "learning_rate": 3.640256959314775e-07, |
| "logits/chosen": -1.34915292263031, |
| "logits/rejected": -1.125140905380249, |
| "logps/chosen": -364.0298156738281, |
| "logps/rejected": -492.75909423828125, |
| "loss": 0.0744, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.043875217437744, |
| "rewards/margins": 9.902002334594727, |
| "rewards/rejected": -11.945877075195312, |
| "step": 680 |
| }, |
| { |
| "epoch": 0.44, |
| "learning_rate": 3.6937901498929336e-07, |
| "logits/chosen": -1.2846548557281494, |
| "logits/rejected": -0.9907848238945007, |
| "logps/chosen": -423.0797424316406, |
| "logps/rejected": -499.71142578125, |
| "loss": 0.0603, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.9811477661132812, |
| "rewards/margins": 9.256712913513184, |
| "rewards/rejected": -12.237860679626465, |
| "step": 690 |
| }, |
| { |
| "epoch": 0.45, |
| "learning_rate": 3.747323340471092e-07, |
| "logits/chosen": -1.3538507223129272, |
| "logits/rejected": -1.1379244327545166, |
| "logps/chosen": -329.5379638671875, |
| "logps/rejected": -413.8023986816406, |
| "loss": 0.0759, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.2614781856536865, |
| "rewards/margins": 8.729515075683594, |
| "rewards/rejected": -10.990991592407227, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.45, |
| "eval_logits/chosen": -1.4886287450790405, |
| "eval_logits/rejected": -1.2180323600769043, |
| "eval_logps/chosen": -348.9230041503906, |
| "eval_logps/rejected": -409.6312255859375, |
| "eval_loss": 0.12044651806354523, |
| "eval_rewards/accuracies": 0.9296875, |
| "eval_rewards/chosen": -1.9559952020645142, |
| "eval_rewards/margins": 7.372057914733887, |
| "eval_rewards/rejected": -9.328052520751953, |
| "eval_runtime": 76.578, |
| "eval_samples_per_second": 13.059, |
| "eval_steps_per_second": 0.418, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.46, |
| "learning_rate": 3.80085653104925e-07, |
| "logits/chosen": -1.3313727378845215, |
| "logits/rejected": -0.9123377799987793, |
| "logps/chosen": -442.8067321777344, |
| "logps/rejected": -468.52984619140625, |
| "loss": 0.0905, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.6124906539916992, |
| "rewards/margins": 8.923405647277832, |
| "rewards/rejected": -10.535896301269531, |
| "step": 710 |
| }, |
| { |
| "epoch": 0.46, |
| "learning_rate": 3.854389721627409e-07, |
| "logits/chosen": -1.4308016300201416, |
| "logits/rejected": -1.0905249118804932, |
| "logps/chosen": -378.02545166015625, |
| "logps/rejected": -479.7505798339844, |
| "loss": 0.0561, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.7049999237060547, |
| "rewards/margins": 8.536346435546875, |
| "rewards/rejected": -10.241347312927246, |
| "step": 720 |
| }, |
| { |
| "epoch": 0.47, |
| "learning_rate": 3.9079229122055676e-07, |
| "logits/chosen": -1.3243951797485352, |
| "logits/rejected": -0.9832109212875366, |
| "logps/chosen": -431.05645751953125, |
| "logps/rejected": -432.52886962890625, |
| "loss": 0.0581, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.067716121673584, |
| "rewards/margins": 8.114839553833008, |
| "rewards/rejected": -10.18255615234375, |
| "step": 730 |
| }, |
| { |
| "epoch": 0.48, |
| "learning_rate": 3.961456102783726e-07, |
| "logits/chosen": -1.258172869682312, |
| "logits/rejected": -1.0175604820251465, |
| "logps/chosen": -404.3645935058594, |
| "logps/rejected": -519.955322265625, |
| "loss": 0.0452, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -1.6845811605453491, |
| "rewards/margins": 10.622332572937012, |
| "rewards/rejected": -12.306914329528809, |
| "step": 740 |
| }, |
| { |
| "epoch": 0.48, |
| "learning_rate": 4.014989293361884e-07, |
| "logits/chosen": -1.28511381149292, |
| "logits/rejected": -1.0871832370758057, |
| "logps/chosen": -378.40478515625, |
| "logps/rejected": -460.6148376464844, |
| "loss": 0.0669, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.8259923458099365, |
| "rewards/margins": 8.321816444396973, |
| "rewards/rejected": -10.147809982299805, |
| "step": 750 |
| }, |
| { |
| "epoch": 0.49, |
| "learning_rate": 4.068522483940043e-07, |
| "logits/chosen": -1.5064611434936523, |
| "logits/rejected": -1.0779759883880615, |
| "logps/chosen": -371.971435546875, |
| "logps/rejected": -416.6922302246094, |
| "loss": 0.0762, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.135917067527771, |
| "rewards/margins": 10.22150993347168, |
| "rewards/rejected": -11.357427597045898, |
| "step": 760 |
| }, |
| { |
| "epoch": 0.49, |
| "learning_rate": 4.122055674518201e-07, |
| "logits/chosen": -1.429022192955017, |
| "logits/rejected": -1.0790882110595703, |
| "logps/chosen": -394.6370544433594, |
| "logps/rejected": -461.9344787597656, |
| "loss": 0.0662, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.259974956512451, |
| "rewards/margins": 9.807271003723145, |
| "rewards/rejected": -12.067245483398438, |
| "step": 770 |
| }, |
| { |
| "epoch": 0.5, |
| "learning_rate": 4.175588865096359e-07, |
| "logits/chosen": -1.296064853668213, |
| "logits/rejected": -0.9478403925895691, |
| "logps/chosen": -376.76727294921875, |
| "logps/rejected": -470.43743896484375, |
| "loss": 0.0513, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.549682378768921, |
| "rewards/margins": 9.523083686828613, |
| "rewards/rejected": -11.072766304016113, |
| "step": 780 |
| }, |
| { |
| "epoch": 0.51, |
| "learning_rate": 4.2291220556745175e-07, |
| "logits/chosen": -1.309073567390442, |
| "logits/rejected": -0.8560987710952759, |
| "logps/chosen": -457.4974670410156, |
| "logps/rejected": -528.8920288085938, |
| "loss": 0.0616, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.5288264751434326, |
| "rewards/margins": 9.966497421264648, |
| "rewards/rejected": -12.495325088500977, |
| "step": 790 |
| }, |
| { |
| "epoch": 0.51, |
| "learning_rate": 4.282655246252677e-07, |
| "logits/chosen": -1.2481553554534912, |
| "logits/rejected": -0.764441728591919, |
| "logps/chosen": -306.9283142089844, |
| "logps/rejected": -417.22491455078125, |
| "loss": 0.0681, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -1.760056734085083, |
| "rewards/margins": 10.229304313659668, |
| "rewards/rejected": -11.989361763000488, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.51, |
| "eval_logits/chosen": -1.4168956279754639, |
| "eval_logits/rejected": -1.0999643802642822, |
| "eval_logps/chosen": -350.21661376953125, |
| "eval_logps/rejected": -414.49041748046875, |
| "eval_loss": 0.1091269925236702, |
| "eval_rewards/accuracies": 0.9609375, |
| "eval_rewards/chosen": -2.085355281829834, |
| "eval_rewards/margins": 7.728612422943115, |
| "eval_rewards/rejected": -9.81396770477295, |
| "eval_runtime": 76.655, |
| "eval_samples_per_second": 13.045, |
| "eval_steps_per_second": 0.417, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.52, |
| "learning_rate": 4.336188436830835e-07, |
| "logits/chosen": -1.092874526977539, |
| "logits/rejected": -0.8753455281257629, |
| "logps/chosen": -367.61846923828125, |
| "logps/rejected": -431.23834228515625, |
| "loss": 0.0496, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.793146848678589, |
| "rewards/margins": 8.786299705505371, |
| "rewards/rejected": -11.579446792602539, |
| "step": 810 |
| }, |
| { |
| "epoch": 0.53, |
| "learning_rate": 4.389721627408993e-07, |
| "logits/chosen": -1.02623450756073, |
| "logits/rejected": -0.5294391512870789, |
| "logps/chosen": -348.81463623046875, |
| "logps/rejected": -391.2160339355469, |
| "loss": 0.0634, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.466538906097412, |
| "rewards/margins": 9.582557678222656, |
| "rewards/rejected": -12.049097061157227, |
| "step": 820 |
| }, |
| { |
| "epoch": 0.53, |
| "learning_rate": 4.443254817987152e-07, |
| "logits/chosen": -1.1547324657440186, |
| "logits/rejected": -0.6096884608268738, |
| "logps/chosen": -426.49798583984375, |
| "logps/rejected": -467.9422302246094, |
| "loss": 0.072, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -2.5411438941955566, |
| "rewards/margins": 9.401772499084473, |
| "rewards/rejected": -11.942916870117188, |
| "step": 830 |
| }, |
| { |
| "epoch": 0.54, |
| "learning_rate": 4.49678800856531e-07, |
| "logits/chosen": -1.247434377670288, |
| "logits/rejected": -0.7016826868057251, |
| "logps/chosen": -393.85430908203125, |
| "logps/rejected": -482.9002990722656, |
| "loss": 0.0467, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.051135301589966, |
| "rewards/margins": 10.815892219543457, |
| "rewards/rejected": -12.867027282714844, |
| "step": 840 |
| }, |
| { |
| "epoch": 0.55, |
| "learning_rate": 4.5503211991434684e-07, |
| "logits/chosen": -1.2653485536575317, |
| "logits/rejected": -0.7517842054367065, |
| "logps/chosen": -393.53643798828125, |
| "logps/rejected": -458.5020446777344, |
| "loss": 0.0962, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.8538535833358765, |
| "rewards/margins": 9.80111312866211, |
| "rewards/rejected": -11.654967308044434, |
| "step": 850 |
| }, |
| { |
| "epoch": 0.55, |
| "learning_rate": 4.603854389721627e-07, |
| "logits/chosen": -1.0140388011932373, |
| "logits/rejected": -0.5941162109375, |
| "logps/chosen": -431.3036193847656, |
| "logps/rejected": -522.5137939453125, |
| "loss": 0.073, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.430864095687866, |
| "rewards/margins": 10.6370267868042, |
| "rewards/rejected": -13.067891120910645, |
| "step": 860 |
| }, |
| { |
| "epoch": 0.56, |
| "learning_rate": 4.657387580299786e-07, |
| "logits/chosen": -0.8724175691604614, |
| "logits/rejected": -0.424823522567749, |
| "logps/chosen": -405.14544677734375, |
| "logps/rejected": -458.218994140625, |
| "loss": 0.0896, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -3.2016243934631348, |
| "rewards/margins": 10.069561958312988, |
| "rewards/rejected": -13.271186828613281, |
| "step": 870 |
| }, |
| { |
| "epoch": 0.57, |
| "learning_rate": 4.710920770877944e-07, |
| "logits/chosen": -0.9462097883224487, |
| "logits/rejected": -0.5895959138870239, |
| "logps/chosen": -384.04046630859375, |
| "logps/rejected": -430.6656188964844, |
| "loss": 0.0703, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -2.604841947555542, |
| "rewards/margins": 8.405369758605957, |
| "rewards/rejected": -11.010213851928711, |
| "step": 880 |
| }, |
| { |
| "epoch": 0.57, |
| "learning_rate": 4.7644539614561024e-07, |
| "logits/chosen": -1.0459139347076416, |
| "logits/rejected": -0.5180732011795044, |
| "logps/chosen": -349.8458557128906, |
| "logps/rejected": -463.24993896484375, |
| "loss": 0.068, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.0111186504364014, |
| "rewards/margins": 10.823290824890137, |
| "rewards/rejected": -12.8344087600708, |
| "step": 890 |
| }, |
| { |
| "epoch": 0.58, |
| "learning_rate": 4.817987152034261e-07, |
| "logits/chosen": -1.0149575471878052, |
| "logits/rejected": -0.3724205493927002, |
| "logps/chosen": -396.1781311035156, |
| "logps/rejected": -512.572265625, |
| "loss": 0.0494, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.9190399646759033, |
| "rewards/margins": 12.252616882324219, |
| "rewards/rejected": -15.171656608581543, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.58, |
| "eval_logits/chosen": -1.1149048805236816, |
| "eval_logits/rejected": -0.723557710647583, |
| "eval_logps/chosen": -371.6729736328125, |
| "eval_logps/rejected": -447.3422546386719, |
| "eval_loss": 0.14773131906986237, |
| "eval_rewards/accuracies": 0.890625, |
| "eval_rewards/chosen": -4.230995178222656, |
| "eval_rewards/margins": 8.868158340454102, |
| "eval_rewards/rejected": -13.099154472351074, |
| "eval_runtime": 76.5391, |
| "eval_samples_per_second": 13.065, |
| "eval_steps_per_second": 0.418, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.58, |
| "learning_rate": 4.871520342612419e-07, |
| "logits/chosen": -1.0406441688537598, |
| "logits/rejected": -0.7672456502914429, |
| "logps/chosen": -367.68988037109375, |
| "logps/rejected": -497.24542236328125, |
| "loss": 0.0497, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.375262498855591, |
| "rewards/margins": 11.108481407165527, |
| "rewards/rejected": -13.483744621276855, |
| "step": 910 |
| }, |
| { |
| "epoch": 0.59, |
| "learning_rate": 4.925053533190578e-07, |
| "logits/chosen": -1.3684611320495605, |
| "logits/rejected": -0.7619699239730835, |
| "logps/chosen": -395.2074279785156, |
| "logps/rejected": -435.02490234375, |
| "loss": 0.0771, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.186323642730713, |
| "rewards/margins": 9.919301986694336, |
| "rewards/rejected": -12.10562515258789, |
| "step": 920 |
| }, |
| { |
| "epoch": 0.6, |
| "learning_rate": 4.978586723768736e-07, |
| "logits/chosen": -1.1214783191680908, |
| "logits/rejected": -0.5191805362701416, |
| "logps/chosen": -415.1065979003906, |
| "logps/rejected": -468.7010192871094, |
| "loss": 0.046, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.4811503887176514, |
| "rewards/margins": 9.906661987304688, |
| "rewards/rejected": -13.387812614440918, |
| "step": 930 |
| }, |
| { |
| "epoch": 0.6, |
| "learning_rate": 4.996429421566293e-07, |
| "logits/chosen": -1.2635540962219238, |
| "logits/rejected": -0.628160834312439, |
| "logps/chosen": -416.54644775390625, |
| "logps/rejected": -465.719970703125, |
| "loss": 0.1156, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -3.1887431144714355, |
| "rewards/margins": 9.464393615722656, |
| "rewards/rejected": -12.65313720703125, |
| "step": 940 |
| }, |
| { |
| "epoch": 0.61, |
| "learning_rate": 4.990478457510116e-07, |
| "logits/chosen": -1.3375509977340698, |
| "logits/rejected": -0.5667593479156494, |
| "logps/chosen": -430.92742919921875, |
| "logps/rejected": -532.2154541015625, |
| "loss": 0.0961, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -2.9553847312927246, |
| "rewards/margins": 12.077500343322754, |
| "rewards/rejected": -15.03288745880127, |
| "step": 950 |
| }, |
| { |
| "epoch": 0.62, |
| "learning_rate": 4.98452749345394e-07, |
| "logits/chosen": -1.117200493812561, |
| "logits/rejected": -0.6453494429588318, |
| "logps/chosen": -424.7369689941406, |
| "logps/rejected": -452.61749267578125, |
| "loss": 0.0766, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.5312678813934326, |
| "rewards/margins": 10.04582405090332, |
| "rewards/rejected": -13.577092170715332, |
| "step": 960 |
| }, |
| { |
| "epoch": 0.62, |
| "learning_rate": 4.978576529397762e-07, |
| "logits/chosen": -1.1340444087982178, |
| "logits/rejected": -0.656354546546936, |
| "logps/chosen": -389.5958557128906, |
| "logps/rejected": -524.0951538085938, |
| "loss": 0.0653, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.064024448394775, |
| "rewards/margins": 11.442460060119629, |
| "rewards/rejected": -16.506484985351562, |
| "step": 970 |
| }, |
| { |
| "epoch": 0.63, |
| "learning_rate": 4.972625565341585e-07, |
| "logits/chosen": -1.2187734842300415, |
| "logits/rejected": -0.6018660664558411, |
| "logps/chosen": -386.0228271484375, |
| "logps/rejected": -454.13140869140625, |
| "loss": 0.0838, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.2799274921417236, |
| "rewards/margins": 9.308359146118164, |
| "rewards/rejected": -12.588286399841309, |
| "step": 980 |
| }, |
| { |
| "epoch": 0.64, |
| "learning_rate": 4.966674601285408e-07, |
| "logits/chosen": -1.111574411392212, |
| "logits/rejected": -0.7768339514732361, |
| "logps/chosen": -366.3212585449219, |
| "logps/rejected": -501.3627014160156, |
| "loss": 0.0817, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -2.8082051277160645, |
| "rewards/margins": 10.757050514221191, |
| "rewards/rejected": -13.565256118774414, |
| "step": 990 |
| }, |
| { |
| "epoch": 0.64, |
| "learning_rate": 4.960723637229232e-07, |
| "logits/chosen": -0.855319619178772, |
| "logits/rejected": -0.4498973786830902, |
| "logps/chosen": -356.251708984375, |
| "logps/rejected": -479.78094482421875, |
| "loss": 0.0735, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.355088710784912, |
| "rewards/margins": 9.392342567443848, |
| "rewards/rejected": -13.747430801391602, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.64, |
| "eval_logits/chosen": -1.069393277168274, |
| "eval_logits/rejected": -0.698665976524353, |
| "eval_logps/chosen": -367.884033203125, |
| "eval_logps/rejected": -449.0148620605469, |
| "eval_loss": 0.12433216720819473, |
| "eval_rewards/accuracies": 0.9609375, |
| "eval_rewards/chosen": -3.8520963191986084, |
| "eval_rewards/margins": 9.414312362670898, |
| "eval_rewards/rejected": -13.26640796661377, |
| "eval_runtime": 76.7244, |
| "eval_samples_per_second": 13.034, |
| "eval_steps_per_second": 0.417, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.65, |
| "learning_rate": 4.954772673173054e-07, |
| "logits/chosen": -0.8880437612533569, |
| "logits/rejected": -0.5571062564849854, |
| "logps/chosen": -427.5099182128906, |
| "logps/rejected": -511.27398681640625, |
| "loss": 0.1211, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -3.051557779312134, |
| "rewards/margins": 10.788399696350098, |
| "rewards/rejected": -13.839956283569336, |
| "step": 1010 |
| }, |
| { |
| "epoch": 0.66, |
| "learning_rate": 4.948821709116876e-07, |
| "logits/chosen": -1.3221044540405273, |
| "logits/rejected": -0.6521639823913574, |
| "logps/chosen": -380.6373596191406, |
| "logps/rejected": -449.31817626953125, |
| "loss": 0.0824, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.038402795791626, |
| "rewards/margins": 11.031317710876465, |
| "rewards/rejected": -13.069720268249512, |
| "step": 1020 |
| }, |
| { |
| "epoch": 0.66, |
| "learning_rate": 4.9428707450607e-07, |
| "logits/chosen": -1.1821175813674927, |
| "logits/rejected": -0.7060034871101379, |
| "logps/chosen": -366.3950500488281, |
| "logps/rejected": -507.5702209472656, |
| "loss": 0.078, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.4800446033477783, |
| "rewards/margins": 10.978177070617676, |
| "rewards/rejected": -14.458221435546875, |
| "step": 1030 |
| }, |
| { |
| "epoch": 0.67, |
| "learning_rate": 4.936919781004522e-07, |
| "logits/chosen": -1.2339891195297241, |
| "logits/rejected": -0.8903535604476929, |
| "logps/chosen": -411.30615234375, |
| "logps/rejected": -553.6360473632812, |
| "loss": 0.0829, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.9073569774627686, |
| "rewards/margins": 10.883973121643066, |
| "rewards/rejected": -14.79133129119873, |
| "step": 1040 |
| }, |
| { |
| "epoch": 0.67, |
| "learning_rate": 4.930968816948346e-07, |
| "logits/chosen": -1.3873369693756104, |
| "logits/rejected": -0.8341131210327148, |
| "logps/chosen": -424.88543701171875, |
| "logps/rejected": -477.947021484375, |
| "loss": 0.0747, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.1361687183380127, |
| "rewards/margins": 11.415266036987305, |
| "rewards/rejected": -13.551434516906738, |
| "step": 1050 |
| }, |
| { |
| "epoch": 0.68, |
| "learning_rate": 4.925017852892168e-07, |
| "logits/chosen": -1.1863329410552979, |
| "logits/rejected": -0.6122242212295532, |
| "logps/chosen": -400.68865966796875, |
| "logps/rejected": -515.4220581054688, |
| "loss": 0.0844, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.35847544670105, |
| "rewards/margins": 13.144007682800293, |
| "rewards/rejected": -15.502484321594238, |
| "step": 1060 |
| }, |
| { |
| "epoch": 0.69, |
| "learning_rate": 4.919066888835991e-07, |
| "logits/chosen": -1.1419920921325684, |
| "logits/rejected": -0.5117210149765015, |
| "logps/chosen": -370.2403564453125, |
| "logps/rejected": -593.1256103515625, |
| "loss": 0.0685, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.8190958499908447, |
| "rewards/margins": 14.543069839477539, |
| "rewards/rejected": -17.362167358398438, |
| "step": 1070 |
| }, |
| { |
| "epoch": 0.69, |
| "learning_rate": 4.913115924779814e-07, |
| "logits/chosen": -1.1333223581314087, |
| "logits/rejected": -0.5887588858604431, |
| "logps/chosen": -438.794677734375, |
| "logps/rejected": -556.837890625, |
| "loss": 0.0702, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.482754945755005, |
| "rewards/margins": 12.69272232055664, |
| "rewards/rejected": -16.175479888916016, |
| "step": 1080 |
| }, |
| { |
| "epoch": 0.7, |
| "learning_rate": 4.907164960723638e-07, |
| "logits/chosen": -1.0477075576782227, |
| "logits/rejected": -0.5950930714607239, |
| "logps/chosen": -385.4281005859375, |
| "logps/rejected": -448.23211669921875, |
| "loss": 0.0694, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -4.315608501434326, |
| "rewards/margins": 10.84793472290039, |
| "rewards/rejected": -15.163541793823242, |
| "step": 1090 |
| }, |
| { |
| "epoch": 0.71, |
| "learning_rate": 4.90121399666746e-07, |
| "logits/chosen": -0.8860788345336914, |
| "logits/rejected": -0.4812610149383545, |
| "logps/chosen": -356.5136413574219, |
| "logps/rejected": -509.6454162597656, |
| "loss": 0.0806, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.0979533195495605, |
| "rewards/margins": 11.64047908782959, |
| "rewards/rejected": -15.738431930541992, |
| "step": 1100 |
| }, |
| { |
| "epoch": 0.71, |
| "eval_logits/chosen": -1.1228324174880981, |
| "eval_logits/rejected": -0.7543247938156128, |
| "eval_logps/chosen": -354.41119384765625, |
| "eval_logps/rejected": -438.4850158691406, |
| "eval_loss": 0.11537463963031769, |
| "eval_rewards/accuracies": 0.9765625, |
| "eval_rewards/chosen": -2.5048139095306396, |
| "eval_rewards/margins": 9.708612442016602, |
| "eval_rewards/rejected": -12.213427543640137, |
| "eval_runtime": 76.7376, |
| "eval_samples_per_second": 13.031, |
| "eval_steps_per_second": 0.417, |
| "step": 1100 |
| }, |
| { |
| "epoch": 0.71, |
| "learning_rate": 4.895263032611282e-07, |
| "logits/chosen": -1.2738934755325317, |
| "logits/rejected": -0.6076444387435913, |
| "logps/chosen": -452.056396484375, |
| "logps/rejected": -505.551513671875, |
| "loss": 0.0556, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -3.597477674484253, |
| "rewards/margins": 11.379976272583008, |
| "rewards/rejected": -14.977453231811523, |
| "step": 1110 |
| }, |
| { |
| "epoch": 0.72, |
| "learning_rate": 4.889312068555106e-07, |
| "logits/chosen": -1.174073576927185, |
| "logits/rejected": -0.7190831303596497, |
| "logps/chosen": -400.78656005859375, |
| "logps/rejected": -528.5311889648438, |
| "loss": 0.0635, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.351175308227539, |
| "rewards/margins": 11.898660659790039, |
| "rewards/rejected": -15.249837875366211, |
| "step": 1120 |
| }, |
| { |
| "epoch": 0.73, |
| "learning_rate": 4.883361104498928e-07, |
| "logits/chosen": -1.2661216259002686, |
| "logits/rejected": -0.7576111555099487, |
| "logps/chosen": -395.97833251953125, |
| "logps/rejected": -508.68719482421875, |
| "loss": 0.0564, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.6894619464874268, |
| "rewards/margins": 13.860685348510742, |
| "rewards/rejected": -17.550146102905273, |
| "step": 1130 |
| }, |
| { |
| "epoch": 0.73, |
| "learning_rate": 4.877410140442752e-07, |
| "logits/chosen": -1.3575642108917236, |
| "logits/rejected": -0.8006072044372559, |
| "logps/chosen": -423.7298889160156, |
| "logps/rejected": -535.811767578125, |
| "loss": 0.0727, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.8236665725708008, |
| "rewards/margins": 12.405186653137207, |
| "rewards/rejected": -14.228856086730957, |
| "step": 1140 |
| }, |
| { |
| "epoch": 0.74, |
| "learning_rate": 4.871459176386574e-07, |
| "logits/chosen": -1.1852174997329712, |
| "logits/rejected": -0.7334953546524048, |
| "logps/chosen": -447.42724609375, |
| "logps/rejected": -565.5848388671875, |
| "loss": 0.0614, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.648531436920166, |
| "rewards/margins": 13.974637031555176, |
| "rewards/rejected": -17.6231689453125, |
| "step": 1150 |
| }, |
| { |
| "epoch": 0.75, |
| "learning_rate": 4.865508212330398e-07, |
| "logits/chosen": -1.1278337240219116, |
| "logits/rejected": -0.5487757325172424, |
| "logps/chosen": -394.2236022949219, |
| "logps/rejected": -564.2678833007812, |
| "loss": 0.05, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.3238348960876465, |
| "rewards/margins": 13.824040412902832, |
| "rewards/rejected": -16.147876739501953, |
| "step": 1160 |
| }, |
| { |
| "epoch": 0.75, |
| "learning_rate": 4.85955724827422e-07, |
| "logits/chosen": -0.7651220560073853, |
| "logits/rejected": -0.06668927520513535, |
| "logps/chosen": -444.3988342285156, |
| "logps/rejected": -548.660888671875, |
| "loss": 0.0583, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.920553684234619, |
| "rewards/margins": 13.667207717895508, |
| "rewards/rejected": -16.5877628326416, |
| "step": 1170 |
| }, |
| { |
| "epoch": 0.76, |
| "learning_rate": 4.853606284218044e-07, |
| "logits/chosen": -1.0830278396606445, |
| "logits/rejected": -0.4501362442970276, |
| "logps/chosen": -332.0751037597656, |
| "logps/rejected": -418.5309143066406, |
| "loss": 0.0789, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.7456166744232178, |
| "rewards/margins": 11.250526428222656, |
| "rewards/rejected": -12.996142387390137, |
| "step": 1180 |
| }, |
| { |
| "epoch": 0.76, |
| "learning_rate": 4.847655320161866e-07, |
| "logits/chosen": -0.6781784296035767, |
| "logits/rejected": -0.27196237444877625, |
| "logps/chosen": -396.753662109375, |
| "logps/rejected": -491.14892578125, |
| "loss": 0.0668, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.8804516792297363, |
| "rewards/margins": 11.853898048400879, |
| "rewards/rejected": -14.734350204467773, |
| "step": 1190 |
| }, |
| { |
| "epoch": 0.77, |
| "learning_rate": 4.841704356105689e-07, |
| "logits/chosen": -0.7234100103378296, |
| "logits/rejected": -0.2980794310569763, |
| "logps/chosen": -367.587158203125, |
| "logps/rejected": -433.124755859375, |
| "loss": 0.0822, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -3.162593364715576, |
| "rewards/margins": 10.242734909057617, |
| "rewards/rejected": -13.405328750610352, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.77, |
| "eval_logits/chosen": -1.0001459121704102, |
| "eval_logits/rejected": -0.6194829940795898, |
| "eval_logps/chosen": -365.28955078125, |
| "eval_logps/rejected": -458.57135009765625, |
| "eval_loss": 0.13022372126579285, |
| "eval_rewards/accuracies": 0.96875, |
| "eval_rewards/chosen": -3.5926475524902344, |
| "eval_rewards/margins": 10.629414558410645, |
| "eval_rewards/rejected": -14.222061157226562, |
| "eval_runtime": 76.5043, |
| "eval_samples_per_second": 13.071, |
| "eval_steps_per_second": 0.418, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.78, |
| "learning_rate": 4.835753392049512e-07, |
| "logits/chosen": -0.6337307691574097, |
| "logits/rejected": -0.23596186935901642, |
| "logps/chosen": -356.19122314453125, |
| "logps/rejected": -514.327392578125, |
| "loss": 0.1085, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -3.958400249481201, |
| "rewards/margins": 13.92591667175293, |
| "rewards/rejected": -17.884319305419922, |
| "step": 1210 |
| }, |
| { |
| "epoch": 0.78, |
| "learning_rate": 4.829802427993334e-07, |
| "logits/chosen": -0.6189112663269043, |
| "logits/rejected": -0.34567025303840637, |
| "logps/chosen": -374.00286865234375, |
| "logps/rejected": -474.2106018066406, |
| "loss": 0.1089, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -3.2338080406188965, |
| "rewards/margins": 9.657389640808105, |
| "rewards/rejected": -12.891199111938477, |
| "step": 1220 |
| }, |
| { |
| "epoch": 0.79, |
| "learning_rate": 4.823851463937158e-07, |
| "logits/chosen": -0.3865962028503418, |
| "logits/rejected": 0.04075580835342407, |
| "logps/chosen": -359.3363342285156, |
| "logps/rejected": -501.40240478515625, |
| "loss": 0.1142, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.306814193725586, |
| "rewards/margins": 13.71052074432373, |
| "rewards/rejected": -18.017335891723633, |
| "step": 1230 |
| }, |
| { |
| "epoch": 0.8, |
| "learning_rate": 4.81790049988098e-07, |
| "logits/chosen": -0.7158625721931458, |
| "logits/rejected": -0.21917279064655304, |
| "logps/chosen": -434.91253662109375, |
| "logps/rejected": -528.3048095703125, |
| "loss": 0.0757, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.414928913116455, |
| "rewards/margins": 12.52367115020752, |
| "rewards/rejected": -15.938600540161133, |
| "step": 1240 |
| }, |
| { |
| "epoch": 0.8, |
| "learning_rate": 4.811949535824804e-07, |
| "logits/chosen": -0.7639582753181458, |
| "logits/rejected": -0.4316403269767761, |
| "logps/chosen": -365.72784423828125, |
| "logps/rejected": -567.1033935546875, |
| "loss": 0.084, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.063331127166748, |
| "rewards/margins": 12.180209159851074, |
| "rewards/rejected": -15.243539810180664, |
| "step": 1250 |
| }, |
| { |
| "epoch": 0.81, |
| "learning_rate": 4.805998571768626e-07, |
| "logits/chosen": -1.2161794900894165, |
| "logits/rejected": -0.6932582259178162, |
| "logps/chosen": -406.9449768066406, |
| "logps/rejected": -520.5137939453125, |
| "loss": 0.0951, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.868889331817627, |
| "rewards/margins": 10.655603408813477, |
| "rewards/rejected": -14.524490356445312, |
| "step": 1260 |
| }, |
| { |
| "epoch": 0.82, |
| "learning_rate": 4.80004760771245e-07, |
| "logits/chosen": -0.9595744013786316, |
| "logits/rejected": -0.43417948484420776, |
| "logps/chosen": -411.44677734375, |
| "logps/rejected": -526.41845703125, |
| "loss": 0.0691, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.0373916625976562, |
| "rewards/margins": 9.937090873718262, |
| "rewards/rejected": -12.97448444366455, |
| "step": 1270 |
| }, |
| { |
| "epoch": 0.82, |
| "learning_rate": 4.794096643656272e-07, |
| "logits/chosen": -1.0694881677627563, |
| "logits/rejected": -0.5757830142974854, |
| "logps/chosen": -382.9389343261719, |
| "logps/rejected": -500.0220642089844, |
| "loss": 0.0743, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.738356113433838, |
| "rewards/margins": 10.426973342895508, |
| "rewards/rejected": -14.165328979492188, |
| "step": 1280 |
| }, |
| { |
| "epoch": 0.83, |
| "learning_rate": 4.788145679600095e-07, |
| "logits/chosen": -0.9496662020683289, |
| "logits/rejected": -0.3601847290992737, |
| "logps/chosen": -358.3319396972656, |
| "logps/rejected": -560.3468627929688, |
| "loss": 0.0742, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.929766893386841, |
| "rewards/margins": 13.46239185333252, |
| "rewards/rejected": -16.39215660095215, |
| "step": 1290 |
| }, |
| { |
| "epoch": 0.84, |
| "learning_rate": 4.782194715543918e-07, |
| "logits/chosen": -0.6319989562034607, |
| "logits/rejected": -0.09417597949504852, |
| "logps/chosen": -379.7413024902344, |
| "logps/rejected": -522.1488037109375, |
| "loss": 0.063, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.54395055770874, |
| "rewards/margins": 13.589025497436523, |
| "rewards/rejected": -18.132978439331055, |
| "step": 1300 |
| }, |
| { |
| "epoch": 0.84, |
| "eval_logits/chosen": -0.9766608476638794, |
| "eval_logits/rejected": -0.467482328414917, |
| "eval_logps/chosen": -374.5415954589844, |
| "eval_logps/rejected": -472.6826477050781, |
| "eval_loss": 0.18041668832302094, |
| "eval_rewards/accuracies": 0.9375, |
| "eval_rewards/chosen": -4.517853260040283, |
| "eval_rewards/margins": 11.115335464477539, |
| "eval_rewards/rejected": -15.633190155029297, |
| "eval_runtime": 76.7717, |
| "eval_samples_per_second": 13.026, |
| "eval_steps_per_second": 0.417, |
| "step": 1300 |
| }, |
| { |
| "epoch": 0.84, |
| "learning_rate": 4.77624375148774e-07, |
| "logits/chosen": -0.6732112169265747, |
| "logits/rejected": 0.014696260914206505, |
| "logps/chosen": -417.83074951171875, |
| "logps/rejected": -488.5224609375, |
| "loss": 0.0634, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.793654441833496, |
| "rewards/margins": 13.286532402038574, |
| "rewards/rejected": -18.08018684387207, |
| "step": 1310 |
| }, |
| { |
| "epoch": 0.85, |
| "learning_rate": 4.770292787431564e-07, |
| "logits/chosen": -0.3928489685058594, |
| "logits/rejected": -0.07063998281955719, |
| "logps/chosen": -365.6458435058594, |
| "logps/rejected": -472.38043212890625, |
| "loss": 0.0885, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -4.527663707733154, |
| "rewards/margins": 11.80773639678955, |
| "rewards/rejected": -16.335399627685547, |
| "step": 1320 |
| }, |
| { |
| "epoch": 0.85, |
| "learning_rate": 4.764341823375387e-07, |
| "logits/chosen": -0.5924497842788696, |
| "logits/rejected": 0.10169048607349396, |
| "logps/chosen": -394.43292236328125, |
| "logps/rejected": -514.439697265625, |
| "loss": 0.0762, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.806927442550659, |
| "rewards/margins": 12.637249946594238, |
| "rewards/rejected": -16.444177627563477, |
| "step": 1330 |
| }, |
| { |
| "epoch": 0.86, |
| "learning_rate": 4.7583908593192097e-07, |
| "logits/chosen": -0.6815664172172546, |
| "logits/rejected": -0.29060396552085876, |
| "logps/chosen": -420.83221435546875, |
| "logps/rejected": -551.6920776367188, |
| "loss": 0.0532, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.1024906635284424, |
| "rewards/margins": 13.700825691223145, |
| "rewards/rejected": -16.80331802368164, |
| "step": 1340 |
| }, |
| { |
| "epoch": 0.87, |
| "learning_rate": 4.752439895263032e-07, |
| "logits/chosen": -0.689470112323761, |
| "logits/rejected": -0.20767569541931152, |
| "logps/chosen": -385.32037353515625, |
| "logps/rejected": -533.0687255859375, |
| "loss": 0.0878, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -3.7341361045837402, |
| "rewards/margins": 14.102251052856445, |
| "rewards/rejected": -17.83638572692871, |
| "step": 1350 |
| }, |
| { |
| "epoch": 0.87, |
| "learning_rate": 4.746488931206855e-07, |
| "logits/chosen": -0.7405373454093933, |
| "logits/rejected": -0.07327382266521454, |
| "logps/chosen": -377.8034973144531, |
| "logps/rejected": -466.72784423828125, |
| "loss": 0.0851, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -3.5931084156036377, |
| "rewards/margins": 12.507670402526855, |
| "rewards/rejected": -16.100778579711914, |
| "step": 1360 |
| }, |
| { |
| "epoch": 0.88, |
| "learning_rate": 4.7405379671506785e-07, |
| "logits/chosen": -0.5811715126037598, |
| "logits/rejected": -0.06195932626724243, |
| "logps/chosen": -411.6004943847656, |
| "logps/rejected": -548.6055908203125, |
| "loss": 0.1093, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -4.2639055252075195, |
| "rewards/margins": 11.680307388305664, |
| "rewards/rejected": -15.944211959838867, |
| "step": 1370 |
| }, |
| { |
| "epoch": 0.89, |
| "learning_rate": 4.734587003094501e-07, |
| "logits/chosen": -0.8370053172111511, |
| "logits/rejected": -0.14809687435626984, |
| "logps/chosen": -447.0265197753906, |
| "logps/rejected": -535.3525390625, |
| "loss": 0.081, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -4.550225734710693, |
| "rewards/margins": 12.257158279418945, |
| "rewards/rejected": -16.807382583618164, |
| "step": 1380 |
| }, |
| { |
| "epoch": 0.89, |
| "learning_rate": 4.728636039038324e-07, |
| "logits/chosen": -0.7418749928474426, |
| "logits/rejected": -0.25607532262802124, |
| "logps/chosen": -374.479248046875, |
| "logps/rejected": -532.3735961914062, |
| "loss": 0.0738, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.4016432762146, |
| "rewards/margins": 13.285806655883789, |
| "rewards/rejected": -18.687450408935547, |
| "step": 1390 |
| }, |
| { |
| "epoch": 0.9, |
| "learning_rate": 4.722685074982147e-07, |
| "logits/chosen": -0.8056036233901978, |
| "logits/rejected": -0.2640933394432068, |
| "logps/chosen": -372.2264709472656, |
| "logps/rejected": -565.7047729492188, |
| "loss": 0.0648, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.8431191444396973, |
| "rewards/margins": 13.19017505645752, |
| "rewards/rejected": -17.033292770385742, |
| "step": 1400 |
| }, |
| { |
| "epoch": 0.9, |
| "eval_logits/chosen": -1.118211030960083, |
| "eval_logits/rejected": -0.5943832397460938, |
| "eval_logps/chosen": -354.5495300292969, |
| "eval_logps/rejected": -446.5892333984375, |
| "eval_loss": 0.13570235669612885, |
| "eval_rewards/accuracies": 0.953125, |
| "eval_rewards/chosen": -2.518648147583008, |
| "eval_rewards/margins": 10.50519847869873, |
| "eval_rewards/rejected": -13.023846626281738, |
| "eval_runtime": 76.6799, |
| "eval_samples_per_second": 13.041, |
| "eval_steps_per_second": 0.417, |
| "step": 1400 |
| }, |
| { |
| "epoch": 0.91, |
| "learning_rate": 4.7167341109259703e-07, |
| "logits/chosen": -1.0566179752349854, |
| "logits/rejected": -0.2900046706199646, |
| "logps/chosen": -398.8577575683594, |
| "logps/rejected": -454.0634765625, |
| "loss": 0.094, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.2813262939453125, |
| "rewards/margins": 11.221616744995117, |
| "rewards/rejected": -13.502942085266113, |
| "step": 1410 |
| }, |
| { |
| "epoch": 0.91, |
| "learning_rate": 4.710783146869793e-07, |
| "logits/chosen": -0.9073772430419922, |
| "logits/rejected": -0.400244802236557, |
| "logps/chosen": -357.8586730957031, |
| "logps/rejected": -546.3787841796875, |
| "loss": 0.0928, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -3.4375388622283936, |
| "rewards/margins": 11.90378475189209, |
| "rewards/rejected": -15.341323852539062, |
| "step": 1420 |
| }, |
| { |
| "epoch": 0.92, |
| "learning_rate": 4.7048321828136157e-07, |
| "logits/chosen": -0.6786088347434998, |
| "logits/rejected": -0.10559716075658798, |
| "logps/chosen": -417.87939453125, |
| "logps/rejected": -530.9561767578125, |
| "loss": 0.042, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.6138105392456055, |
| "rewards/margins": 13.705339431762695, |
| "rewards/rejected": -19.319150924682617, |
| "step": 1430 |
| }, |
| { |
| "epoch": 0.93, |
| "learning_rate": 4.698881218757438e-07, |
| "logits/chosen": -0.5866619348526001, |
| "logits/rejected": -0.026242520660161972, |
| "logps/chosen": -422.72613525390625, |
| "logps/rejected": -541.8472900390625, |
| "loss": 0.1036, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -4.567929267883301, |
| "rewards/margins": 11.613755226135254, |
| "rewards/rejected": -16.181686401367188, |
| "step": 1440 |
| }, |
| { |
| "epoch": 0.93, |
| "learning_rate": 4.692930254701261e-07, |
| "logits/chosen": -0.7376483082771301, |
| "logits/rejected": -0.09872325509786606, |
| "logps/chosen": -425.328857421875, |
| "logps/rejected": -529.0191650390625, |
| "loss": 0.0628, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -4.07589054107666, |
| "rewards/margins": 12.2763032913208, |
| "rewards/rejected": -16.352191925048828, |
| "step": 1450 |
| }, |
| { |
| "epoch": 0.94, |
| "learning_rate": 4.6869792906450845e-07, |
| "logits/chosen": -0.5996996760368347, |
| "logits/rejected": 0.09782281517982483, |
| "logps/chosen": -426.3692321777344, |
| "logps/rejected": -495.6768493652344, |
| "loss": 0.0448, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.688590049743652, |
| "rewards/margins": 13.677145004272461, |
| "rewards/rejected": -18.36573600769043, |
| "step": 1460 |
| }, |
| { |
| "epoch": 0.94, |
| "learning_rate": 4.6810283265889075e-07, |
| "logits/chosen": -0.6974693536758423, |
| "logits/rejected": 0.09173402935266495, |
| "logps/chosen": -361.08746337890625, |
| "logps/rejected": -483.73211669921875, |
| "loss": 0.0571, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.233609199523926, |
| "rewards/margins": 13.777618408203125, |
| "rewards/rejected": -18.011228561401367, |
| "step": 1470 |
| }, |
| { |
| "epoch": 0.95, |
| "learning_rate": 4.67507736253273e-07, |
| "logits/chosen": -0.8001763224601746, |
| "logits/rejected": -0.1673848032951355, |
| "logps/chosen": -368.32196044921875, |
| "logps/rejected": -508.84368896484375, |
| "loss": 0.0676, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.7094593048095703, |
| "rewards/margins": 13.913045883178711, |
| "rewards/rejected": -16.622507095336914, |
| "step": 1480 |
| }, |
| { |
| "epoch": 0.96, |
| "learning_rate": 4.669126398476553e-07, |
| "logits/chosen": -0.589401125907898, |
| "logits/rejected": 0.09411343187093735, |
| "logps/chosen": -448.2190856933594, |
| "logps/rejected": -579.1790161132812, |
| "loss": 0.07, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.9501609802246094, |
| "rewards/margins": 15.417715072631836, |
| "rewards/rejected": -19.367877960205078, |
| "step": 1490 |
| }, |
| { |
| "epoch": 0.96, |
| "learning_rate": 4.6631754344203763e-07, |
| "logits/chosen": -0.6391473412513733, |
| "logits/rejected": -0.1050008162856102, |
| "logps/chosen": -376.50579833984375, |
| "logps/rejected": -544.2982177734375, |
| "loss": 0.0714, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -3.8969063758850098, |
| "rewards/margins": 13.172216415405273, |
| "rewards/rejected": -17.069122314453125, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.96, |
| "eval_logits/chosen": -1.0033119916915894, |
| "eval_logits/rejected": -0.4300505220890045, |
| "eval_logps/chosen": -374.3122863769531, |
| "eval_logps/rejected": -470.21783447265625, |
| "eval_loss": 0.12438826262950897, |
| "eval_rewards/accuracies": 0.9453125, |
| "eval_rewards/chosen": -4.494921684265137, |
| "eval_rewards/margins": 10.89178466796875, |
| "eval_rewards/rejected": -15.38670539855957, |
| "eval_runtime": 76.5689, |
| "eval_samples_per_second": 13.06, |
| "eval_steps_per_second": 0.418, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.97, |
| "learning_rate": 4.657224470364199e-07, |
| "logits/chosen": -0.43436574935913086, |
| "logits/rejected": 0.07220065593719482, |
| "logps/chosen": -391.37725830078125, |
| "logps/rejected": -588.6260986328125, |
| "loss": 0.0546, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.231613636016846, |
| "rewards/margins": 13.30431842803955, |
| "rewards/rejected": -18.535930633544922, |
| "step": 1510 |
| }, |
| { |
| "epoch": 0.98, |
| "learning_rate": 4.6512735063080217e-07, |
| "logits/chosen": -0.6157822608947754, |
| "logits/rejected": 0.005131366662681103, |
| "logps/chosen": -411.13946533203125, |
| "logps/rejected": -483.9326171875, |
| "loss": 0.0527, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.003513336181641, |
| "rewards/margins": 10.31053638458252, |
| "rewards/rejected": -15.314050674438477, |
| "step": 1520 |
| }, |
| { |
| "epoch": 0.98, |
| "learning_rate": 4.6453225422518447e-07, |
| "logits/chosen": -0.6011049747467041, |
| "logits/rejected": -0.10970073938369751, |
| "logps/chosen": -406.9378356933594, |
| "logps/rejected": -468.97314453125, |
| "loss": 0.0739, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.8249969482421875, |
| "rewards/margins": 11.424562454223633, |
| "rewards/rejected": -16.24955940246582, |
| "step": 1530 |
| }, |
| { |
| "epoch": 0.99, |
| "learning_rate": 4.6393715781956676e-07, |
| "logits/chosen": -0.6942557096481323, |
| "logits/rejected": 0.16295239329338074, |
| "logps/chosen": -407.1282653808594, |
| "logps/rejected": -560.19970703125, |
| "loss": 0.0629, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.603503704071045, |
| "rewards/margins": 13.417546272277832, |
| "rewards/rejected": -18.02104949951172, |
| "step": 1540 |
| }, |
| { |
| "epoch": 1.0, |
| "learning_rate": 4.6334206141394905e-07, |
| "logits/chosen": -0.3027026057243347, |
| "logits/rejected": 0.10327012836933136, |
| "logps/chosen": -405.692626953125, |
| "logps/rejected": -576.5963134765625, |
| "loss": 0.0673, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.6844029426574707, |
| "rewards/margins": 13.106195449829102, |
| "rewards/rejected": -16.790597915649414, |
| "step": 1550 |
| }, |
| { |
| "epoch": 1.0, |
| "learning_rate": 4.6274696500833135e-07, |
| "logits/chosen": -0.5037144422531128, |
| "logits/rejected": 0.06204764172434807, |
| "logps/chosen": -412.0623474121094, |
| "logps/rejected": -525.1038818359375, |
| "loss": 0.061, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.6065430641174316, |
| "rewards/margins": 14.261393547058105, |
| "rewards/rejected": -16.867937088012695, |
| "step": 1560 |
| }, |
| { |
| "epoch": 1.01, |
| "learning_rate": 4.621518686027136e-07, |
| "logits/chosen": -0.526481568813324, |
| "logits/rejected": 0.24668976664543152, |
| "logps/chosen": -410.71746826171875, |
| "logps/rejected": -473.8052673339844, |
| "loss": 0.026, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.839338779449463, |
| "rewards/margins": 13.472638130187988, |
| "rewards/rejected": -16.31197738647461, |
| "step": 1570 |
| }, |
| { |
| "epoch": 1.02, |
| "learning_rate": 4.6155677219709594e-07, |
| "logits/chosen": -0.2807101607322693, |
| "logits/rejected": 0.3707982003688812, |
| "logps/chosen": -430.853759765625, |
| "logps/rejected": -465.86328125, |
| "loss": 0.016, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.3595848083496094, |
| "rewards/margins": 12.908388137817383, |
| "rewards/rejected": -16.267974853515625, |
| "step": 1580 |
| }, |
| { |
| "epoch": 1.02, |
| "learning_rate": 4.6096167579147823e-07, |
| "logits/chosen": -0.2973923683166504, |
| "logits/rejected": 0.30676135420799255, |
| "logps/chosen": -384.2185974121094, |
| "logps/rejected": -527.7244873046875, |
| "loss": 0.0162, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.731484889984131, |
| "rewards/margins": 15.676348686218262, |
| "rewards/rejected": -19.407833099365234, |
| "step": 1590 |
| }, |
| { |
| "epoch": 1.03, |
| "learning_rate": 4.603665793858605e-07, |
| "logits/chosen": -0.5543831586837769, |
| "logits/rejected": 0.3015816807746887, |
| "logps/chosen": -395.59368896484375, |
| "logps/rejected": -544.031494140625, |
| "loss": 0.0095, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.2728145122528076, |
| "rewards/margins": 15.476274490356445, |
| "rewards/rejected": -18.74909019470215, |
| "step": 1600 |
| }, |
| { |
| "epoch": 1.03, |
| "eval_logits/chosen": -0.7791086435317993, |
| "eval_logits/rejected": -0.20377439260482788, |
| "eval_logps/chosen": -380.9189453125, |
| "eval_logps/rejected": -480.3507995605469, |
| "eval_loss": 0.10078604519367218, |
| "eval_rewards/accuracies": 0.9375, |
| "eval_rewards/chosen": -5.155591011047363, |
| "eval_rewards/margins": 11.244410514831543, |
| "eval_rewards/rejected": -16.40000343322754, |
| "eval_runtime": 76.553, |
| "eval_samples_per_second": 13.063, |
| "eval_steps_per_second": 0.418, |
| "step": 1600 |
| }, |
| { |
| "epoch": 1.03, |
| "learning_rate": 4.5977148298024277e-07, |
| "logits/chosen": -0.5526180267333984, |
| "logits/rejected": 0.3482429087162018, |
| "logps/chosen": -391.2712707519531, |
| "logps/rejected": -509.62847900390625, |
| "loss": 0.0064, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.7447268962860107, |
| "rewards/margins": 15.208663940429688, |
| "rewards/rejected": -18.95339012145996, |
| "step": 1610 |
| }, |
| { |
| "epoch": 1.04, |
| "learning_rate": 4.5917638657462507e-07, |
| "logits/chosen": -0.4088626801967621, |
| "logits/rejected": 0.3109140992164612, |
| "logps/chosen": -426.03643798828125, |
| "logps/rejected": -556.555908203125, |
| "loss": 0.0053, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.894583702087402, |
| "rewards/margins": 14.70109748840332, |
| "rewards/rejected": -19.595678329467773, |
| "step": 1620 |
| }, |
| { |
| "epoch": 1.05, |
| "learning_rate": 4.5858129016900736e-07, |
| "logits/chosen": -0.5371668934822083, |
| "logits/rejected": 0.3167404532432556, |
| "logps/chosen": -393.68524169921875, |
| "logps/rejected": -562.7149658203125, |
| "loss": 0.0129, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.771970272064209, |
| "rewards/margins": 15.018495559692383, |
| "rewards/rejected": -19.79046630859375, |
| "step": 1630 |
| }, |
| { |
| "epoch": 1.05, |
| "learning_rate": 4.5798619376338966e-07, |
| "logits/chosen": -0.247820645570755, |
| "logits/rejected": 0.3391680121421814, |
| "logps/chosen": -370.0538024902344, |
| "logps/rejected": -491.40252685546875, |
| "loss": 0.0067, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.405149936676025, |
| "rewards/margins": 14.295089721679688, |
| "rewards/rejected": -18.700239181518555, |
| "step": 1640 |
| }, |
| { |
| "epoch": 1.06, |
| "learning_rate": 4.5739109735777195e-07, |
| "logits/chosen": -0.2800091505050659, |
| "logits/rejected": 0.2654980719089508, |
| "logps/chosen": -439.1228942871094, |
| "logps/rejected": -581.3944091796875, |
| "loss": 0.0115, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.183625221252441, |
| "rewards/margins": 16.414287567138672, |
| "rewards/rejected": -22.597911834716797, |
| "step": 1650 |
| }, |
| { |
| "epoch": 1.07, |
| "learning_rate": 4.567960009521542e-07, |
| "logits/chosen": -0.32536178827285767, |
| "logits/rejected": 0.2308044731616974, |
| "logps/chosen": -362.37945556640625, |
| "logps/rejected": -542.333984375, |
| "loss": 0.0122, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.910571575164795, |
| "rewards/margins": 15.097010612487793, |
| "rewards/rejected": -19.007579803466797, |
| "step": 1660 |
| }, |
| { |
| "epoch": 1.07, |
| "learning_rate": 4.5620090454653654e-07, |
| "logits/chosen": -0.44402211904525757, |
| "logits/rejected": 0.07348278164863586, |
| "logps/chosen": -377.78759765625, |
| "logps/rejected": -574.8585205078125, |
| "loss": 0.0117, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.063399314880371, |
| "rewards/margins": 16.107744216918945, |
| "rewards/rejected": -21.171140670776367, |
| "step": 1670 |
| }, |
| { |
| "epoch": 1.08, |
| "learning_rate": 4.5560580814091884e-07, |
| "logits/chosen": -0.6580984592437744, |
| "logits/rejected": 0.15282706916332245, |
| "logps/chosen": -388.4255065917969, |
| "logps/rejected": -543.6886596679688, |
| "loss": 0.0331, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.285706996917725, |
| "rewards/margins": 15.84802532196045, |
| "rewards/rejected": -20.13373374938965, |
| "step": 1680 |
| }, |
| { |
| "epoch": 1.09, |
| "learning_rate": 4.550107117353011e-07, |
| "logits/chosen": -0.5916538834571838, |
| "logits/rejected": 0.042367033660411835, |
| "logps/chosen": -425.89013671875, |
| "logps/rejected": -556.3162841796875, |
| "loss": 0.0158, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.548110008239746, |
| "rewards/margins": 14.834815979003906, |
| "rewards/rejected": -19.38292694091797, |
| "step": 1690 |
| }, |
| { |
| "epoch": 1.09, |
| "learning_rate": 4.5441561532968337e-07, |
| "logits/chosen": -0.6858727335929871, |
| "logits/rejected": -0.02261565811932087, |
| "logps/chosen": -391.042724609375, |
| "logps/rejected": -572.4518432617188, |
| "loss": 0.0096, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.9488537311553955, |
| "rewards/margins": 17.172945022583008, |
| "rewards/rejected": -21.121801376342773, |
| "step": 1700 |
| }, |
| { |
| "epoch": 1.09, |
| "eval_logits/chosen": -0.9992736577987671, |
| "eval_logits/rejected": -0.40412917733192444, |
| "eval_logps/chosen": -377.3610534667969, |
| "eval_logps/rejected": -483.4149169921875, |
| "eval_loss": 0.1330709010362625, |
| "eval_rewards/accuracies": 0.953125, |
| "eval_rewards/chosen": -4.799799919128418, |
| "eval_rewards/margins": 11.906618118286133, |
| "eval_rewards/rejected": -16.706417083740234, |
| "eval_runtime": 76.6396, |
| "eval_samples_per_second": 13.048, |
| "eval_steps_per_second": 0.418, |
| "step": 1700 |
| }, |
| { |
| "epoch": 1.1, |
| "learning_rate": 4.538205189240657e-07, |
| "logits/chosen": -0.4306742548942566, |
| "logits/rejected": 0.3547573983669281, |
| "logps/chosen": -398.1122741699219, |
| "logps/rejected": -550.4100341796875, |
| "loss": 0.0132, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.38922643661499, |
| "rewards/margins": 16.458080291748047, |
| "rewards/rejected": -21.847307205200195, |
| "step": 1710 |
| }, |
| { |
| "epoch": 1.11, |
| "learning_rate": 4.5322542251844796e-07, |
| "logits/chosen": -0.5155312418937683, |
| "logits/rejected": -0.017366236075758934, |
| "logps/chosen": -410.91412353515625, |
| "logps/rejected": -553.8331298828125, |
| "loss": 0.0058, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.796280384063721, |
| "rewards/margins": 16.963947296142578, |
| "rewards/rejected": -21.76022720336914, |
| "step": 1720 |
| }, |
| { |
| "epoch": 1.11, |
| "learning_rate": 4.5263032611283026e-07, |
| "logits/chosen": -0.12997198104858398, |
| "logits/rejected": 0.2565564215183258, |
| "logps/chosen": -405.981201171875, |
| "logps/rejected": -620.6024169921875, |
| "loss": 0.014, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.828683376312256, |
| "rewards/margins": 18.5574893951416, |
| "rewards/rejected": -23.386173248291016, |
| "step": 1730 |
| }, |
| { |
| "epoch": 1.12, |
| "learning_rate": 4.5203522970721255e-07, |
| "logits/chosen": -0.0680803433060646, |
| "logits/rejected": 0.4802146852016449, |
| "logps/chosen": -421.7552185058594, |
| "logps/rejected": -488.4523010253906, |
| "loss": 0.0131, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.658025026321411, |
| "rewards/margins": 15.093057632446289, |
| "rewards/rejected": -18.751081466674805, |
| "step": 1740 |
| }, |
| { |
| "epoch": 1.12, |
| "learning_rate": 4.5144013330159485e-07, |
| "logits/chosen": -0.3530596196651459, |
| "logits/rejected": 0.2572212815284729, |
| "logps/chosen": -413.51629638671875, |
| "logps/rejected": -577.3765869140625, |
| "loss": 0.0179, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.011878967285156, |
| "rewards/margins": 15.384442329406738, |
| "rewards/rejected": -19.39632225036621, |
| "step": 1750 |
| }, |
| { |
| "epoch": 1.13, |
| "learning_rate": 4.5084503689597714e-07, |
| "logits/chosen": -0.28345853090286255, |
| "logits/rejected": 0.42750459909439087, |
| "logps/chosen": -437.33941650390625, |
| "logps/rejected": -554.0641479492188, |
| "loss": 0.0124, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.84152889251709, |
| "rewards/margins": 14.463783264160156, |
| "rewards/rejected": -19.305313110351562, |
| "step": 1760 |
| }, |
| { |
| "epoch": 1.14, |
| "learning_rate": 4.5024994049035944e-07, |
| "logits/chosen": -0.09664113819599152, |
| "logits/rejected": 0.22168950736522675, |
| "logps/chosen": -378.29632568359375, |
| "logps/rejected": -565.90185546875, |
| "loss": 0.0134, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.932016372680664, |
| "rewards/margins": 17.979869842529297, |
| "rewards/rejected": -22.911888122558594, |
| "step": 1770 |
| }, |
| { |
| "epoch": 1.14, |
| "learning_rate": 4.496548440847417e-07, |
| "logits/chosen": -0.049262501299381256, |
| "logits/rejected": 0.4335893988609314, |
| "logps/chosen": -402.48980712890625, |
| "logps/rejected": -552.8167114257812, |
| "loss": 0.0279, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.7815442085266113, |
| "rewards/margins": 15.186810493469238, |
| "rewards/rejected": -18.968353271484375, |
| "step": 1780 |
| }, |
| { |
| "epoch": 1.15, |
| "learning_rate": 4.49059747679124e-07, |
| "logits/chosen": -0.28909042477607727, |
| "logits/rejected": 0.3799073398113251, |
| "logps/chosen": -430.60760498046875, |
| "logps/rejected": -537.1256713867188, |
| "loss": 0.0168, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.874712944030762, |
| "rewards/margins": 15.4542875289917, |
| "rewards/rejected": -20.32900047302246, |
| "step": 1790 |
| }, |
| { |
| "epoch": 1.16, |
| "learning_rate": 4.484646512735063e-07, |
| "logits/chosen": -0.3206165134906769, |
| "logits/rejected": 0.06656259298324585, |
| "logps/chosen": -471.58001708984375, |
| "logps/rejected": -507.40301513671875, |
| "loss": 0.0293, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.206088066101074, |
| "rewards/margins": 13.663434982299805, |
| "rewards/rejected": -17.869524002075195, |
| "step": 1800 |
| }, |
| { |
| "epoch": 1.16, |
| "eval_logits/chosen": -0.8347997665405273, |
| "eval_logits/rejected": -0.41860231757164, |
| "eval_logps/chosen": -372.9380187988281, |
| "eval_logps/rejected": -465.8204345703125, |
| "eval_loss": 0.14275716245174408, |
| "eval_rewards/accuracies": 0.9609375, |
| "eval_rewards/chosen": -4.35749626159668, |
| "eval_rewards/margins": 10.589475631713867, |
| "eval_rewards/rejected": -14.94697093963623, |
| "eval_runtime": 76.7934, |
| "eval_samples_per_second": 13.022, |
| "eval_steps_per_second": 0.417, |
| "step": 1800 |
| }, |
| { |
| "epoch": 1.16, |
| "learning_rate": 4.4786955486788856e-07, |
| "logits/chosen": -0.3631385266780853, |
| "logits/rejected": 0.07067543268203735, |
| "logps/chosen": -377.46551513671875, |
| "logps/rejected": -519.6616821289062, |
| "loss": 0.0359, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.011387586593628, |
| "rewards/margins": 15.720315933227539, |
| "rewards/rejected": -18.73170280456543, |
| "step": 1810 |
| }, |
| { |
| "epoch": 1.17, |
| "learning_rate": 4.4727445846227086e-07, |
| "logits/chosen": -0.36144647002220154, |
| "logits/rejected": -0.008104220032691956, |
| "logps/chosen": -397.82940673828125, |
| "logps/rejected": -593.34521484375, |
| "loss": 0.0099, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.28765869140625, |
| "rewards/margins": 15.561511039733887, |
| "rewards/rejected": -20.849170684814453, |
| "step": 1820 |
| }, |
| { |
| "epoch": 1.18, |
| "learning_rate": 4.4667936205665315e-07, |
| "logits/chosen": -0.1907232105731964, |
| "logits/rejected": 0.313555508852005, |
| "logps/chosen": -393.31719970703125, |
| "logps/rejected": -510.4048767089844, |
| "loss": 0.0247, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.2387800216674805, |
| "rewards/margins": 14.292640686035156, |
| "rewards/rejected": -18.531421661376953, |
| "step": 1830 |
| }, |
| { |
| "epoch": 1.18, |
| "learning_rate": 4.4608426565103545e-07, |
| "logits/chosen": -0.6258490085601807, |
| "logits/rejected": 0.040771596133708954, |
| "logps/chosen": -419.44549560546875, |
| "logps/rejected": -544.8959350585938, |
| "loss": 0.0159, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.875296115875244, |
| "rewards/margins": 15.84538459777832, |
| "rewards/rejected": -19.720678329467773, |
| "step": 1840 |
| }, |
| { |
| "epoch": 1.19, |
| "learning_rate": 4.4548916924541774e-07, |
| "logits/chosen": -0.3880535066127777, |
| "logits/rejected": -0.07375472038984299, |
| "logps/chosen": -354.2195739746094, |
| "logps/rejected": -516.4483642578125, |
| "loss": 0.024, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.647956609725952, |
| "rewards/margins": 14.303915023803711, |
| "rewards/rejected": -16.95186996459961, |
| "step": 1850 |
| }, |
| { |
| "epoch": 1.2, |
| "learning_rate": 4.4489407283980004e-07, |
| "logits/chosen": 0.03841676935553551, |
| "logits/rejected": 0.44405698776245117, |
| "logps/chosen": -405.54351806640625, |
| "logps/rejected": -583.1129760742188, |
| "loss": 0.0113, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.536320686340332, |
| "rewards/margins": 15.479736328125, |
| "rewards/rejected": -21.016056060791016, |
| "step": 1860 |
| }, |
| { |
| "epoch": 1.2, |
| "learning_rate": 4.442989764341823e-07, |
| "logits/chosen": 0.409410297870636, |
| "logits/rejected": 0.6078455448150635, |
| "logps/chosen": -425.523193359375, |
| "logps/rejected": -559.0198364257812, |
| "loss": 0.0116, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -7.118569374084473, |
| "rewards/margins": 15.779667854309082, |
| "rewards/rejected": -22.898235321044922, |
| "step": 1870 |
| }, |
| { |
| "epoch": 1.21, |
| "learning_rate": 4.437038800285646e-07, |
| "logits/chosen": 0.023495376110076904, |
| "logits/rejected": 0.8307549357414246, |
| "logps/chosen": -367.8580017089844, |
| "logps/rejected": -482.8976135253906, |
| "loss": 0.0228, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.339402198791504, |
| "rewards/margins": 14.32616901397705, |
| "rewards/rejected": -18.665569305419922, |
| "step": 1880 |
| }, |
| { |
| "epoch": 1.21, |
| "learning_rate": 4.431087836229469e-07, |
| "logits/chosen": -0.1776140034198761, |
| "logits/rejected": 0.4008842408657074, |
| "logps/chosen": -368.744873046875, |
| "logps/rejected": -584.6207885742188, |
| "loss": 0.0183, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.053328514099121, |
| "rewards/margins": 16.808618545532227, |
| "rewards/rejected": -20.861948013305664, |
| "step": 1890 |
| }, |
| { |
| "epoch": 1.22, |
| "learning_rate": 4.4251368721732916e-07, |
| "logits/chosen": -0.10556875169277191, |
| "logits/rejected": 0.6400087475776672, |
| "logps/chosen": -395.7705993652344, |
| "logps/rejected": -568.189697265625, |
| "loss": 0.0327, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.5353851318359375, |
| "rewards/margins": 15.502912521362305, |
| "rewards/rejected": -20.038297653198242, |
| "step": 1900 |
| }, |
| { |
| "epoch": 1.22, |
| "eval_logits/chosen": -0.3349679708480835, |
| "eval_logits/rejected": 0.1110042855143547, |
| "eval_logps/chosen": -412.0855712890625, |
| "eval_logps/rejected": -525.799560546875, |
| "eval_loss": 0.16337917745113373, |
| "eval_rewards/accuracies": 0.90625, |
| "eval_rewards/chosen": -8.27225112915039, |
| "eval_rewards/margins": 12.672636032104492, |
| "eval_rewards/rejected": -20.94488525390625, |
| "eval_runtime": 76.7623, |
| "eval_samples_per_second": 13.027, |
| "eval_steps_per_second": 0.417, |
| "step": 1900 |
| }, |
| { |
| "epoch": 1.23, |
| "learning_rate": 4.4191859081171146e-07, |
| "logits/chosen": 0.13517367839813232, |
| "logits/rejected": 0.3502965569496155, |
| "logps/chosen": -432.887451171875, |
| "logps/rejected": -610.2424926757812, |
| "loss": 0.0176, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.2362380027771, |
| "rewards/margins": 17.042871475219727, |
| "rewards/rejected": -22.279109954833984, |
| "step": 1910 |
| }, |
| { |
| "epoch": 1.23, |
| "learning_rate": 4.413234944060938e-07, |
| "logits/chosen": -0.011228932067751884, |
| "logits/rejected": 0.4079880714416504, |
| "logps/chosen": -397.6374206542969, |
| "logps/rejected": -609.8255615234375, |
| "loss": 0.0084, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.824807643890381, |
| "rewards/margins": 15.99610424041748, |
| "rewards/rejected": -19.820913314819336, |
| "step": 1920 |
| }, |
| { |
| "epoch": 1.24, |
| "learning_rate": 4.4072839800047605e-07, |
| "logits/chosen": -0.048855990171432495, |
| "logits/rejected": 0.2614760994911194, |
| "logps/chosen": -425.06182861328125, |
| "logps/rejected": -693.6744384765625, |
| "loss": 0.0236, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.437471866607666, |
| "rewards/margins": 17.300281524658203, |
| "rewards/rejected": -21.73775291442871, |
| "step": 1930 |
| }, |
| { |
| "epoch": 1.25, |
| "learning_rate": 4.4013330159485834e-07, |
| "logits/chosen": -0.6388794183731079, |
| "logits/rejected": -0.16498331725597382, |
| "logps/chosen": -455.348388671875, |
| "logps/rejected": -607.502685546875, |
| "loss": 0.0343, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.9607555866241455, |
| "rewards/margins": 16.582752227783203, |
| "rewards/rejected": -19.543508529663086, |
| "step": 1940 |
| }, |
| { |
| "epoch": 1.25, |
| "learning_rate": 4.3953820518924064e-07, |
| "logits/chosen": -0.7284079194068909, |
| "logits/rejected": -0.07122499495744705, |
| "logps/chosen": -416.238525390625, |
| "logps/rejected": -586.7880249023438, |
| "loss": 0.0142, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.067997932434082, |
| "rewards/margins": 16.282215118408203, |
| "rewards/rejected": -20.350215911865234, |
| "step": 1950 |
| }, |
| { |
| "epoch": 1.26, |
| "learning_rate": 4.3894310878362293e-07, |
| "logits/chosen": -0.7452508211135864, |
| "logits/rejected": 0.1260381042957306, |
| "logps/chosen": -449.06268310546875, |
| "logps/rejected": -529.99072265625, |
| "loss": 0.0258, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.9167428016662598, |
| "rewards/margins": 15.735280990600586, |
| "rewards/rejected": -18.65202522277832, |
| "step": 1960 |
| }, |
| { |
| "epoch": 1.27, |
| "learning_rate": 4.3834801237800523e-07, |
| "logits/chosen": -0.20991234481334686, |
| "logits/rejected": 0.1470331847667694, |
| "logps/chosen": -391.4617919921875, |
| "logps/rejected": -590.724365234375, |
| "loss": 0.0332, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.234528541564941, |
| "rewards/margins": 17.127511978149414, |
| "rewards/rejected": -21.362041473388672, |
| "step": 1970 |
| }, |
| { |
| "epoch": 1.27, |
| "learning_rate": 4.377529159723875e-07, |
| "logits/chosen": -0.1375136375427246, |
| "logits/rejected": 0.26706498861312866, |
| "logps/chosen": -444.4725646972656, |
| "logps/rejected": -529.6869506835938, |
| "loss": 0.0262, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.4430928230285645, |
| "rewards/margins": 15.911623001098633, |
| "rewards/rejected": -21.354717254638672, |
| "step": 1980 |
| }, |
| { |
| "epoch": 1.28, |
| "learning_rate": 4.3715781956676976e-07, |
| "logits/chosen": -0.21475636959075928, |
| "logits/rejected": 0.3777112364768982, |
| "logps/chosen": -406.0322570800781, |
| "logps/rejected": -613.7669067382812, |
| "loss": 0.0202, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.220062732696533, |
| "rewards/margins": 17.0797119140625, |
| "rewards/rejected": -21.299776077270508, |
| "step": 1990 |
| }, |
| { |
| "epoch": 1.29, |
| "learning_rate": 4.365627231611521e-07, |
| "logits/chosen": -0.42644554376602173, |
| "logits/rejected": 0.14130757749080658, |
| "logps/chosen": -414.5654296875, |
| "logps/rejected": -642.5784912109375, |
| "loss": 0.0182, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.209155321121216, |
| "rewards/margins": 17.73525619506836, |
| "rewards/rejected": -20.944412231445312, |
| "step": 2000 |
| }, |
| { |
| "epoch": 1.29, |
| "eval_logits/chosen": -0.6391128301620483, |
| "eval_logits/rejected": -0.05341078341007233, |
| "eval_logps/chosen": -384.2879638671875, |
| "eval_logps/rejected": -496.6387023925781, |
| "eval_loss": 0.15107305347919464, |
| "eval_rewards/accuracies": 0.953125, |
| "eval_rewards/chosen": -5.492494106292725, |
| "eval_rewards/margins": 12.536298751831055, |
| "eval_rewards/rejected": -18.028793334960938, |
| "eval_runtime": 76.6391, |
| "eval_samples_per_second": 13.048, |
| "eval_steps_per_second": 0.418, |
| "step": 2000 |
| }, |
| { |
| "epoch": 1.29, |
| "learning_rate": 4.359676267555344e-07, |
| "logits/chosen": -0.516070544719696, |
| "logits/rejected": 0.47864165902137756, |
| "logps/chosen": -344.65240478515625, |
| "logps/rejected": -633.1707763671875, |
| "loss": 0.0116, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.215503692626953, |
| "rewards/margins": 20.52490234375, |
| "rewards/rejected": -24.740406036376953, |
| "step": 2010 |
| }, |
| { |
| "epoch": 1.3, |
| "learning_rate": 4.3537253034991665e-07, |
| "logits/chosen": -0.2483837604522705, |
| "logits/rejected": 0.22826921939849854, |
| "logps/chosen": -390.77874755859375, |
| "logps/rejected": -561.1676025390625, |
| "loss": 0.0096, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.410299777984619, |
| "rewards/margins": 18.708133697509766, |
| "rewards/rejected": -23.11842918395996, |
| "step": 2020 |
| }, |
| { |
| "epoch": 1.3, |
| "learning_rate": 4.3477743394429894e-07, |
| "logits/chosen": -0.058499228209257126, |
| "logits/rejected": 0.5287091135978699, |
| "logps/chosen": -399.7431640625, |
| "logps/rejected": -509.27593994140625, |
| "loss": 0.0097, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.519265651702881, |
| "rewards/margins": 16.500688552856445, |
| "rewards/rejected": -21.01995277404785, |
| "step": 2030 |
| }, |
| { |
| "epoch": 1.31, |
| "learning_rate": 4.3418233753868124e-07, |
| "logits/chosen": 0.22348590195178986, |
| "logits/rejected": 0.7664919495582581, |
| "logps/chosen": -411.91668701171875, |
| "logps/rejected": -567.1552124023438, |
| "loss": 0.0393, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.37440299987793, |
| "rewards/margins": 16.45658302307129, |
| "rewards/rejected": -21.830982208251953, |
| "step": 2040 |
| }, |
| { |
| "epoch": 1.32, |
| "learning_rate": 4.3358724113306353e-07, |
| "logits/chosen": 0.23079347610473633, |
| "logits/rejected": 0.849484920501709, |
| "logps/chosen": -399.9824523925781, |
| "logps/rejected": -545.28271484375, |
| "loss": 0.0132, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.708091735839844, |
| "rewards/margins": 18.776620864868164, |
| "rewards/rejected": -24.484712600708008, |
| "step": 2050 |
| }, |
| { |
| "epoch": 1.32, |
| "learning_rate": 4.3299214472744583e-07, |
| "logits/chosen": 0.5551694631576538, |
| "logits/rejected": 1.1838432550430298, |
| "logps/chosen": -406.03460693359375, |
| "logps/rejected": -533.4246826171875, |
| "loss": 0.0198, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.767632961273193, |
| "rewards/margins": 15.5044527053833, |
| "rewards/rejected": -22.27208709716797, |
| "step": 2060 |
| }, |
| { |
| "epoch": 1.33, |
| "learning_rate": 4.323970483218281e-07, |
| "logits/chosen": 0.5585372447967529, |
| "logits/rejected": 1.1072343587875366, |
| "logps/chosen": -390.73870849609375, |
| "logps/rejected": -581.7123413085938, |
| "loss": 0.0109, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.473759651184082, |
| "rewards/margins": 17.37258529663086, |
| "rewards/rejected": -22.846345901489258, |
| "step": 2070 |
| }, |
| { |
| "epoch": 1.34, |
| "learning_rate": 4.3180195191621036e-07, |
| "logits/chosen": 0.19852328300476074, |
| "logits/rejected": 0.6014672517776489, |
| "logps/chosen": -408.9750061035156, |
| "logps/rejected": -573.5753784179688, |
| "loss": 0.0317, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.9805541038513184, |
| "rewards/margins": 16.70760154724121, |
| "rewards/rejected": -20.688152313232422, |
| "step": 2080 |
| }, |
| { |
| "epoch": 1.34, |
| "learning_rate": 4.312068555105927e-07, |
| "logits/chosen": 0.2863103151321411, |
| "logits/rejected": 1.09657883644104, |
| "logps/chosen": -437.265380859375, |
| "logps/rejected": -549.7987060546875, |
| "loss": 0.013, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.587510108947754, |
| "rewards/margins": 17.130268096923828, |
| "rewards/rejected": -21.7177791595459, |
| "step": 2090 |
| }, |
| { |
| "epoch": 1.35, |
| "learning_rate": 4.30611759104975e-07, |
| "logits/chosen": 0.16401129961013794, |
| "logits/rejected": 0.43381816148757935, |
| "logps/chosen": -367.1972351074219, |
| "logps/rejected": -574.0045776367188, |
| "loss": 0.0181, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.683870315551758, |
| "rewards/margins": 18.48448944091797, |
| "rewards/rejected": -22.168359756469727, |
| "step": 2100 |
| }, |
| { |
| "epoch": 1.35, |
| "eval_logits/chosen": -0.38096773624420166, |
| "eval_logits/rejected": 0.2048163115978241, |
| "eval_logps/chosen": -379.0364074707031, |
| "eval_logps/rejected": -488.2851867675781, |
| "eval_loss": 0.16009199619293213, |
| "eval_rewards/accuracies": 0.9140625, |
| "eval_rewards/chosen": -4.9673357009887695, |
| "eval_rewards/margins": 12.22611141204834, |
| "eval_rewards/rejected": -17.19344711303711, |
| "eval_runtime": 77.0334, |
| "eval_samples_per_second": 12.981, |
| "eval_steps_per_second": 0.415, |
| "step": 2100 |
| }, |
| { |
| "epoch": 1.36, |
| "learning_rate": 4.3001666269935725e-07, |
| "logits/chosen": 0.4979768395423889, |
| "logits/rejected": 0.9932268261909485, |
| "logps/chosen": -439.8809509277344, |
| "logps/rejected": -572.9110107421875, |
| "loss": 0.0196, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.661294937133789, |
| "rewards/margins": 16.660076141357422, |
| "rewards/rejected": -21.32137107849121, |
| "step": 2110 |
| }, |
| { |
| "epoch": 1.36, |
| "learning_rate": 4.2942156629373954e-07, |
| "logits/chosen": 0.38851994276046753, |
| "logits/rejected": 1.0913450717926025, |
| "logps/chosen": -391.9513854980469, |
| "logps/rejected": -557.0726318359375, |
| "loss": 0.0222, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.788923263549805, |
| "rewards/margins": 16.37869644165039, |
| "rewards/rejected": -21.167619705200195, |
| "step": 2120 |
| }, |
| { |
| "epoch": 1.37, |
| "learning_rate": 4.288264698881219e-07, |
| "logits/chosen": 0.7406013607978821, |
| "logits/rejected": 1.1288832426071167, |
| "logps/chosen": -434.83880615234375, |
| "logps/rejected": -587.9083251953125, |
| "loss": 0.0055, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.380326271057129, |
| "rewards/margins": 18.681997299194336, |
| "rewards/rejected": -24.06232261657715, |
| "step": 2130 |
| }, |
| { |
| "epoch": 1.38, |
| "learning_rate": 4.2823137348250413e-07, |
| "logits/chosen": 0.521850049495697, |
| "logits/rejected": 0.9274239540100098, |
| "logps/chosen": -380.3211669921875, |
| "logps/rejected": -581.7938232421875, |
| "loss": 0.0179, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.798120021820068, |
| "rewards/margins": 17.523136138916016, |
| "rewards/rejected": -22.321256637573242, |
| "step": 2140 |
| }, |
| { |
| "epoch": 1.38, |
| "learning_rate": 4.2763627707688643e-07, |
| "logits/chosen": 0.12944528460502625, |
| "logits/rejected": 0.8914273381233215, |
| "logps/chosen": -443.59954833984375, |
| "logps/rejected": -575.8558349609375, |
| "loss": 0.0128, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.318188190460205, |
| "rewards/margins": 17.411693572998047, |
| "rewards/rejected": -21.729881286621094, |
| "step": 2150 |
| }, |
| { |
| "epoch": 1.39, |
| "learning_rate": 4.270411806712687e-07, |
| "logits/chosen": -0.12583580613136292, |
| "logits/rejected": 0.797269344329834, |
| "logps/chosen": -445.8939514160156, |
| "logps/rejected": -576.5032958984375, |
| "loss": 0.0453, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.454132080078125, |
| "rewards/margins": 17.303661346435547, |
| "rewards/rejected": -23.757795333862305, |
| "step": 2160 |
| }, |
| { |
| "epoch": 1.39, |
| "learning_rate": 4.26446084265651e-07, |
| "logits/chosen": -0.158376082777977, |
| "logits/rejected": 0.6286741495132446, |
| "logps/chosen": -425.6141662597656, |
| "logps/rejected": -587.5155029296875, |
| "loss": 0.004, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -6.210710048675537, |
| "rewards/margins": 16.975194931030273, |
| "rewards/rejected": -23.18590545654297, |
| "step": 2170 |
| }, |
| { |
| "epoch": 1.4, |
| "learning_rate": 4.258509878600333e-07, |
| "logits/chosen": -0.22286149859428406, |
| "logits/rejected": 0.8965922594070435, |
| "logps/chosen": -482.9183654785156, |
| "logps/rejected": -620.5008544921875, |
| "loss": 0.0323, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -7.59103536605835, |
| "rewards/margins": 16.49752426147461, |
| "rewards/rejected": -24.088560104370117, |
| "step": 2180 |
| }, |
| { |
| "epoch": 1.41, |
| "learning_rate": 4.252558914544156e-07, |
| "logits/chosen": 0.22360090911388397, |
| "logits/rejected": 0.7913219928741455, |
| "logps/chosen": -467.44757080078125, |
| "logps/rejected": -694.8656616210938, |
| "loss": 0.0094, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -8.326881408691406, |
| "rewards/margins": 19.117225646972656, |
| "rewards/rejected": -27.444107055664062, |
| "step": 2190 |
| }, |
| { |
| "epoch": 1.41, |
| "learning_rate": 4.2466079504879785e-07, |
| "logits/chosen": -0.1699143350124359, |
| "logits/rejected": 0.9544417262077332, |
| "logps/chosen": -444.05224609375, |
| "logps/rejected": -574.4688110351562, |
| "loss": 0.0107, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -7.666506767272949, |
| "rewards/margins": 15.971659660339355, |
| "rewards/rejected": -23.638164520263672, |
| "step": 2200 |
| }, |
| { |
| "epoch": 1.41, |
| "eval_logits/chosen": -0.5077850222587585, |
| "eval_logits/rejected": 0.1793254315853119, |
| "eval_logps/chosen": -393.3812561035156, |
| "eval_logps/rejected": -497.8894958496094, |
| "eval_loss": 0.1433764100074768, |
| "eval_rewards/accuracies": 0.9296875, |
| "eval_rewards/chosen": -6.401820182800293, |
| "eval_rewards/margins": 11.752058029174805, |
| "eval_rewards/rejected": -18.15387725830078, |
| "eval_runtime": 76.5649, |
| "eval_samples_per_second": 13.061, |
| "eval_steps_per_second": 0.418, |
| "step": 2200 |
| }, |
| { |
| "epoch": 1.42, |
| "learning_rate": 4.240656986431802e-07, |
| "logits/chosen": 0.25252458453178406, |
| "logits/rejected": 0.6517130136489868, |
| "logps/chosen": -439.65478515625, |
| "logps/rejected": -589.1119995117188, |
| "loss": 0.023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.0924177169799805, |
| "rewards/margins": 17.32897186279297, |
| "rewards/rejected": -22.421390533447266, |
| "step": 2210 |
| }, |
| { |
| "epoch": 1.43, |
| "learning_rate": 4.234706022375625e-07, |
| "logits/chosen": 0.16541361808776855, |
| "logits/rejected": 0.7442089319229126, |
| "logps/chosen": -340.91363525390625, |
| "logps/rejected": -542.9158935546875, |
| "loss": 0.0378, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.545586585998535, |
| "rewards/margins": 16.971027374267578, |
| "rewards/rejected": -23.51661491394043, |
| "step": 2220 |
| }, |
| { |
| "epoch": 1.43, |
| "learning_rate": 4.2287550583194473e-07, |
| "logits/chosen": 0.12309785187244415, |
| "logits/rejected": 0.7068944573402405, |
| "logps/chosen": -416.27166748046875, |
| "logps/rejected": -671.1712646484375, |
| "loss": 0.0373, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.756946563720703, |
| "rewards/margins": 19.396137237548828, |
| "rewards/rejected": -26.1530818939209, |
| "step": 2230 |
| }, |
| { |
| "epoch": 1.44, |
| "learning_rate": 4.2228040942632703e-07, |
| "logits/chosen": -0.17779667675495148, |
| "logits/rejected": 0.7514945268630981, |
| "logps/chosen": -387.315185546875, |
| "logps/rejected": -554.4407348632812, |
| "loss": 0.0103, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -6.591644287109375, |
| "rewards/margins": 16.485776901245117, |
| "rewards/rejected": -23.07741928100586, |
| "step": 2240 |
| }, |
| { |
| "epoch": 1.45, |
| "learning_rate": 4.216853130207093e-07, |
| "logits/chosen": -0.4565064311027527, |
| "logits/rejected": 0.8017956018447876, |
| "logps/chosen": -407.85028076171875, |
| "logps/rejected": -557.0631103515625, |
| "loss": 0.0244, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.639117240905762, |
| "rewards/margins": 16.890506744384766, |
| "rewards/rejected": -21.529621124267578, |
| "step": 2250 |
| }, |
| { |
| "epoch": 1.45, |
| "learning_rate": 4.210902166150916e-07, |
| "logits/chosen": -0.1683514416217804, |
| "logits/rejected": 0.797057569026947, |
| "logps/chosen": -448.7789611816406, |
| "logps/rejected": -617.1752319335938, |
| "loss": 0.0067, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.5801100730896, |
| "rewards/margins": 17.799306869506836, |
| "rewards/rejected": -24.379417419433594, |
| "step": 2260 |
| }, |
| { |
| "epoch": 1.46, |
| "learning_rate": 4.204951202094739e-07, |
| "logits/chosen": 0.12932386994361877, |
| "logits/rejected": 0.9783223867416382, |
| "logps/chosen": -392.2380676269531, |
| "logps/rejected": -563.818603515625, |
| "loss": 0.0214, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.884858131408691, |
| "rewards/margins": 17.019224166870117, |
| "rewards/rejected": -22.904081344604492, |
| "step": 2270 |
| }, |
| { |
| "epoch": 1.47, |
| "learning_rate": 4.199000238038562e-07, |
| "logits/chosen": 0.17704737186431885, |
| "logits/rejected": 0.9605581164360046, |
| "logps/chosen": -393.4834289550781, |
| "logps/rejected": -520.2174072265625, |
| "loss": 0.0254, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.810626983642578, |
| "rewards/margins": 17.23440170288086, |
| "rewards/rejected": -22.045028686523438, |
| "step": 2280 |
| }, |
| { |
| "epoch": 1.47, |
| "learning_rate": 4.1930492739823845e-07, |
| "logits/chosen": 0.23696310818195343, |
| "logits/rejected": 0.8548523783683777, |
| "logps/chosen": -413.873046875, |
| "logps/rejected": -593.3323974609375, |
| "loss": 0.0362, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.712353229522705, |
| "rewards/margins": 18.40825653076172, |
| "rewards/rejected": -24.120609283447266, |
| "step": 2290 |
| }, |
| { |
| "epoch": 1.48, |
| "learning_rate": 4.187098309926208e-07, |
| "logits/chosen": 0.11162440478801727, |
| "logits/rejected": 0.8643198013305664, |
| "logps/chosen": -446.72216796875, |
| "logps/rejected": -609.0733642578125, |
| "loss": 0.0207, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -7.024069786071777, |
| "rewards/margins": 17.165082931518555, |
| "rewards/rejected": -24.18915367126465, |
| "step": 2300 |
| }, |
| { |
| "epoch": 1.48, |
| "eval_logits/chosen": -0.3969336152076721, |
| "eval_logits/rejected": 0.30485790967941284, |
| "eval_logps/chosen": -409.54925537109375, |
| "eval_logps/rejected": -527.1227416992188, |
| "eval_loss": 0.1328170895576477, |
| "eval_rewards/accuracies": 0.9609375, |
| "eval_rewards/chosen": -8.018616676330566, |
| "eval_rewards/margins": 13.058581352233887, |
| "eval_rewards/rejected": -21.077199935913086, |
| "eval_runtime": 76.7293, |
| "eval_samples_per_second": 13.033, |
| "eval_steps_per_second": 0.417, |
| "step": 2300 |
| }, |
| { |
| "epoch": 1.48, |
| "learning_rate": 4.181147345870031e-07, |
| "logits/chosen": -0.14238300919532776, |
| "logits/rejected": 0.5450645685195923, |
| "logps/chosen": -449.2244567871094, |
| "logps/rejected": -655.3807373046875, |
| "loss": 0.0182, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.570483207702637, |
| "rewards/margins": 16.494831085205078, |
| "rewards/rejected": -23.065311431884766, |
| "step": 2310 |
| }, |
| { |
| "epoch": 1.49, |
| "learning_rate": 4.1751963818138534e-07, |
| "logits/chosen": -0.011528102681040764, |
| "logits/rejected": 0.896866500377655, |
| "logps/chosen": -477.01226806640625, |
| "logps/rejected": -661.6298217773438, |
| "loss": 0.0098, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.549506187438965, |
| "rewards/margins": 19.142070770263672, |
| "rewards/rejected": -25.691574096679688, |
| "step": 2320 |
| }, |
| { |
| "epoch": 1.5, |
| "learning_rate": 4.1692454177576763e-07, |
| "logits/chosen": -0.2801293432712555, |
| "logits/rejected": 0.5617603063583374, |
| "logps/chosen": -442.40167236328125, |
| "logps/rejected": -573.85205078125, |
| "loss": 0.0289, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.618128776550293, |
| "rewards/margins": 17.062252044677734, |
| "rewards/rejected": -23.680381774902344, |
| "step": 2330 |
| }, |
| { |
| "epoch": 1.5, |
| "learning_rate": 4.1632944537015e-07, |
| "logits/chosen": -0.1246495246887207, |
| "logits/rejected": 0.732117772102356, |
| "logps/chosen": -445.9453125, |
| "logps/rejected": -648.7326049804688, |
| "loss": 0.0567, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -7.793832302093506, |
| "rewards/margins": 16.911283493041992, |
| "rewards/rejected": -24.705114364624023, |
| "step": 2340 |
| }, |
| { |
| "epoch": 1.51, |
| "learning_rate": 4.157343489645322e-07, |
| "logits/chosen": -0.13531716167926788, |
| "logits/rejected": 0.6813434362411499, |
| "logps/chosen": -500.760986328125, |
| "logps/rejected": -748.8970947265625, |
| "loss": 0.0249, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -8.698297500610352, |
| "rewards/margins": 22.551307678222656, |
| "rewards/rejected": -31.249608993530273, |
| "step": 2350 |
| }, |
| { |
| "epoch": 1.52, |
| "learning_rate": 4.151392525589145e-07, |
| "logits/chosen": -0.17930714786052704, |
| "logits/rejected": 0.5675514340400696, |
| "logps/chosen": -507.093994140625, |
| "logps/rejected": -617.2818603515625, |
| "loss": 0.0077, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -9.938199043273926, |
| "rewards/margins": 18.702774047851562, |
| "rewards/rejected": -28.640972137451172, |
| "step": 2360 |
| }, |
| { |
| "epoch": 1.52, |
| "learning_rate": 4.145441561532968e-07, |
| "logits/chosen": -0.02535531297326088, |
| "logits/rejected": 0.5533861517906189, |
| "logps/chosen": -441.52362060546875, |
| "logps/rejected": -658.696044921875, |
| "loss": 0.0219, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -7.094923496246338, |
| "rewards/margins": 19.90808868408203, |
| "rewards/rejected": -27.00301170349121, |
| "step": 2370 |
| }, |
| { |
| "epoch": 1.53, |
| "learning_rate": 4.139490597476791e-07, |
| "logits/chosen": -0.01725180074572563, |
| "logits/rejected": 0.545805811882019, |
| "logps/chosen": -431.83197021484375, |
| "logps/rejected": -565.3300170898438, |
| "loss": 0.0116, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.328795909881592, |
| "rewards/margins": 17.926631927490234, |
| "rewards/rejected": -22.255428314208984, |
| "step": 2380 |
| }, |
| { |
| "epoch": 1.54, |
| "learning_rate": 4.133539633420614e-07, |
| "logits/chosen": -0.21750584244728088, |
| "logits/rejected": 0.8260287046432495, |
| "logps/chosen": -426.7081604003906, |
| "logps/rejected": -571.2562255859375, |
| "loss": 0.0139, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.599783897399902, |
| "rewards/margins": 18.57124900817871, |
| "rewards/rejected": -23.17103385925293, |
| "step": 2390 |
| }, |
| { |
| "epoch": 1.54, |
| "learning_rate": 4.127588669364437e-07, |
| "logits/chosen": 0.12132026255130768, |
| "logits/rejected": 0.586942195892334, |
| "logps/chosen": -387.7074890136719, |
| "logps/rejected": -575.88623046875, |
| "loss": 0.0237, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.400722980499268, |
| "rewards/margins": 19.196186065673828, |
| "rewards/rejected": -24.596908569335938, |
| "step": 2400 |
| }, |
| { |
| "epoch": 1.54, |
| "eval_logits/chosen": -0.4777054190635681, |
| "eval_logits/rejected": 0.22216691076755524, |
| "eval_logps/chosen": -392.7734069824219, |
| "eval_logps/rejected": -515.349365234375, |
| "eval_loss": 0.1322525590658188, |
| "eval_rewards/accuracies": 0.9765625, |
| "eval_rewards/chosen": -6.341035842895508, |
| "eval_rewards/margins": 13.558823585510254, |
| "eval_rewards/rejected": -19.899858474731445, |
| "eval_runtime": 76.575, |
| "eval_samples_per_second": 13.059, |
| "eval_steps_per_second": 0.418, |
| "step": 2400 |
| }, |
| { |
| "epoch": 1.55, |
| "learning_rate": 4.1216377053082594e-07, |
| "logits/chosen": 0.19748568534851074, |
| "logits/rejected": 0.791664719581604, |
| "logps/chosen": -461.057861328125, |
| "logps/rejected": -645.6085205078125, |
| "loss": 0.0145, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -6.15015172958374, |
| "rewards/margins": 19.398887634277344, |
| "rewards/rejected": -25.549039840698242, |
| "step": 2410 |
| }, |
| { |
| "epoch": 1.56, |
| "learning_rate": 4.115686741252083e-07, |
| "logits/chosen": 0.5446011424064636, |
| "logits/rejected": 0.8287714123725891, |
| "logps/chosen": -428.01068115234375, |
| "logps/rejected": -596.6788330078125, |
| "loss": 0.0192, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.512643337249756, |
| "rewards/margins": 18.827342987060547, |
| "rewards/rejected": -25.33998680114746, |
| "step": 2420 |
| }, |
| { |
| "epoch": 1.56, |
| "learning_rate": 4.109735777195906e-07, |
| "logits/chosen": 0.2738257646560669, |
| "logits/rejected": 0.6724745035171509, |
| "logps/chosen": -443.29852294921875, |
| "logps/rejected": -617.8185424804688, |
| "loss": 0.0311, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.934741497039795, |
| "rewards/margins": 18.175350189208984, |
| "rewards/rejected": -23.110088348388672, |
| "step": 2430 |
| }, |
| { |
| "epoch": 1.57, |
| "learning_rate": 4.103784813139728e-07, |
| "logits/chosen": -0.21786899864673615, |
| "logits/rejected": 0.508806049823761, |
| "logps/chosen": -386.5865173339844, |
| "logps/rejected": -540.3848876953125, |
| "loss": 0.011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.1630804538726807, |
| "rewards/margins": 18.559303283691406, |
| "rewards/rejected": -21.72238540649414, |
| "step": 2440 |
| }, |
| { |
| "epoch": 1.57, |
| "learning_rate": 4.097833849083551e-07, |
| "logits/chosen": 0.27359846234321594, |
| "logits/rejected": 0.8219190835952759, |
| "logps/chosen": -489.2318420410156, |
| "logps/rejected": -654.9146118164062, |
| "loss": 0.0167, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.5973005294799805, |
| "rewards/margins": 19.05999183654785, |
| "rewards/rejected": -25.65729331970215, |
| "step": 2450 |
| }, |
| { |
| "epoch": 1.58, |
| "learning_rate": 4.091882885027374e-07, |
| "logits/chosen": 0.23141400516033173, |
| "logits/rejected": 0.8404422998428345, |
| "logps/chosen": -470.26300048828125, |
| "logps/rejected": -647.7288208007812, |
| "loss": 0.0143, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.572993278503418, |
| "rewards/margins": 19.47740364074707, |
| "rewards/rejected": -25.050395965576172, |
| "step": 2460 |
| }, |
| { |
| "epoch": 1.59, |
| "learning_rate": 4.0859319209711976e-07, |
| "logits/chosen": 0.21321940422058105, |
| "logits/rejected": 0.9413207769393921, |
| "logps/chosen": -481.8892517089844, |
| "logps/rejected": -568.3132934570312, |
| "loss": 0.0106, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.117605686187744, |
| "rewards/margins": 17.3239688873291, |
| "rewards/rejected": -22.441572189331055, |
| "step": 2470 |
| }, |
| { |
| "epoch": 1.59, |
| "learning_rate": 4.07998095691502e-07, |
| "logits/chosen": 0.34440717101097107, |
| "logits/rejected": 1.0420324802398682, |
| "logps/chosen": -412.06756591796875, |
| "logps/rejected": -661.225830078125, |
| "loss": 0.0085, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.927117347717285, |
| "rewards/margins": 19.294017791748047, |
| "rewards/rejected": -25.22113609313965, |
| "step": 2480 |
| }, |
| { |
| "epoch": 1.6, |
| "learning_rate": 4.074029992858843e-07, |
| "logits/chosen": 0.617647647857666, |
| "logits/rejected": 1.0356745719909668, |
| "logps/chosen": -466.6419372558594, |
| "logps/rejected": -593.55517578125, |
| "loss": 0.027, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.356731414794922, |
| "rewards/margins": 18.503705978393555, |
| "rewards/rejected": -23.860441207885742, |
| "step": 2490 |
| }, |
| { |
| "epoch": 1.61, |
| "learning_rate": 4.0680790288026654e-07, |
| "logits/chosen": 0.6595112681388855, |
| "logits/rejected": 1.4305124282836914, |
| "logps/chosen": -434.178466796875, |
| "logps/rejected": -573.2113647460938, |
| "loss": 0.0269, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.551410675048828, |
| "rewards/margins": 15.67224407196045, |
| "rewards/rejected": -22.223651885986328, |
| "step": 2500 |
| }, |
| { |
| "epoch": 1.61, |
| "eval_logits/chosen": -0.08062884956598282, |
| "eval_logits/rejected": 0.4895774722099304, |
| "eval_logps/chosen": -393.9366149902344, |
| "eval_logps/rejected": -509.43963623046875, |
| "eval_loss": 0.151441290974617, |
| "eval_rewards/accuracies": 0.96875, |
| "eval_rewards/chosen": -6.457356929779053, |
| "eval_rewards/margins": 12.851531982421875, |
| "eval_rewards/rejected": -19.308889389038086, |
| "eval_runtime": 76.6294, |
| "eval_samples_per_second": 13.05, |
| "eval_steps_per_second": 0.418, |
| "step": 2500 |
| }, |
| { |
| "epoch": 1.61, |
| "learning_rate": 4.062128064746489e-07, |
| "logits/chosen": 0.7047534584999084, |
| "logits/rejected": 1.2956479787826538, |
| "logps/chosen": -373.82366943359375, |
| "logps/rejected": -593.6276245117188, |
| "loss": 0.0182, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.4311394691467285, |
| "rewards/margins": 15.736666679382324, |
| "rewards/rejected": -22.167804718017578, |
| "step": 2510 |
| }, |
| { |
| "epoch": 1.62, |
| "learning_rate": 4.056177100690312e-07, |
| "logits/chosen": 0.23054015636444092, |
| "logits/rejected": 0.9308466911315918, |
| "logps/chosen": -403.5622253417969, |
| "logps/rejected": -522.812744140625, |
| "loss": 0.03, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -7.561907768249512, |
| "rewards/margins": 15.371617317199707, |
| "rewards/rejected": -22.933523178100586, |
| "step": 2520 |
| }, |
| { |
| "epoch": 1.63, |
| "learning_rate": 4.050226136634135e-07, |
| "logits/chosen": 0.061969585716724396, |
| "logits/rejected": 0.7286633253097534, |
| "logps/chosen": -414.0792541503906, |
| "logps/rejected": -600.1736450195312, |
| "loss": 0.0162, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.574122428894043, |
| "rewards/margins": 19.657848358154297, |
| "rewards/rejected": -25.231969833374023, |
| "step": 2530 |
| }, |
| { |
| "epoch": 1.63, |
| "learning_rate": 4.044275172577957e-07, |
| "logits/chosen": 0.05762529373168945, |
| "logits/rejected": 1.0145411491394043, |
| "logps/chosen": -445.4334411621094, |
| "logps/rejected": -645.9708251953125, |
| "loss": 0.0138, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -7.179165840148926, |
| "rewards/margins": 19.949363708496094, |
| "rewards/rejected": -27.128530502319336, |
| "step": 2540 |
| }, |
| { |
| "epoch": 1.64, |
| "learning_rate": 4.0383242085217806e-07, |
| "logits/chosen": 0.2469167709350586, |
| "logits/rejected": 0.8062774538993835, |
| "logps/chosen": -454.03668212890625, |
| "logps/rejected": -624.8360595703125, |
| "loss": 0.0212, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.777960777282715, |
| "rewards/margins": 17.905128479003906, |
| "rewards/rejected": -24.68309211730957, |
| "step": 2550 |
| }, |
| { |
| "epoch": 1.65, |
| "learning_rate": 4.0323732444656036e-07, |
| "logits/chosen": 0.52375328540802, |
| "logits/rejected": 1.2141704559326172, |
| "logps/chosen": -423.7889099121094, |
| "logps/rejected": -609.6292114257812, |
| "loss": 0.0147, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -6.806221961975098, |
| "rewards/margins": 18.522531509399414, |
| "rewards/rejected": -25.328754425048828, |
| "step": 2560 |
| }, |
| { |
| "epoch": 1.65, |
| "learning_rate": 4.026422280409426e-07, |
| "logits/chosen": 0.5363117456436157, |
| "logits/rejected": 0.9718011617660522, |
| "logps/chosen": -423.1117248535156, |
| "logps/rejected": -559.2742309570312, |
| "loss": 0.0108, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.507771015167236, |
| "rewards/margins": 17.65149688720703, |
| "rewards/rejected": -23.159269332885742, |
| "step": 2570 |
| }, |
| { |
| "epoch": 1.66, |
| "learning_rate": 4.020471316353249e-07, |
| "logits/chosen": 0.7948285341262817, |
| "logits/rejected": 1.6965904235839844, |
| "logps/chosen": -450.24945068359375, |
| "logps/rejected": -587.7679443359375, |
| "loss": 0.0174, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -7.323611259460449, |
| "rewards/margins": 19.197834014892578, |
| "rewards/rejected": -26.521448135375977, |
| "step": 2580 |
| }, |
| { |
| "epoch": 1.66, |
| "learning_rate": 4.0145203522970724e-07, |
| "logits/chosen": 0.6111919283866882, |
| "logits/rejected": 1.2076199054718018, |
| "logps/chosen": -393.1330871582031, |
| "logps/rejected": -658.733154296875, |
| "loss": 0.0212, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -6.268210411071777, |
| "rewards/margins": 22.285709381103516, |
| "rewards/rejected": -28.553918838500977, |
| "step": 2590 |
| }, |
| { |
| "epoch": 1.67, |
| "learning_rate": 4.008569388240895e-07, |
| "logits/chosen": 0.8838823437690735, |
| "logits/rejected": 1.2930363416671753, |
| "logps/chosen": -433.79248046875, |
| "logps/rejected": -662.343994140625, |
| "loss": 0.0183, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -6.181797027587891, |
| "rewards/margins": 20.533702850341797, |
| "rewards/rejected": -26.715499877929688, |
| "step": 2600 |
| }, |
| { |
| "epoch": 1.67, |
| "eval_logits/chosen": 0.019558865576982498, |
| "eval_logits/rejected": 0.6054279804229736, |
| "eval_logps/chosen": -407.05767822265625, |
| "eval_logps/rejected": -531.6161499023438, |
| "eval_loss": 0.177625834941864, |
| "eval_rewards/accuracies": 0.9375, |
| "eval_rewards/chosen": -7.769463539123535, |
| "eval_rewards/margins": 13.75707721710205, |
| "eval_rewards/rejected": -21.526540756225586, |
| "eval_runtime": 76.8929, |
| "eval_samples_per_second": 13.005, |
| "eval_steps_per_second": 0.416, |
| "step": 2600 |
| }, |
| { |
| "epoch": 1.68, |
| "learning_rate": 4.002618424184718e-07, |
| "logits/chosen": 0.37892434000968933, |
| "logits/rejected": 0.9920595288276672, |
| "logps/chosen": -391.6972351074219, |
| "logps/rejected": -585.7908935546875, |
| "loss": 0.0218, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.025783538818359, |
| "rewards/margins": 17.12643814086914, |
| "rewards/rejected": -22.1522216796875, |
| "step": 2610 |
| }, |
| { |
| "epoch": 1.68, |
| "learning_rate": 3.996667460128541e-07, |
| "logits/chosen": 0.08298696577548981, |
| "logits/rejected": 0.9364360570907593, |
| "logps/chosen": -374.4390563964844, |
| "logps/rejected": -576.79541015625, |
| "loss": 0.0334, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.216673851013184, |
| "rewards/margins": 18.307117462158203, |
| "rewards/rejected": -23.523792266845703, |
| "step": 2620 |
| }, |
| { |
| "epoch": 1.69, |
| "learning_rate": 3.990716496072363e-07, |
| "logits/chosen": 0.30024105310440063, |
| "logits/rejected": 0.9240363836288452, |
| "logps/chosen": -419.29510498046875, |
| "logps/rejected": -590.0726928710938, |
| "loss": 0.0232, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -6.619715690612793, |
| "rewards/margins": 17.221851348876953, |
| "rewards/rejected": -23.841564178466797, |
| "step": 2630 |
| }, |
| { |
| "epoch": 1.7, |
| "learning_rate": 3.9847655320161867e-07, |
| "logits/chosen": 0.2850233018398285, |
| "logits/rejected": 0.6386219263076782, |
| "logps/chosen": -397.0573425292969, |
| "logps/rejected": -577.4563598632812, |
| "loss": 0.0089, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.049335479736328, |
| "rewards/margins": 16.51738739013672, |
| "rewards/rejected": -22.56671905517578, |
| "step": 2640 |
| }, |
| { |
| "epoch": 1.7, |
| "learning_rate": 3.9788145679600096e-07, |
| "logits/chosen": 0.2091820240020752, |
| "logits/rejected": 0.9494321942329407, |
| "logps/chosen": -415.09869384765625, |
| "logps/rejected": -584.553466796875, |
| "loss": 0.0211, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.250141620635986, |
| "rewards/margins": 17.63558578491211, |
| "rewards/rejected": -22.88572883605957, |
| "step": 2650 |
| }, |
| { |
| "epoch": 1.71, |
| "learning_rate": 3.972863603903832e-07, |
| "logits/chosen": 0.15588752925395966, |
| "logits/rejected": 0.9101377725601196, |
| "logps/chosen": -440.03546142578125, |
| "logps/rejected": -556.76953125, |
| "loss": 0.0258, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.330071449279785, |
| "rewards/margins": 16.938446044921875, |
| "rewards/rejected": -23.26851463317871, |
| "step": 2660 |
| }, |
| { |
| "epoch": 1.72, |
| "learning_rate": 3.966912639847655e-07, |
| "logits/chosen": 0.43947821855545044, |
| "logits/rejected": 1.1439796686172485, |
| "logps/chosen": -408.59869384765625, |
| "logps/rejected": -561.8624267578125, |
| "loss": 0.0274, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.567991733551025, |
| "rewards/margins": 18.473587036132812, |
| "rewards/rejected": -25.041580200195312, |
| "step": 2670 |
| }, |
| { |
| "epoch": 1.72, |
| "learning_rate": 3.9609616757914784e-07, |
| "logits/chosen": 0.639157772064209, |
| "logits/rejected": 1.2126656770706177, |
| "logps/chosen": -433.49432373046875, |
| "logps/rejected": -545.498779296875, |
| "loss": 0.0441, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.019906044006348, |
| "rewards/margins": 16.54349708557129, |
| "rewards/rejected": -22.563400268554688, |
| "step": 2680 |
| }, |
| { |
| "epoch": 1.73, |
| "learning_rate": 3.955010711735301e-07, |
| "logits/chosen": 1.0705523490905762, |
| "logits/rejected": 1.574342966079712, |
| "logps/chosen": -402.82073974609375, |
| "logps/rejected": -556.3751831054688, |
| "loss": 0.0333, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.881136417388916, |
| "rewards/margins": 16.096803665161133, |
| "rewards/rejected": -22.977941513061523, |
| "step": 2690 |
| }, |
| { |
| "epoch": 1.74, |
| "learning_rate": 3.949059747679124e-07, |
| "logits/chosen": 0.6093908548355103, |
| "logits/rejected": 1.4320948123931885, |
| "logps/chosen": -416.2469177246094, |
| "logps/rejected": -563.9771118164062, |
| "loss": 0.0219, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.77686071395874, |
| "rewards/margins": 17.765539169311523, |
| "rewards/rejected": -23.54239845275879, |
| "step": 2700 |
| }, |
| { |
| "epoch": 1.74, |
| "eval_logits/chosen": -0.009464547038078308, |
| "eval_logits/rejected": 0.5373153686523438, |
| "eval_logps/chosen": -408.0279541015625, |
| "eval_logps/rejected": -531.8214721679688, |
| "eval_loss": 0.14935775101184845, |
| "eval_rewards/accuracies": 0.9296875, |
| "eval_rewards/chosen": -7.866490364074707, |
| "eval_rewards/margins": 13.680585861206055, |
| "eval_rewards/rejected": -21.547077178955078, |
| "eval_runtime": 76.6739, |
| "eval_samples_per_second": 13.042, |
| "eval_steps_per_second": 0.417, |
| "step": 2700 |
| }, |
| { |
| "epoch": 1.74, |
| "learning_rate": 3.943108783622947e-07, |
| "logits/chosen": 0.5636851787567139, |
| "logits/rejected": 0.969277024269104, |
| "logps/chosen": -465.9778747558594, |
| "logps/rejected": -580.30810546875, |
| "loss": 0.0097, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -7.8137688636779785, |
| "rewards/margins": 17.516647338867188, |
| "rewards/rejected": -25.330415725708008, |
| "step": 2710 |
| }, |
| { |
| "epoch": 1.75, |
| "learning_rate": 3.9371578195667697e-07, |
| "logits/chosen": 0.28570881485939026, |
| "logits/rejected": 1.0974262952804565, |
| "logps/chosen": -420.02520751953125, |
| "logps/rejected": -557.4937744140625, |
| "loss": 0.0242, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.895697116851807, |
| "rewards/margins": 18.642253875732422, |
| "rewards/rejected": -24.53795051574707, |
| "step": 2720 |
| }, |
| { |
| "epoch": 1.75, |
| "learning_rate": 3.9312068555105927e-07, |
| "logits/chosen": 0.2833688259124756, |
| "logits/rejected": 1.2028071880340576, |
| "logps/chosen": -428.30242919921875, |
| "logps/rejected": -601.4442138671875, |
| "loss": 0.022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.424818992614746, |
| "rewards/margins": 16.82640838623047, |
| "rewards/rejected": -22.2512264251709, |
| "step": 2730 |
| }, |
| { |
| "epoch": 1.76, |
| "learning_rate": 3.9252558914544156e-07, |
| "logits/chosen": -0.17283181846141815, |
| "logits/rejected": 0.6285573840141296, |
| "logps/chosen": -407.96063232421875, |
| "logps/rejected": -656.8048095703125, |
| "loss": 0.0084, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.4999213218688965, |
| "rewards/margins": 18.290767669677734, |
| "rewards/rejected": -23.790691375732422, |
| "step": 2740 |
| }, |
| { |
| "epoch": 1.77, |
| "learning_rate": 3.919304927398238e-07, |
| "logits/chosen": 0.1602792888879776, |
| "logits/rejected": 0.6874667406082153, |
| "logps/chosen": -375.77960205078125, |
| "logps/rejected": -546.3209228515625, |
| "loss": 0.0288, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.720765113830566, |
| "rewards/margins": 16.913997650146484, |
| "rewards/rejected": -22.634761810302734, |
| "step": 2750 |
| }, |
| { |
| "epoch": 1.77, |
| "learning_rate": 3.9133539633420615e-07, |
| "logits/chosen": 0.2454497367143631, |
| "logits/rejected": 0.6986191272735596, |
| "logps/chosen": -390.337890625, |
| "logps/rejected": -560.3748779296875, |
| "loss": 0.0155, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.7264814376831055, |
| "rewards/margins": 17.356287002563477, |
| "rewards/rejected": -23.082767486572266, |
| "step": 2760 |
| }, |
| { |
| "epoch": 1.78, |
| "learning_rate": 3.9074029992858845e-07, |
| "logits/chosen": 0.38854673504829407, |
| "logits/rejected": 1.452370047569275, |
| "logps/chosen": -415.73297119140625, |
| "logps/rejected": -565.9852294921875, |
| "loss": 0.016, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -6.159702777862549, |
| "rewards/margins": 17.43438148498535, |
| "rewards/rejected": -23.594083786010742, |
| "step": 2770 |
| }, |
| { |
| "epoch": 1.79, |
| "learning_rate": 3.901452035229707e-07, |
| "logits/chosen": 0.7071353197097778, |
| "logits/rejected": 1.089143991470337, |
| "logps/chosen": -402.5384216308594, |
| "logps/rejected": -598.9990234375, |
| "loss": 0.0203, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -7.571539402008057, |
| "rewards/margins": 18.129215240478516, |
| "rewards/rejected": -25.700754165649414, |
| "step": 2780 |
| }, |
| { |
| "epoch": 1.79, |
| "learning_rate": 3.89550107117353e-07, |
| "logits/chosen": 0.5019019842147827, |
| "logits/rejected": 1.2631019353866577, |
| "logps/chosen": -412.53607177734375, |
| "logps/rejected": -484.9837341308594, |
| "loss": 0.0231, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.363646984100342, |
| "rewards/margins": 15.107401847839355, |
| "rewards/rejected": -19.471050262451172, |
| "step": 2790 |
| }, |
| { |
| "epoch": 1.8, |
| "learning_rate": 3.8895501071173533e-07, |
| "logits/chosen": 0.5883782505989075, |
| "logits/rejected": 1.2746624946594238, |
| "logps/chosen": -431.44287109375, |
| "logps/rejected": -637.4125366210938, |
| "loss": 0.0335, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.525457859039307, |
| "rewards/margins": 19.470340728759766, |
| "rewards/rejected": -25.995798110961914, |
| "step": 2800 |
| }, |
| { |
| "epoch": 1.8, |
| "eval_logits/chosen": -0.053867146372795105, |
| "eval_logits/rejected": 0.6149381399154663, |
| "eval_logps/chosen": -404.7615051269531, |
| "eval_logps/rejected": -530.05126953125, |
| "eval_loss": 0.15537042915821075, |
| "eval_rewards/accuracies": 0.9609375, |
| "eval_rewards/chosen": -7.539842128753662, |
| "eval_rewards/margins": 13.830206871032715, |
| "eval_rewards/rejected": -21.37004852294922, |
| "eval_runtime": 76.7186, |
| "eval_samples_per_second": 13.035, |
| "eval_steps_per_second": 0.417, |
| "step": 2800 |
| }, |
| { |
| "epoch": 1.81, |
| "learning_rate": 3.8835991430611757e-07, |
| "logits/chosen": 0.6285505294799805, |
| "logits/rejected": 1.1924630403518677, |
| "logps/chosen": -426.795166015625, |
| "logps/rejected": -537.3714599609375, |
| "loss": 0.0291, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.4925079345703125, |
| "rewards/margins": 17.20037078857422, |
| "rewards/rejected": -23.69287872314453, |
| "step": 2810 |
| }, |
| { |
| "epoch": 1.81, |
| "learning_rate": 3.8776481790049987e-07, |
| "logits/chosen": 0.3737705945968628, |
| "logits/rejected": 1.088739275932312, |
| "logps/chosen": -472.11669921875, |
| "logps/rejected": -613.3414916992188, |
| "loss": 0.0219, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.82584285736084, |
| "rewards/margins": 20.265369415283203, |
| "rewards/rejected": -26.09120750427246, |
| "step": 2820 |
| }, |
| { |
| "epoch": 1.82, |
| "learning_rate": 3.8716972149488216e-07, |
| "logits/chosen": 0.4724366068840027, |
| "logits/rejected": 1.5313258171081543, |
| "logps/chosen": -474.59326171875, |
| "logps/rejected": -591.4876098632812, |
| "loss": 0.0295, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.544611930847168, |
| "rewards/margins": 20.092144012451172, |
| "rewards/rejected": -25.636754989624023, |
| "step": 2830 |
| }, |
| { |
| "epoch": 1.83, |
| "learning_rate": 3.865746250892644e-07, |
| "logits/chosen": 0.6960526704788208, |
| "logits/rejected": 1.0484097003936768, |
| "logps/chosen": -442.74298095703125, |
| "logps/rejected": -641.082763671875, |
| "loss": 0.0359, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.976676940917969, |
| "rewards/margins": 18.82603645324707, |
| "rewards/rejected": -24.802715301513672, |
| "step": 2840 |
| }, |
| { |
| "epoch": 1.83, |
| "learning_rate": 3.8597952868364675e-07, |
| "logits/chosen": 0.6734079122543335, |
| "logits/rejected": 1.2869551181793213, |
| "logps/chosen": -410.61846923828125, |
| "logps/rejected": -535.3711547851562, |
| "loss": 0.0256, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.3238630294799805, |
| "rewards/margins": 16.99979019165039, |
| "rewards/rejected": -22.323650360107422, |
| "step": 2850 |
| }, |
| { |
| "epoch": 1.84, |
| "learning_rate": 3.8538443227802905e-07, |
| "logits/chosen": 0.39041590690612793, |
| "logits/rejected": 1.0508239269256592, |
| "logps/chosen": -455.080322265625, |
| "logps/rejected": -550.6773681640625, |
| "loss": 0.0194, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.037131309509277, |
| "rewards/margins": 17.071130752563477, |
| "rewards/rejected": -21.10826301574707, |
| "step": 2860 |
| }, |
| { |
| "epoch": 1.84, |
| "learning_rate": 3.847893358724113e-07, |
| "logits/chosen": 0.21260789036750793, |
| "logits/rejected": 1.1434606313705444, |
| "logps/chosen": -434.0419006347656, |
| "logps/rejected": -555.1299438476562, |
| "loss": 0.0233, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.177916526794434, |
| "rewards/margins": 15.855340957641602, |
| "rewards/rejected": -21.03325843811035, |
| "step": 2870 |
| }, |
| { |
| "epoch": 1.85, |
| "learning_rate": 3.841942394667936e-07, |
| "logits/chosen": 0.26587316393852234, |
| "logits/rejected": 1.1289548873901367, |
| "logps/chosen": -388.0738830566406, |
| "logps/rejected": -598.3447265625, |
| "loss": 0.0343, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.688809394836426, |
| "rewards/margins": 18.783828735351562, |
| "rewards/rejected": -24.472637176513672, |
| "step": 2880 |
| }, |
| { |
| "epoch": 1.86, |
| "learning_rate": 3.8359914306117593e-07, |
| "logits/chosen": 0.9435780644416809, |
| "logits/rejected": 1.0387550592422485, |
| "logps/chosen": -420.2930603027344, |
| "logps/rejected": -602.9673461914062, |
| "loss": 0.0159, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.682863712310791, |
| "rewards/margins": 18.17510986328125, |
| "rewards/rejected": -24.857975006103516, |
| "step": 2890 |
| }, |
| { |
| "epoch": 1.86, |
| "learning_rate": 3.8300404665555817e-07, |
| "logits/chosen": 0.5122434496879578, |
| "logits/rejected": 1.1984080076217651, |
| "logps/chosen": -479.920654296875, |
| "logps/rejected": -683.6166381835938, |
| "loss": 0.0145, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -6.739544868469238, |
| "rewards/margins": 21.551074981689453, |
| "rewards/rejected": -28.290618896484375, |
| "step": 2900 |
| }, |
| { |
| "epoch": 1.86, |
| "eval_logits/chosen": -0.22228558361530304, |
| "eval_logits/rejected": 0.4165645241737366, |
| "eval_logps/chosen": -403.52783203125, |
| "eval_logps/rejected": -530.5889892578125, |
| "eval_loss": 0.1386074274778366, |
| "eval_rewards/accuracies": 0.9609375, |
| "eval_rewards/chosen": -7.416477680206299, |
| "eval_rewards/margins": 14.007347106933594, |
| "eval_rewards/rejected": -21.423826217651367, |
| "eval_runtime": 76.7881, |
| "eval_samples_per_second": 13.023, |
| "eval_steps_per_second": 0.417, |
| "step": 2900 |
| }, |
| { |
| "epoch": 1.87, |
| "learning_rate": 3.8240895024994047e-07, |
| "logits/chosen": 0.35297220945358276, |
| "logits/rejected": 1.0394313335418701, |
| "logps/chosen": -442.6025390625, |
| "logps/rejected": -655.9295654296875, |
| "loss": 0.0235, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -6.260979175567627, |
| "rewards/margins": 20.33395004272461, |
| "rewards/rejected": -26.59493064880371, |
| "step": 2910 |
| }, |
| { |
| "epoch": 1.88, |
| "learning_rate": 3.8181385384432276e-07, |
| "logits/chosen": 0.402191162109375, |
| "logits/rejected": 1.116523265838623, |
| "logps/chosen": -393.9117126464844, |
| "logps/rejected": -693.4283447265625, |
| "loss": 0.0116, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.633070468902588, |
| "rewards/margins": 21.96987533569336, |
| "rewards/rejected": -26.602941513061523, |
| "step": 2920 |
| }, |
| { |
| "epoch": 1.88, |
| "learning_rate": 3.8121875743870506e-07, |
| "logits/chosen": 0.16651079058647156, |
| "logits/rejected": 0.6804031133651733, |
| "logps/chosen": -412.31512451171875, |
| "logps/rejected": -665.4926147460938, |
| "loss": 0.0146, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.822356700897217, |
| "rewards/margins": 19.13541030883789, |
| "rewards/rejected": -23.957767486572266, |
| "step": 2930 |
| }, |
| { |
| "epoch": 1.89, |
| "learning_rate": 3.8062366103308735e-07, |
| "logits/chosen": 0.1117088571190834, |
| "logits/rejected": 1.083188772201538, |
| "logps/chosen": -369.11370849609375, |
| "logps/rejected": -570.9990844726562, |
| "loss": 0.0157, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.329820156097412, |
| "rewards/margins": 19.094078063964844, |
| "rewards/rejected": -22.423898696899414, |
| "step": 2940 |
| }, |
| { |
| "epoch": 1.9, |
| "learning_rate": 3.8002856462746965e-07, |
| "logits/chosen": 0.11049242317676544, |
| "logits/rejected": 0.9105658531188965, |
| "logps/chosen": -433.6129455566406, |
| "logps/rejected": -589.2352294921875, |
| "loss": 0.0138, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.2948660850524902, |
| "rewards/margins": 18.181903839111328, |
| "rewards/rejected": -21.47677230834961, |
| "step": 2950 |
| }, |
| { |
| "epoch": 1.9, |
| "learning_rate": 3.794334682218519e-07, |
| "logits/chosen": -0.10087742656469345, |
| "logits/rejected": 0.8825875520706177, |
| "logps/chosen": -401.3361511230469, |
| "logps/rejected": -503.46710205078125, |
| "loss": 0.0172, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.8647804260253906, |
| "rewards/margins": 15.990776062011719, |
| "rewards/rejected": -19.85555648803711, |
| "step": 2960 |
| }, |
| { |
| "epoch": 1.91, |
| "learning_rate": 3.7883837181623424e-07, |
| "logits/chosen": 0.07600893825292587, |
| "logits/rejected": 0.9553617238998413, |
| "logps/chosen": -408.1060485839844, |
| "logps/rejected": -534.631591796875, |
| "loss": 0.0123, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.9009666442871094, |
| "rewards/margins": 17.22827911376953, |
| "rewards/rejected": -21.12924575805664, |
| "step": 2970 |
| }, |
| { |
| "epoch": 1.92, |
| "learning_rate": 3.7824327541061653e-07, |
| "logits/chosen": 0.2739563286304474, |
| "logits/rejected": 0.8527243733406067, |
| "logps/chosen": -437.29638671875, |
| "logps/rejected": -695.6834106445312, |
| "loss": 0.0105, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.193564414978027, |
| "rewards/margins": 21.4495792388916, |
| "rewards/rejected": -26.643142700195312, |
| "step": 2980 |
| }, |
| { |
| "epoch": 1.92, |
| "learning_rate": 3.776481790049988e-07, |
| "logits/chosen": 0.43024197220802307, |
| "logits/rejected": 1.1188406944274902, |
| "logps/chosen": -386.9818115234375, |
| "logps/rejected": -554.8717041015625, |
| "loss": 0.0291, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.847230911254883, |
| "rewards/margins": 17.605852127075195, |
| "rewards/rejected": -23.453083038330078, |
| "step": 2990 |
| }, |
| { |
| "epoch": 1.93, |
| "learning_rate": 3.7705308259938107e-07, |
| "logits/chosen": 0.6990832090377808, |
| "logits/rejected": 1.3339582681655884, |
| "logps/chosen": -375.6474609375, |
| "logps/rejected": -568.9274291992188, |
| "loss": 0.0173, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.8585574626922607, |
| "rewards/margins": 18.85175323486328, |
| "rewards/rejected": -22.710311889648438, |
| "step": 3000 |
| }, |
| { |
| "epoch": 1.93, |
| "eval_logits/chosen": -0.04803978279232979, |
| "eval_logits/rejected": 0.6017779111862183, |
| "eval_logps/chosen": -375.2929992675781, |
| "eval_logps/rejected": -498.0708312988281, |
| "eval_loss": 0.1309323012828827, |
| "eval_rewards/accuracies": 0.953125, |
| "eval_rewards/chosen": -4.592996120452881, |
| "eval_rewards/margins": 13.579015731811523, |
| "eval_rewards/rejected": -18.17201042175293, |
| "eval_runtime": 76.759, |
| "eval_samples_per_second": 13.028, |
| "eval_steps_per_second": 0.417, |
| "step": 3000 |
| }, |
| { |
| "epoch": 1.93, |
| "learning_rate": 3.764579861937634e-07, |
| "logits/chosen": 0.5929206609725952, |
| "logits/rejected": 1.196090579032898, |
| "logps/chosen": -398.5981750488281, |
| "logps/rejected": -553.6544799804688, |
| "loss": 0.0145, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.344789028167725, |
| "rewards/margins": 19.267980575561523, |
| "rewards/rejected": -23.612768173217773, |
| "step": 3010 |
| }, |
| { |
| "epoch": 1.94, |
| "learning_rate": 3.7586288978814566e-07, |
| "logits/chosen": 0.46780499815940857, |
| "logits/rejected": 1.3755557537078857, |
| "logps/chosen": -408.9537658691406, |
| "logps/rejected": -639.6270141601562, |
| "loss": 0.0271, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.787570953369141, |
| "rewards/margins": 19.320308685302734, |
| "rewards/rejected": -26.107879638671875, |
| "step": 3020 |
| }, |
| { |
| "epoch": 1.95, |
| "learning_rate": 3.7526779338252795e-07, |
| "logits/chosen": 0.47549518942832947, |
| "logits/rejected": 1.1248539686203003, |
| "logps/chosen": -458.66864013671875, |
| "logps/rejected": -612.8165283203125, |
| "loss": 0.0257, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.648183345794678, |
| "rewards/margins": 18.065431594848633, |
| "rewards/rejected": -23.713613510131836, |
| "step": 3030 |
| }, |
| { |
| "epoch": 1.95, |
| "learning_rate": 3.7467269697691025e-07, |
| "logits/chosen": 0.022746117785573006, |
| "logits/rejected": 1.0212655067443848, |
| "logps/chosen": -437.48583984375, |
| "logps/rejected": -603.9195556640625, |
| "loss": 0.0137, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.6978535652160645, |
| "rewards/margins": 19.632654190063477, |
| "rewards/rejected": -25.330509185791016, |
| "step": 3040 |
| }, |
| { |
| "epoch": 1.96, |
| "learning_rate": 3.740776005712925e-07, |
| "logits/chosen": -0.12851546704769135, |
| "logits/rejected": 0.7596645951271057, |
| "logps/chosen": -444.29461669921875, |
| "logps/rejected": -648.9785766601562, |
| "loss": 0.0098, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.788307189941406, |
| "rewards/margins": 18.562524795532227, |
| "rewards/rejected": -24.350830078125, |
| "step": 3050 |
| }, |
| { |
| "epoch": 1.97, |
| "learning_rate": 3.7348250416567484e-07, |
| "logits/chosen": 0.014139672741293907, |
| "logits/rejected": 0.8716185688972473, |
| "logps/chosen": -395.7372131347656, |
| "logps/rejected": -608.2803955078125, |
| "loss": 0.0161, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.413556098937988, |
| "rewards/margins": 19.46448516845703, |
| "rewards/rejected": -23.878040313720703, |
| "step": 3060 |
| }, |
| { |
| "epoch": 1.97, |
| "learning_rate": 3.7288740776005713e-07, |
| "logits/chosen": 0.32760128378868103, |
| "logits/rejected": 0.5136176943778992, |
| "logps/chosen": -420.23480224609375, |
| "logps/rejected": -669.7601318359375, |
| "loss": 0.017, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.701635360717773, |
| "rewards/margins": 19.267168045043945, |
| "rewards/rejected": -23.96880531311035, |
| "step": 3070 |
| }, |
| { |
| "epoch": 1.98, |
| "learning_rate": 3.722923113544394e-07, |
| "logits/chosen": 0.4951232373714447, |
| "logits/rejected": 1.083812952041626, |
| "logps/chosen": -412.07464599609375, |
| "logps/rejected": -682.0787353515625, |
| "loss": 0.0255, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.654479026794434, |
| "rewards/margins": 18.33094596862793, |
| "rewards/rejected": -22.985427856445312, |
| "step": 3080 |
| }, |
| { |
| "epoch": 1.99, |
| "learning_rate": 3.7169721494882167e-07, |
| "logits/chosen": 0.7839171290397644, |
| "logits/rejected": 1.5858978033065796, |
| "logps/chosen": -398.9652099609375, |
| "logps/rejected": -555.3340454101562, |
| "loss": 0.022, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.493814945220947, |
| "rewards/margins": 17.03528594970703, |
| "rewards/rejected": -21.52910041809082, |
| "step": 3090 |
| }, |
| { |
| "epoch": 1.99, |
| "learning_rate": 3.71102118543204e-07, |
| "logits/chosen": 0.8546144366264343, |
| "logits/rejected": 1.6159175634384155, |
| "logps/chosen": -427.02978515625, |
| "logps/rejected": -589.3820190429688, |
| "loss": 0.0187, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.833371639251709, |
| "rewards/margins": 18.236379623413086, |
| "rewards/rejected": -25.069753646850586, |
| "step": 3100 |
| }, |
| { |
| "epoch": 1.99, |
| "eval_logits/chosen": 0.0045381635427474976, |
| "eval_logits/rejected": 0.7645629048347473, |
| "eval_logps/chosen": -400.106201171875, |
| "eval_logps/rejected": -530.0531616210938, |
| "eval_loss": 0.13586518168449402, |
| "eval_rewards/accuracies": 0.953125, |
| "eval_rewards/chosen": -7.074316024780273, |
| "eval_rewards/margins": 14.295927047729492, |
| "eval_rewards/rejected": -21.370241165161133, |
| "eval_runtime": 76.6017, |
| "eval_samples_per_second": 13.055, |
| "eval_steps_per_second": 0.418, |
| "step": 3100 |
| }, |
| { |
| "epoch": 2.0, |
| "learning_rate": 3.7050702213758626e-07, |
| "logits/chosen": 0.6572461128234863, |
| "logits/rejected": 1.7969856262207031, |
| "logps/chosen": -398.08453369140625, |
| "logps/rejected": -607.6263427734375, |
| "loss": 0.0082, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.214932918548584, |
| "rewards/margins": 21.25518035888672, |
| "rewards/rejected": -27.470117568969727, |
| "step": 3110 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 9336, |
| "num_train_epochs": 6, |
| "save_steps": 500, |
| "total_flos": 0.0, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
