{
  "best_metric": 0.4865521490573883,
  "best_model_checkpoint": "models/llama3.2-3b-dpo-vanilla/checkpoint-10000",
  "epoch": 0.9999813887699838,
  "eval_steps": 10000,
  "global_step": 26865,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 3.722246003238354e-05,
      "grad_norm": 2.7675779324505863,
      "learning_rate": 1.8608113137327875e-10,
      "logits/chosen": -0.82421875,
      "logits/rejected": -0.4453125,
      "logps/chosen": -162.0,
      "logps/rejected": -158.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0003722246003238354,
      "grad_norm": 2.7033149841946615,
      "learning_rate": 1.8608113137327875e-09,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.79296875,
      "logps/chosen": -189.0,
      "logps/rejected": -191.0,
      "loss": 0.692,
      "rewards/accuracies": 0.1388888955116272,
      "rewards/chosen": -0.0003299713134765625,
      "rewards/margins": -0.00058746337890625,
      "rewards/rejected": 0.0002613067626953125,
      "step": 10
    },
    {
      "epoch": 0.0007444492006476708,
      "grad_norm": 2.6175600962535452,
      "learning_rate": 3.721622627465575e-09,
      "logits/chosen": -1.0,
      "logits/rejected": -0.89453125,
      "logps/chosen": -220.0,
      "logps/rejected": -202.0,
      "loss": 0.6926,
      "rewards/accuracies": 0.23749999701976776,
      "rewards/chosen": 0.0007171630859375,
      "rewards/margins": 0.00090789794921875,
      "rewards/rejected": -0.0001888275146484375,
      "step": 20
    },
    {
      "epoch": 0.0011166738009715062,
      "grad_norm": 2.555968700387509,
      "learning_rate": 5.5824339411983625e-09,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.72265625,
      "logps/chosen": -208.0,
      "logps/rejected": -190.0,
      "loss": 0.6926,
      "rewards/accuracies": 0.26249998807907104,
      "rewards/chosen": 0.000690460205078125,
      "rewards/margins": -0.0002498626708984375,
      "rewards/rejected": 0.00093841552734375,
      "step": 30
    },
    {
      "epoch": 0.0014888984012953416,
      "grad_norm": 2.401959563497523,
      "learning_rate": 7.44324525493115e-09,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.79296875,
      "logps/chosen": -215.0,
      "logps/rejected": -188.0,
      "loss": 0.6928,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": 0.0005950927734375,
      "rewards/margins": 0.0006256103515625,
      "rewards/rejected": -3.1948089599609375e-05,
      "step": 40
    },
    {
      "epoch": 0.001861123001619177,
      "grad_norm": 2.566988114994063,
      "learning_rate": 9.304056568663937e-09,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.73046875,
      "logps/chosen": -202.0,
      "logps/rejected": -177.0,
      "loss": 0.6926,
      "rewards/accuracies": 0.2874999940395355,
      "rewards/chosen": 9.393692016601562e-05,
      "rewards/margins": 0.0003604888916015625,
      "rewards/rejected": -0.00026702880859375,
      "step": 50
    },
    {
      "epoch": 0.0022333476019430125,
      "grad_norm": 2.5016128490820444,
      "learning_rate": 1.1164867882396725e-08,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.875,
      "logps/chosen": -194.0,
      "logps/rejected": -169.0,
      "loss": 0.6929,
      "rewards/accuracies": 0.23749999701976776,
      "rewards/chosen": -0.0004062652587890625,
      "rewards/margins": -0.00092315673828125,
      "rewards/rejected": 0.000514984130859375,
      "step": 60
    },
    {
      "epoch": 0.002605572202266848,
      "grad_norm": 2.6360171846049814,
      "learning_rate": 1.3025679196129512e-08,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.80859375,
      "logps/chosen": -205.0,
      "logps/rejected": -193.0,
      "loss": 0.6923,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.000202178955078125,
      "rewards/margins": 0.00054931640625,
      "rewards/rejected": -0.0003452301025390625,
      "step": 70
    },
    {
      "epoch": 0.002977796802590683,
      "grad_norm": 2.4692735353151547,
      "learning_rate": 1.48864905098623e-08,
      "logits/chosen": -0.87109375,
      "logits/rejected": -0.72265625,
      "logps/chosen": -186.0,
      "logps/rejected": -170.0,
      "loss": 0.6921,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": 0.00098419189453125,
      "rewards/margins": 0.00099945068359375,
      "rewards/rejected": -1.5616416931152344e-05,
      "step": 80
    },
    {
      "epoch": 0.0033500214029145185,
      "grad_norm": 2.7685147528245437,
      "learning_rate": 1.6747301823595087e-08,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.91796875,
      "logps/chosen": -220.0,
      "logps/rejected": -210.0,
      "loss": 0.6925,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.0006256103515625,
      "rewards/margins": 0.0003108978271484375,
      "rewards/rejected": 0.00031280517578125,
      "step": 90
    },
    {
      "epoch": 0.003722246003238354,
      "grad_norm": 2.7355589831651916,
      "learning_rate": 1.8608113137327873e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.84375,
      "logps/chosen": -234.0,
      "logps/rejected": -203.0,
      "loss": 0.6924,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.001129150390625,
      "rewards/margins": -6.246566772460938e-05,
      "rewards/rejected": 0.001190185546875,
      "step": 100
    },
    {
      "epoch": 0.004094470603562189,
      "grad_norm": 2.480396607881251,
      "learning_rate": 2.0468924451060663e-08,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.796875,
      "logps/chosen": -218.0,
      "logps/rejected": -200.0,
      "loss": 0.6924,
      "rewards/accuracies": 0.2874999940395355,
      "rewards/chosen": -0.000141143798828125,
      "rewards/margins": -0.00090789794921875,
      "rewards/rejected": 0.000766754150390625,
      "step": 110
    },
    {
      "epoch": 0.004466695203886025,
      "grad_norm": 2.3461996293834537,
      "learning_rate": 2.232973576479345e-08,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.7890625,
      "logps/chosen": -206.0,
      "logps/rejected": -199.0,
      "loss": 0.6922,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": 0.00154876708984375,
      "rewards/margins": 0.000843048095703125,
      "rewards/rejected": 0.000705718994140625,
      "step": 120
    },
    {
      "epoch": 0.00483891980420986,
      "grad_norm": 2.6216367130161755,
      "learning_rate": 2.4190547078526237e-08,
      "logits/chosen": -1.125,
      "logits/rejected": -0.8125,
      "logps/chosen": -213.0,
      "logps/rejected": -205.0,
      "loss": 0.6925,
      "rewards/accuracies": 0.29374998807907104,
      "rewards/chosen": 0.0020294189453125,
      "rewards/margins": 0.000518798828125,
      "rewards/rejected": 0.00151824951171875,
      "step": 130
    },
    {
      "epoch": 0.005211144404533696,
      "grad_norm": 2.5565500850009806,
      "learning_rate": 2.6051358392259023e-08,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.7109375,
      "logps/chosen": -214.0,
      "logps/rejected": -193.0,
      "loss": 0.6924,
      "rewards/accuracies": 0.33125001192092896,
      "rewards/chosen": 0.0029449462890625,
      "rewards/margins": 0.00054931640625,
      "rewards/rejected": 0.0023956298828125,
      "step": 140
    },
    {
      "epoch": 0.005583369004857531,
      "grad_norm": 2.718235555865212,
      "learning_rate": 2.791216970599181e-08,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.73828125,
      "logps/chosen": -203.0,
      "logps/rejected": -184.0,
      "loss": 0.6923,
      "rewards/accuracies": 0.29374998807907104,
      "rewards/chosen": 0.0026092529296875,
      "rewards/margins": 0.000659942626953125,
      "rewards/rejected": 0.001953125,
      "step": 150
    },
    {
      "epoch": 0.005955593605181366,
      "grad_norm": 2.8493869496809676,
      "learning_rate": 2.97729810197246e-08,
      "logits/chosen": -1.125,
      "logits/rejected": -0.8203125,
      "logps/chosen": -200.0,
      "logps/rejected": -180.0,
      "loss": 0.6919,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": 0.00384521484375,
      "rewards/margins": 0.00092315673828125,
      "rewards/rejected": 0.0029296875,
      "step": 160
    },
    {
      "epoch": 0.006327818205505202,
      "grad_norm": 2.6132709561904597,
      "learning_rate": 3.163379233345739e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.6796875,
      "logps/chosen": -195.0,
      "logps/rejected": -182.0,
      "loss": 0.6919,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00518798828125,
      "rewards/margins": 0.000782012939453125,
      "rewards/rejected": 0.00439453125,
      "step": 170
    },
    {
      "epoch": 0.006700042805829037,
      "grad_norm": 2.3690978508762686,
      "learning_rate": 3.3494603647190173e-08,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.84375,
      "logps/chosen": -210.0,
      "logps/rejected": -186.0,
      "loss": 0.6921,
      "rewards/accuracies": 0.3812499940395355,
      "rewards/chosen": 0.007415771484375,
      "rewards/margins": 0.00090789794921875,
      "rewards/rejected": 0.00653076171875,
      "step": 180
    },
    {
      "epoch": 0.007072267406152873,
      "grad_norm": 3.1139980833389385,
      "learning_rate": 3.535541496092296e-08,
      "logits/chosen": -1.0546875,
      "logits/rejected": -0.7734375,
      "logps/chosen": -188.0,
      "logps/rejected": -184.0,
      "loss": 0.692,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": 0.009033203125,
      "rewards/margins": 0.00238037109375,
      "rewards/rejected": 0.006683349609375,
      "step": 190
    },
    {
      "epoch": 0.007444492006476708,
      "grad_norm": 2.737543394253561,
      "learning_rate": 3.721622627465575e-08,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.8984375,
      "logps/chosen": -217.0,
      "logps/rejected": -206.0,
      "loss": 0.6917,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.01043701171875,
      "rewards/margins": 0.00225830078125,
      "rewards/rejected": 0.0081787109375,
      "step": 200
    },
    {
      "epoch": 0.007816716606800543,
      "grad_norm": 2.93454804198715,
      "learning_rate": 3.9077037588388533e-08,
      "logits/chosen": -1.0546875,
      "logits/rejected": -0.76953125,
      "logps/chosen": -200.0,
      "logps/rejected": -188.0,
      "loss": 0.692,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": 0.010986328125,
      "rewards/margins": 0.00173187255859375,
      "rewards/rejected": 0.00927734375,
      "step": 210
    },
    {
      "epoch": 0.008188941207124378,
      "grad_norm": 2.3697025414150232,
      "learning_rate": 4.093784890212133e-08,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.67578125,
      "logps/chosen": -201.0,
      "logps/rejected": -185.0,
      "loss": 0.6916,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": 0.01361083984375,
      "rewards/margins": 0.0020294189453125,
      "rewards/rejected": 0.0115966796875,
      "step": 220
    },
    {
      "epoch": 0.008561165807448215,
      "grad_norm": 2.899179002653,
      "learning_rate": 4.2798660215854113e-08,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.75,
      "logps/chosen": -196.0,
      "logps/rejected": -175.0,
      "loss": 0.6916,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": 0.0164794921875,
      "rewards/margins": 0.00084686279296875,
      "rewards/rejected": 0.015625,
      "step": 230
    },
    {
      "epoch": 0.00893339040777205,
      "grad_norm": 2.4593483723867093,
      "learning_rate": 4.46594715295869e-08,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.8125,
      "logps/chosen": -210.0,
      "logps/rejected": -180.0,
      "loss": 0.6913,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.021240234375,
      "rewards/margins": 0.0029754638671875,
      "rewards/rejected": 0.018310546875,
      "step": 240
    },
    {
      "epoch": 0.009305615008095885,
      "grad_norm": 2.74046179714292,
      "learning_rate": 4.652028284331969e-08,
      "logits/chosen": -0.94921875,
      "logits/rejected": -0.73046875,
      "logps/chosen": -202.0,
      "logps/rejected": -201.0,
      "loss": 0.6908,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.0263671875,
      "rewards/margins": 0.006072998046875,
      "rewards/rejected": 0.0203857421875,
      "step": 250
    },
    {
      "epoch": 0.00967783960841972,
      "grad_norm": 2.7118399339076786,
      "learning_rate": 4.8381094157052473e-08,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.83203125,
      "logps/chosen": -207.0,
      "logps/rejected": -203.0,
      "loss": 0.6904,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 0.030029296875,
      "rewards/margins": 0.0023651123046875,
      "rewards/rejected": 0.027587890625,
      "step": 260
    },
    {
      "epoch": 0.010050064208743556,
      "grad_norm": 2.6088806693495794,
      "learning_rate": 5.024190547078526e-08,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.7109375,
      "logps/chosen": -203.0,
      "logps/rejected": -206.0,
      "loss": 0.6906,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": 0.03515625,
      "rewards/margins": 0.006134033203125,
      "rewards/rejected": 0.0289306640625,
      "step": 270
    },
    {
      "epoch": 0.010422288809067391,
      "grad_norm": 2.3559580463984906,
      "learning_rate": 5.210271678451805e-08,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.69921875,
      "logps/chosen": -213.0,
      "logps/rejected": -202.0,
      "loss": 0.6899,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.040283203125,
      "rewards/margins": 0.009033203125,
      "rewards/rejected": 0.03125,
      "step": 280
    },
    {
      "epoch": 0.010794513409391226,
      "grad_norm": 2.578959278622856,
      "learning_rate": 5.3963528098250833e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.734375,
      "logps/chosen": -208.0,
      "logps/rejected": -184.0,
      "loss": 0.6901,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.04052734375,
      "rewards/margins": 0.00347900390625,
      "rewards/rejected": 0.036865234375,
      "step": 290
    },
    {
      "epoch": 0.011166738009715063,
      "grad_norm": 2.688366990658729,
      "learning_rate": 5.582433941198362e-08,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.8125,
      "logps/chosen": -202.0,
      "logps/rejected": -193.0,
      "loss": 0.6897,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.049560546875,
      "rewards/margins": 0.006561279296875,
      "rewards/rejected": 0.04296875,
      "step": 300
    },
    {
      "epoch": 0.011538962610038898,
      "grad_norm": 2.791491844904741,
      "learning_rate": 5.7685150725716413e-08,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.85546875,
      "logps/chosen": -205.0,
      "logps/rejected": -183.0,
      "loss": 0.6893,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.05517578125,
      "rewards/margins": 0.00531005859375,
      "rewards/rejected": 0.0498046875,
      "step": 310
    },
    {
      "epoch": 0.011911187210362733,
      "grad_norm": 2.5725740562094574,
      "learning_rate": 5.95459620394492e-08,
      "logits/chosen": -1.25,
      "logits/rejected": -0.96875,
      "logps/chosen": -194.0,
      "logps/rejected": -181.0,
      "loss": 0.6886,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.068359375,
      "rewards/margins": 0.01068115234375,
      "rewards/rejected": 0.057861328125,
      "step": 320
    },
    {
      "epoch": 0.012283411810686568,
      "grad_norm": 2.6290519847912335,
      "learning_rate": 6.140677335318198e-08,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.9375,
      "logps/chosen": -189.0,
      "logps/rejected": -174.0,
      "loss": 0.6868,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": 0.07373046875,
      "rewards/margins": 0.0146484375,
      "rewards/rejected": 0.05908203125,
      "step": 330
    },
    {
      "epoch": 0.012655636411010404,
      "grad_norm": 2.655380089335054,
      "learning_rate": 6.326758466691477e-08,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.734375,
      "logps/chosen": -202.0,
      "logps/rejected": -181.0,
      "loss": 0.686,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08544921875,
      "rewards/margins": 0.015380859375,
      "rewards/rejected": 0.06982421875,
      "step": 340
    },
    {
      "epoch": 0.013027861011334239,
      "grad_norm": 2.599427604735967,
      "learning_rate": 6.512839598064757e-08,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.90234375,
      "logps/chosen": -215.0,
      "logps/rejected": -198.0,
      "loss": 0.6858,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.09033203125,
      "rewards/margins": 0.00994873046875,
      "rewards/rejected": 0.08056640625,
      "step": 350
    },
    {
      "epoch": 0.013400085611658074,
      "grad_norm": 2.6808995091057453,
      "learning_rate": 6.698920729438035e-08,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.984375,
      "logps/chosen": -199.0,
      "logps/rejected": -185.0,
      "loss": 0.6841,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.09423828125,
      "rewards/margins": 0.01300048828125,
      "rewards/rejected": 0.0810546875,
      "step": 360
    },
    {
      "epoch": 0.01377231021198191,
      "grad_norm": 2.380188165864687,
      "learning_rate": 6.885001860811314e-08,
      "logits/chosen": -1.25,
      "logits/rejected": -0.9765625,
      "logps/chosen": -186.0,
      "logps/rejected": -187.0,
      "loss": 0.6837,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.109375,
      "rewards/margins": 0.0191650390625,
      "rewards/rejected": 0.09033203125,
      "step": 370
    },
    {
      "epoch": 0.014144534812305746,
      "grad_norm": 2.573319303434875,
      "learning_rate": 7.071082992184592e-08,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.734375,
      "logps/chosen": -185.0,
      "logps/rejected": -176.0,
      "loss": 0.6823,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.11474609375,
      "rewards/margins": 0.0240478515625,
      "rewards/rejected": 0.0908203125,
      "step": 380
    },
    {
      "epoch": 0.01451675941262958,
      "grad_norm": 2.554363391928645,
      "learning_rate": 7.257164123557871e-08,
      "logits/chosen": -1.28125,
      "logits/rejected": -1.046875,
      "logps/chosen": -199.0,
      "logps/rejected": -172.0,
      "loss": 0.6829,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.11865234375,
      "rewards/margins": 0.0220947265625,
      "rewards/rejected": 0.0966796875,
      "step": 390
    },
    {
      "epoch": 0.014888984012953415,
      "grad_norm": 2.731833439551786,
      "learning_rate": 7.44324525493115e-08,
      "logits/chosen": -1.3203125,
      "logits/rejected": -1.0546875,
      "logps/chosen": -214.0,
      "logps/rejected": -198.0,
      "loss": 0.6801,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.12890625,
      "rewards/margins": 0.02685546875,
      "rewards/rejected": 0.1015625,
      "step": 400
    },
    {
      "epoch": 0.015261208613277252,
      "grad_norm": 2.455720810886261,
      "learning_rate": 7.629326386304429e-08,
      "logits/chosen": -1.3125,
      "logits/rejected": -1.046875,
      "logps/chosen": -203.0,
      "logps/rejected": -181.0,
      "loss": 0.6801,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": 0.123046875,
      "rewards/margins": 0.021728515625,
      "rewards/rejected": 0.10107421875,
      "step": 410
    },
    {
      "epoch": 0.015633433213601087,
      "grad_norm": 2.4822317977316457,
      "learning_rate": 7.815407517677707e-08,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.921875,
      "logps/chosen": -195.0,
      "logps/rejected": -184.0,
      "loss": 0.6809,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12890625,
      "rewards/margins": 0.0244140625,
      "rewards/rejected": 0.10400390625,
      "step": 420
    },
    {
      "epoch": 0.016005657813924924,
      "grad_norm": 2.5054758300448836,
      "learning_rate": 8.001488649050986e-08,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.9296875,
      "logps/chosen": -194.0,
      "logps/rejected": -183.0,
      "loss": 0.6796,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.126953125,
      "rewards/margins": 0.0216064453125,
      "rewards/rejected": 0.10498046875,
      "step": 430
    },
    {
      "epoch": 0.016377882414248757,
      "grad_norm": 2.5007014901630336,
      "learning_rate": 8.187569780424265e-08,
      "logits/chosen": -1.2578125,
      "logits/rejected": -1.09375,
      "logps/chosen": -196.0,
      "logps/rejected": -186.0,
      "loss": 0.6798,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.1298828125,
      "rewards/margins": 0.021728515625,
      "rewards/rejected": 0.1083984375,
      "step": 440
    },
    {
      "epoch": 0.016750107014572593,
      "grad_norm": 2.5028898951769087,
      "learning_rate": 8.373650911797543e-08,
      "logits/chosen": -1.3125,
      "logits/rejected": -1.1328125,
      "logps/chosen": -191.0,
      "logps/rejected": -181.0,
      "loss": 0.6784,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1376953125,
      "rewards/margins": 0.027099609375,
      "rewards/rejected": 0.10986328125,
      "step": 450
    },
    {
      "epoch": 0.01712233161489643,
      "grad_norm": 2.7091527142250453,
      "learning_rate": 8.559732043170823e-08,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.984375,
      "logps/chosen": -206.0,
      "logps/rejected": -177.0,
      "loss": 0.6781,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": 0.134765625,
      "rewards/margins": 0.02294921875,
      "rewards/rejected": 0.1123046875,
      "step": 460
    },
    {
      "epoch": 0.017494556215220263,
      "grad_norm": 2.6283451405500298,
      "learning_rate": 8.745813174544101e-08,
      "logits/chosen": -1.3125,
      "logits/rejected": -1.0234375,
      "logps/chosen": -186.0,
      "logps/rejected": -186.0,
      "loss": 0.6781,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.142578125,
      "rewards/margins": 0.038330078125,
      "rewards/rejected": 0.1044921875,
      "step": 470
    },
    {
      "epoch": 0.0178667808155441,
      "grad_norm": 2.5151002867958145,
      "learning_rate": 8.93189430591738e-08,
      "logits/chosen": -1.2890625,
      "logits/rejected": -1.171875,
      "logps/chosen": -207.0,
      "logps/rejected": -202.0,
      "loss": 0.675,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.134765625,
      "rewards/margins": 0.0274658203125,
      "rewards/rejected": 0.10693359375,
      "step": 480
    },
    {
      "epoch": 0.018239005415867936,
      "grad_norm": 2.3878249562977802,
      "learning_rate": 9.117975437290658e-08,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.03125,
      "logps/chosen": -199.0,
      "logps/rejected": -186.0,
      "loss": 0.676,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.146484375,
      "rewards/margins": 0.0458984375,
      "rewards/rejected": 0.10009765625,
      "step": 490
    },
    {
      "epoch": 0.01861123001619177,
      "grad_norm": 2.5583158654030154,
      "learning_rate": 9.304056568663937e-08,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.0625,
      "logps/chosen": -214.0,
      "logps/rejected": -200.0,
      "loss": 0.6752,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.1337890625,
      "rewards/margins": 0.048583984375,
      "rewards/rejected": 0.08544921875,
      "step": 500
    },
    {
      "epoch": 0.018983454616515606,
      "grad_norm": 2.696900722028163,
      "learning_rate": 9.490137700037215e-08,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.9921875,
      "logps/chosen": -209.0,
      "logps/rejected": -189.0,
      "loss": 0.6749,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.1396484375,
      "rewards/margins": 0.0458984375,
      "rewards/rejected": 0.09375,
      "step": 510
    },
    {
      "epoch": 0.01935567921683944,
      "grad_norm": 2.829063555393864,
      "learning_rate": 9.676218831410495e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.21875,
      "logps/chosen": -210.0,
      "logps/rejected": -198.0,
      "loss": 0.6729,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.134765625,
      "rewards/margins": 0.04638671875,
      "rewards/rejected": 0.08837890625,
      "step": 520
    },
    {
      "epoch": 0.019727903817163276,
      "grad_norm": 2.5806064906437918,
      "learning_rate": 9.862299962783774e-08,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.0625,
      "logps/chosen": -206.0,
      "logps/rejected": -190.0,
      "loss": 0.6756,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.109375,
      "rewards/margins": 0.036376953125,
      "rewards/rejected": 0.07275390625,
      "step": 530
    },
    {
      "epoch": 0.020100128417487113,
      "grad_norm": 2.640926126338795,
      "learning_rate": 1.0048381094157052e-07,
      "logits/chosen": -1.4375,
      "logits/rejected": -1.125,
      "logps/chosen": -193.0,
      "logps/rejected": -180.0,
      "loss": 0.6732,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.11474609375,
      "rewards/margins": 0.0400390625,
      "rewards/rejected": 0.07470703125,
      "step": 540
    },
    {
      "epoch": 0.020472353017810946,
      "grad_norm": 2.617657793885317,
      "learning_rate": 1.0234462225530331e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.140625,
      "logps/chosen": -207.0,
      "logps/rejected": -203.0,
      "loss": 0.6711,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.10791015625,
      "rewards/margins": 0.0380859375,
      "rewards/rejected": 0.06982421875,
      "step": 550
    },
    {
      "epoch": 0.020844577618134782,
      "grad_norm": 2.6638518711759365,
      "learning_rate": 1.042054335690361e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.125,
      "logps/chosen": -192.0,
      "logps/rejected": -165.0,
      "loss": 0.6723,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": 0.109375,
      "rewards/margins": 0.037353515625,
      "rewards/rejected": 0.072265625,
      "step": 560
    },
    {
      "epoch": 0.02121680221845862,
      "grad_norm": 2.482529458968985,
      "learning_rate": 1.0606624488276889e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -1.1484375,
      "logps/chosen": -192.0,
      "logps/rejected": -186.0,
      "loss": 0.6698,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.1123046875,
      "rewards/margins": 0.042236328125,
      "rewards/rejected": 0.0703125,
      "step": 570
    },
    {
      "epoch": 0.021589026818782452,
      "grad_norm": 2.659341004854578,
      "learning_rate": 1.0792705619650167e-07,
      "logits/chosen": -1.4609375,
      "logits/rejected": -1.2421875,
      "logps/chosen": -202.0,
      "logps/rejected": -197.0,
      "loss": 0.6683,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.10400390625,
      "rewards/margins": 0.053955078125,
      "rewards/rejected": 0.05029296875,
      "step": 580
    },
    {
      "epoch": 0.02196125141910629,
      "grad_norm": 2.540918835657275,
      "learning_rate": 1.0978786751023446e-07,
      "logits/chosen": -1.5078125,
      "logits/rejected": -1.046875,
      "logps/chosen": -197.0,
      "logps/rejected": -202.0,
      "loss": 0.6678,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.1171875,
      "rewards/margins": 0.06884765625,
      "rewards/rejected": 0.048095703125,
      "step": 590
    },
    {
      "epoch": 0.022333476019430126,
      "grad_norm": 2.7559196656568092,
      "learning_rate": 1.1164867882396724e-07,
      "logits/chosen": -1.4921875,
      "logits/rejected": -1.3203125,
      "logps/chosen": -221.0,
      "logps/rejected": -199.0,
      "loss": 0.6659,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.08642578125,
      "rewards/margins": 0.0546875,
      "rewards/rejected": 0.031982421875,
      "step": 600
    },
    {
      "epoch": 0.02270570061975396,
      "grad_norm": 2.361824644749682,
      "learning_rate": 1.1350949013770003e-07,
      "logits/chosen": -1.4609375,
      "logits/rejected": -1.125,
      "logps/chosen": -187.0,
      "logps/rejected": -170.0,
      "loss": 0.6669,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0771484375,
      "rewards/margins": 0.058349609375,
      "rewards/rejected": 0.01904296875,
      "step": 610
    },
    {
      "epoch": 0.023077925220077795,
      "grad_norm": 2.6932602341933234,
      "learning_rate": 1.1537030145143283e-07,
      "logits/chosen": -1.5,
      "logits/rejected": -1.21875,
      "logps/chosen": -188.0,
      "logps/rejected": -185.0,
      "loss": 0.6652,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.0625,
      "rewards/margins": 0.06982421875,
      "rewards/rejected": -0.00714111328125,
      "step": 620
    },
    {
      "epoch": 0.023450149820401632,
      "grad_norm": 2.694676198661149,
      "learning_rate": 1.1723111276516561e-07,
      "logits/chosen": -1.4453125,
      "logits/rejected": -1.0703125,
      "logps/chosen": -192.0,
      "logps/rejected": -181.0,
      "loss": 0.6625,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.08056640625,
      "rewards/margins": 0.06494140625,
      "rewards/rejected": 0.015625,
      "step": 630
    },
    {
      "epoch": 0.023822374420725465,
      "grad_norm": 2.9381947427346997,
      "learning_rate": 1.190919240788984e-07,
      "logits/chosen": -1.484375,
      "logits/rejected": -1.21875,
      "logps/chosen": -192.0,
      "logps/rejected": -198.0,
      "loss": 0.6648,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.06298828125,
      "rewards/margins": 0.0673828125,
      "rewards/rejected": -0.004669189453125,
      "step": 640
    },
    {
      "epoch": 0.024194599021049302,
      "grad_norm": 2.835375859977096,
      "learning_rate": 1.2095273539263117e-07,
      "logits/chosen": -1.484375,
      "logits/rejected": -1.1484375,
      "logps/chosen": -212.0,
      "logps/rejected": -202.0,
      "loss": 0.662,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.044921875,
      "rewards/margins": 0.0712890625,
      "rewards/rejected": -0.0263671875,
      "step": 650
    },
    {
      "epoch": 0.024566823621373135,
      "grad_norm": 2.8253024837696947,
      "learning_rate": 1.2281354670636396e-07,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.3046875,
      "logps/chosen": -210.0,
      "logps/rejected": -202.0,
      "loss": 0.6614,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.04345703125,
      "rewards/margins": 0.0810546875,
      "rewards/rejected": -0.03759765625,
      "step": 660
    },
    {
      "epoch": 0.02493904822169697,
      "grad_norm": 2.9394531274938327,
      "learning_rate": 1.2467435802009675e-07,
      "logits/chosen": -1.4921875,
      "logits/rejected": -1.25,
      "logps/chosen": -210.0,
      "logps/rejected": -207.0,
      "loss": 0.6597,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0103759765625,
      "rewards/margins": 0.0703125,
      "rewards/rejected": -0.059814453125,
      "step": 670
    },
    {
      "epoch": 0.02531127282202081,
      "grad_norm": 2.957746675786797,
      "learning_rate": 1.2653516933382955e-07,
      "logits/chosen": -1.6484375,
      "logits/rejected": -1.515625,
      "logps/chosen": -234.0,
      "logps/rejected": -214.0,
      "loss": 0.6567,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.00482177734375,
      "rewards/margins": 0.050048828125,
      "rewards/rejected": -0.054931640625,
      "step": 680
    },
    {
      "epoch": 0.02568349742234464,
      "grad_norm": 2.794377827725235,
      "learning_rate": 1.2839598064756231e-07,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.390625,
      "logps/chosen": -212.0,
      "logps/rejected": -194.0,
      "loss": 0.6526,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.00616455078125,
      "rewards/margins": 0.078125,
      "rewards/rejected": -0.083984375,
      "step": 690
    },
    {
      "epoch": 0.026055722022668478,
      "grad_norm": 2.919571370441016,
      "learning_rate": 1.3025679196129513e-07,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.28125,
      "logps/chosen": -195.0,
      "logps/rejected": -201.0,
      "loss": 0.6522,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.031494140625,
      "rewards/margins": 0.08544921875,
      "rewards/rejected": -0.1171875,
      "step": 700
    },
    {
      "epoch": 0.026427946622992315,
      "grad_norm": 2.99987589276164,
      "learning_rate": 1.321176032750279e-07,
      "logits/chosen": -1.6640625,
      "logits/rejected": -1.421875,
      "logps/chosen": -208.0,
      "logps/rejected": -201.0,
      "loss": 0.6553,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0247802734375,
      "rewards/margins": 0.072265625,
      "rewards/rejected": -0.09716796875,
      "step": 710
    },
    {
      "epoch": 0.026800171223316148,
      "grad_norm": 3.00640233840768,
      "learning_rate": 1.339784145887607e-07,
      "logits/chosen": -1.671875,
      "logits/rejected": -1.4140625,
      "logps/chosen": -194.0,
      "logps/rejected": -194.0,
      "loss": 0.6486,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.042236328125,
      "rewards/margins": 0.10498046875,
      "rewards/rejected": -0.1474609375,
      "step": 720
    },
    {
      "epoch": 0.027172395823639985,
      "grad_norm": 2.842015055202482,
      "learning_rate": 1.3583922590249346e-07,
      "logits/chosen": -1.703125,
      "logits/rejected": -1.375,
      "logps/chosen": -213.0,
      "logps/rejected": -218.0,
      "loss": 0.6467,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.041015625,
      "rewards/margins": 0.1328125,
      "rewards/rejected": -0.173828125,
      "step": 730
    },
    {
      "epoch": 0.02754462042396382,
      "grad_norm": 2.7899016546344524,
      "learning_rate": 1.3770003721622628e-07,
      "logits/chosen": -1.640625,
      "logits/rejected": -1.421875,
      "logps/chosen": -207.0,
      "logps/rejected": -185.0,
      "loss": 0.6432,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06787109375,
      "rewards/margins": 0.09765625,
      "rewards/rejected": -0.166015625,
      "step": 740
    },
    {
      "epoch": 0.027916845024287654,
      "grad_norm": 2.7635578213833782,
      "learning_rate": 1.3956084852995905e-07,
      "logits/chosen": -1.8203125,
      "logits/rejected": -1.578125,
      "logps/chosen": -247.0,
      "logps/rejected": -217.0,
      "loss": 0.6388,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.09326171875,
      "rewards/margins": 0.125,
      "rewards/rejected": -0.2177734375,
      "step": 750
    },
    {
      "epoch": 0.02828906962461149,
      "grad_norm": 3.046552160563264,
      "learning_rate": 1.4142165984369184e-07,
      "logits/chosen": -1.7109375,
      "logits/rejected": -1.421875,
      "logps/chosen": -206.0,
      "logps/rejected": -229.0,
      "loss": 0.6379,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.095703125,
      "rewards/margins": 0.1640625,
      "rewards/rejected": -0.259765625,
      "step": 760
    },
    {
      "epoch": 0.028661294224935328,
      "grad_norm": 2.9590943598018775,
      "learning_rate": 1.432824711574246e-07,
      "logits/chosen": -1.6328125,
      "logits/rejected": -1.4765625,
      "logps/chosen": -215.0,
      "logps/rejected": -205.0,
      "loss": 0.6349,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.1171875,
      "rewards/margins": 0.126953125,
      "rewards/rejected": -0.244140625,
      "step": 770
    },
    {
      "epoch": 0.02903351882525916,
      "grad_norm": 3.1299330065382676,
      "learning_rate": 1.4514328247115743e-07,
      "logits/chosen": -1.7578125,
      "logits/rejected": -1.59375,
      "logps/chosen": -219.0,
      "logps/rejected": -233.0,
      "loss": 0.6315,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.142578125,
      "rewards/margins": 0.1669921875,
      "rewards/rejected": -0.310546875,
      "step": 780
    },
    {
      "epoch": 0.029405743425582997,
      "grad_norm": 3.1526181765553245,
      "learning_rate": 1.470040937848902e-07,
      "logits/chosen": -1.8359375,
      "logits/rejected": -1.5859375,
      "logps/chosen": -233.0,
      "logps/rejected": -243.0,
      "loss": 0.6236,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.15234375,
      "rewards/margins": 0.15625,
      "rewards/rejected": -0.30859375,
      "step": 790
    },
    {
      "epoch": 0.02977796802590683,
      "grad_norm": 3.316061776999472,
      "learning_rate": 1.48864905098623e-07,
      "logits/chosen": -1.734375,
      "logits/rejected": -1.5625,
      "logps/chosen": -234.0,
      "logps/rejected": -258.0,
      "loss": 0.6325,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.14453125,
      "rewards/margins": 0.2041015625,
      "rewards/rejected": -0.34765625,
      "step": 800
    },
    {
      "epoch": 0.030150192626230667,
      "grad_norm": 3.355995008606445,
      "learning_rate": 1.5072571641235578e-07,
      "logits/chosen": -1.796875,
      "logits/rejected": -1.6171875,
      "logps/chosen": -234.0,
      "logps/rejected": -243.0,
      "loss": 0.6266,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1826171875,
      "rewards/margins": 0.1982421875,
      "rewards/rejected": -0.380859375,
      "step": 810
    },
    {
      "epoch": 0.030522417226554504,
      "grad_norm": 3.167208342273511,
      "learning_rate": 1.5258652772608857e-07,
      "logits/chosen": -1.75,
      "logits/rejected": -1.546875,
      "logps/chosen": -214.0,
      "logps/rejected": -214.0,
      "loss": 0.6244,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2216796875,
      "rewards/margins": 0.13671875,
      "rewards/rejected": -0.359375,
      "step": 820
    },
    {
      "epoch": 0.030894641826878337,
      "grad_norm": 3.2104963098081827,
      "learning_rate": 1.5444733903982134e-07,
      "logits/chosen": -1.828125,
      "logits/rejected": -1.5234375,
      "logps/chosen": -214.0,
      "logps/rejected": -222.0,
      "loss": 0.6192,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.2021484375,
      "rewards/margins": 0.201171875,
      "rewards/rejected": -0.40234375,
      "step": 830
    },
    {
      "epoch": 0.031266866427202174,
      "grad_norm": 3.1912461924596607,
      "learning_rate": 1.5630815035355413e-07,
      "logits/chosen": -1.8046875,
      "logits/rejected": -1.53125,
      "logps/chosen": -223.0,
      "logps/rejected": -226.0,
      "loss": 0.607,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.2060546875,
      "rewards/margins": 0.248046875,
      "rewards/rejected": -0.453125,
      "step": 840
    },
    {
      "epoch": 0.03163909102752601,
      "grad_norm": 3.231345640134611,
      "learning_rate": 1.5816896166728693e-07,
      "logits/chosen": -1.7421875,
      "logits/rejected": -1.4921875,
      "logps/chosen": -206.0,
      "logps/rejected": -219.0,
      "loss": 0.6102,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.2275390625,
      "rewards/margins": 0.2333984375,
      "rewards/rejected": -0.4609375,
      "step": 850
    },
    {
      "epoch": 0.03201131562784985,
      "grad_norm": 3.5729242763514457,
      "learning_rate": 1.6002977298101972e-07,
      "logits/chosen": -1.890625,
      "logits/rejected": -1.6875,
      "logps/chosen": -240.0,
      "logps/rejected": -214.0,
      "loss": 0.6207,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.263671875,
      "rewards/margins": 0.150390625,
      "rewards/rejected": -0.4140625,
      "step": 860
    },
    {
      "epoch": 0.03238354022817368,
      "grad_norm": 3.518587492920565,
      "learning_rate": 1.618905842947525e-07,
      "logits/chosen": -1.9921875,
      "logits/rejected": -1.7890625,
      "logps/chosen": -245.0,
      "logps/rejected": -255.0,
      "loss": 0.616,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.3046875,
      "rewards/margins": 0.21875,
      "rewards/rejected": -0.5234375,
      "step": 870
    },
    {
      "epoch": 0.03275576482849751,
      "grad_norm": 3.1066086505075305,
      "learning_rate": 1.637513956084853e-07,
      "logits/chosen": -1.734375,
      "logits/rejected": -1.5078125,
      "logps/chosen": -244.0,
      "logps/rejected": -235.0,
      "loss": 0.6271,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.3359375,
      "rewards/margins": 0.1689453125,
      "rewards/rejected": -0.50390625,
      "step": 880
    },
    {
      "epoch": 0.03312798942882135,
      "grad_norm": 3.4545394807860443,
      "learning_rate": 1.6561220692221807e-07,
      "logits/chosen": -1.6875,
      "logits/rejected": -1.5703125,
      "logps/chosen": -226.0,
      "logps/rejected": -234.0,
      "loss": 0.6115,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.267578125,
      "rewards/margins": 0.251953125,
      "rewards/rejected": -0.51953125,
      "step": 890
    },
    {
      "epoch": 0.03350021402914519,
      "grad_norm": 3.350176984992763,
      "learning_rate": 1.6747301823595087e-07,
      "logits/chosen": -1.828125,
      "logits/rejected": -1.578125,
      "logps/chosen": -231.0,
      "logps/rejected": -240.0,
      "loss": 0.6145,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.287109375,
      "rewards/margins": 0.263671875,
      "rewards/rejected": -0.55078125,
      "step": 900
    },
    {
      "epoch": 0.03387243862946902,
      "grad_norm": 3.1378038779134982,
      "learning_rate": 1.6933382954968363e-07,
      "logits/chosen": -1.8046875,
      "logits/rejected": -1.671875,
      "logps/chosen": -242.0,
      "logps/rejected": -256.0,
      "loss": 0.5996,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.318359375,
      "rewards/margins": 0.310546875,
      "rewards/rejected": -0.62890625,
      "step": 910
    },
    {
      "epoch": 0.03424466322979286,
      "grad_norm": 3.357028329499462,
      "learning_rate": 1.7119464086341645e-07,
      "logits/chosen": -1.8359375,
      "logits/rejected": -1.65625,
      "logps/chosen": -230.0,
      "logps/rejected": -234.0,
      "loss": 0.6105,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.330078125,
      "rewards/margins": 0.185546875,
      "rewards/rejected": -0.515625,
      "step": 920
    },
    {
      "epoch": 0.03461688783011669,
      "grad_norm": 3.33023712813301,
      "learning_rate": 1.7305545217714922e-07,
      "logits/chosen": -1.7890625,
      "logits/rejected": -1.640625,
      "logps/chosen": -245.0,
      "logps/rejected": -238.0,
      "loss": 0.6127,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.28515625,
      "rewards/margins": 0.251953125,
      "rewards/rejected": -0.53515625,
      "step": 930
    },
    {
      "epoch": 0.034989112430440526,
      "grad_norm": 3.6019916606381446,
      "learning_rate": 1.7491626349088201e-07,
      "logits/chosen": -1.75,
      "logits/rejected": -1.625,
      "logps/chosen": -241.0,
      "logps/rejected": -253.0,
      "loss": 0.6067,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.34765625,
      "rewards/margins": 0.2314453125,
      "rewards/rejected": -0.578125,
      "step": 940
    },
    {
      "epoch": 0.03536133703076436,
      "grad_norm": 3.610099066473779,
      "learning_rate": 1.767770748046148e-07,
      "logits/chosen": -1.8671875,
      "logits/rejected": -1.609375,
      "logps/chosen": -240.0,
      "logps/rejected": -258.0,
      "loss": 0.6074,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.333984375,
      "rewards/margins": 0.2470703125,
      "rewards/rejected": -0.58203125,
      "step": 950
    },
    {
      "epoch": 0.0357335616310882,
      "grad_norm": 3.6143368074565587,
      "learning_rate": 1.786378861183476e-07,
      "logits/chosen": -1.765625,
      "logits/rejected": -1.59375,
      "logps/chosen": -236.0,
      "logps/rejected": -248.0,
      "loss": 0.6095,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.36328125,
      "rewards/margins": 0.25,
      "rewards/rejected": -0.61328125,
      "step": 960
    },
    {
      "epoch": 0.036105786231412036,
      "grad_norm": 3.2999778527726265,
      "learning_rate": 1.8049869743208037e-07,
      "logits/chosen": -1.8125,
      "logits/rejected": -1.6640625,
      "logps/chosen": -260.0,
      "logps/rejected": -254.0,
      "loss": 0.6101,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.365234375,
      "rewards/margins": 0.181640625,
      "rewards/rejected": -0.546875,
      "step": 970
    },
    {
      "epoch": 0.03647801083173587,
      "grad_norm": 3.4863623397732133,
      "learning_rate": 1.8235950874581316e-07,
      "logits/chosen": -1.9140625,
      "logits/rejected": -1.765625,
      "logps/chosen": -253.0,
      "logps/rejected": -256.0,
      "loss": 0.5986,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.30859375,
      "rewards/margins": 0.27734375,
      "rewards/rejected": -0.5859375,
      "step": 980
    },
    {
      "epoch": 0.0368502354320597,
      "grad_norm": 3.422012248664094,
      "learning_rate": 1.8422032005954595e-07,
      "logits/chosen": -1.7890625,
      "logits/rejected": -1.6640625,
      "logps/chosen": -264.0,
      "logps/rejected": -278.0,
      "loss": 0.6005,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.365234375,
      "rewards/margins": 0.28515625,
      "rewards/rejected": -0.6484375,
      "step": 990
    },
    {
      "epoch": 0.03722246003238354,
      "grad_norm": 3.773030552032995,
      "learning_rate": 1.8608113137327875e-07,
      "logits/chosen": -1.953125,
      "logits/rejected": -1.78125,
      "logps/chosen": -252.0,
      "logps/rejected": -292.0,
      "loss": 0.5874,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3671875,
      "rewards/margins": 0.333984375,
      "rewards/rejected": -0.703125,
      "step": 1000
    },
    {
      "epoch": 0.037594684632707376,
      "grad_norm": 3.7272462304896066,
      "learning_rate": 1.8794194268701151e-07,
      "logits/chosen": -1.8828125,
      "logits/rejected": -1.6953125,
      "logps/chosen": -252.0,
      "logps/rejected": -272.0,
      "loss": 0.5822,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.380859375,
      "rewards/margins": 0.32421875,
      "rewards/rejected": -0.703125,
      "step": 1010
    },
    {
      "epoch": 0.03796690923303121,
      "grad_norm": 3.9969033243794225,
      "learning_rate": 1.898027540007443e-07,
      "logits/chosen": -1.9296875,
      "logits/rejected": -1.6796875,
      "logps/chosen": -254.0,
      "logps/rejected": -264.0,
      "loss": 0.5896,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.466796875,
      "rewards/margins": 0.40234375,
      "rewards/rejected": -0.8671875,
      "step": 1020
    },
    {
      "epoch": 0.03833913383335505,
      "grad_norm": 4.018336869539786,
      "learning_rate": 1.916635653144771e-07,
      "logits/chosen": -1.921875,
      "logits/rejected": -1.734375,
      "logps/chosen": -251.0,
      "logps/rejected": -288.0,
      "loss": 0.5857,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.44921875,
      "rewards/margins": 0.439453125,
      "rewards/rejected": -0.88671875,
      "step": 1030
    },
    {
      "epoch": 0.03871135843367888,
      "grad_norm": 3.658759409322556,
      "learning_rate": 1.935243766282099e-07,
      "logits/chosen": -1.921875,
      "logits/rejected": -1.6484375,
      "logps/chosen": -244.0,
      "logps/rejected": -280.0,
      "loss": 0.5854,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.482421875,
      "rewards/margins": 0.392578125,
      "rewards/rejected": -0.875,
      "step": 1040
    },
    {
      "epoch": 0.039083583034002715,
      "grad_norm": 4.087203274847401,
      "learning_rate": 1.9538518794194266e-07,
      "logits/chosen": -1.84375,
      "logits/rejected": -1.6796875,
      "logps/chosen": -266.0,
      "logps/rejected": -308.0,
      "loss": 0.5914,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.546875,
      "rewards/margins": 0.34375,
      "rewards/rejected": -0.890625,
      "step": 1050
    },
    {
      "epoch": 0.03945580763432655,
      "grad_norm": 4.249709258704824,
      "learning_rate": 1.9724599925567548e-07,
      "logits/chosen": -1.9296875,
      "logits/rejected": -1.703125,
      "logps/chosen": -251.0,
      "logps/rejected": -286.0,
      "loss": 0.5752,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.490234375,
      "rewards/margins": 0.376953125,
      "rewards/rejected": -0.8671875,
      "step": 1060
    },
    {
      "epoch": 0.03982803223465039,
      "grad_norm": 5.021425780408093,
      "learning_rate": 1.9910681056940825e-07,
      "logits/chosen": -1.921875,
      "logits/rejected": -1.765625,
      "logps/chosen": -260.0,
      "logps/rejected": -318.0,
      "loss": 0.5647,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.490234375,
      "rewards/margins": 0.54296875,
      "rewards/rejected": -1.03125,
      "step": 1070
    },
    {
      "epoch": 0.040200256834974225,
      "grad_norm": 4.594169196099619,
      "learning_rate": 2.0096762188314104e-07,
      "logits/chosen": -1.9765625,
      "logits/rejected": -1.7578125,
      "logps/chosen": -278.0,
      "logps/rejected": -282.0,
      "loss": 0.5768,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.546875,
      "rewards/margins": 0.3671875,
      "rewards/rejected": -0.9140625,
      "step": 1080
    },
    {
      "epoch": 0.04057248143529806,
      "grad_norm": 4.183724972388384,
      "learning_rate": 2.028284331968738e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.828125,
      "logps/chosen": -270.0,
      "logps/rejected": -302.0,
      "loss": 0.5772,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.498046875,
      "rewards/margins": 0.423828125,
      "rewards/rejected": -0.921875,
      "step": 1090
    },
    {
      "epoch": 0.04094470603562189,
      "grad_norm": 4.443785835104889,
      "learning_rate": 2.0468924451060663e-07,
      "logits/chosen": -1.9609375,
      "logits/rejected": -1.796875,
      "logps/chosen": -250.0,
      "logps/rejected": -284.0,
      "loss": 0.5555,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.5078125,
      "rewards/margins": 0.5234375,
      "rewards/rejected": -1.03125,
      "step": 1100
    },
    {
      "epoch": 0.04131693063594573,
      "grad_norm": 4.41337757685986,
      "learning_rate": 2.065500558243394e-07,
      "logits/chosen": -1.7734375,
      "logits/rejected": -1.609375,
      "logps/chosen": -255.0,
      "logps/rejected": -290.0,
      "loss": 0.5517,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.5390625,
      "rewards/margins": 0.53125,
      "rewards/rejected": -1.0703125,
      "step": 1110
    },
    {
      "epoch": 0.041689155236269565,
      "grad_norm": 4.934770052630334,
      "learning_rate": 2.084108671380722e-07,
      "logits/chosen": -1.9609375,
      "logits/rejected": -1.7734375,
      "logps/chosen": -260.0,
      "logps/rejected": -280.0,
      "loss": 0.5848,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.6484375,
      "rewards/margins": 0.40234375,
      "rewards/rejected": -1.046875,
      "step": 1120
    },
    {
      "epoch": 0.0420613798365934,
      "grad_norm": 5.28511671830687,
      "learning_rate": 2.1027167845180498e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.7265625,
      "logps/chosen": -258.0,
      "logps/rejected": -306.0,
      "loss": 0.5603,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.5,
      "rewards/margins": 0.57421875,
      "rewards/rejected": -1.0703125,
      "step": 1130
    },
    {
      "epoch": 0.04243360443691724,
      "grad_norm": 4.614773115212574,
      "learning_rate": 2.1213248976553777e-07,
      "logits/chosen": -1.9453125,
      "logits/rejected": -1.875,
      "logps/chosen": -272.0,
      "logps/rejected": -290.0,
      "loss": 0.5688,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.63671875,
      "rewards/margins": 0.400390625,
      "rewards/rejected": -1.0390625,
      "step": 1140
    },
    {
      "epoch": 0.04280582903724107,
      "grad_norm": 4.203975435198822,
      "learning_rate": 2.1399330107927054e-07,
      "logits/chosen": -1.984375,
      "logits/rejected": -1.7265625,
      "logps/chosen": -252.0,
      "logps/rejected": -284.0,
      "loss": 0.5613,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.625,
      "rewards/margins": 0.43359375,
      "rewards/rejected": -1.0546875,
      "step": 1150
    },
    {
      "epoch": 0.043178053637564905,
      "grad_norm": 6.231922851336398,
      "learning_rate": 2.1585411239300333e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.8046875,
      "logps/chosen": -276.0,
      "logps/rejected": -310.0,
      "loss": 0.5488,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.6484375,
      "rewards/margins": 0.470703125,
      "rewards/rejected": -1.1171875,
      "step": 1160
    },
    {
      "epoch": 0.04355027823788874,
      "grad_norm": 5.089260370092706,
      "learning_rate": 2.1771492370673613e-07,
      "logits/chosen": -1.8515625,
      "logits/rejected": -1.6171875,
      "logps/chosen": -258.0,
      "logps/rejected": -302.0,
      "loss": 0.5681,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.578125,
      "rewards/margins": 0.486328125,
      "rewards/rejected": -1.0625,
      "step": 1170
    },
    {
      "epoch": 0.04392250283821258,
      "grad_norm": 6.048176952853639,
      "learning_rate": 2.1957573502046892e-07,
      "logits/chosen": -1.9609375,
      "logits/rejected": -1.6484375,
      "logps/chosen": -262.0,
      "logps/rejected": -308.0,
      "loss": 0.5409,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.56640625,
      "rewards/margins": 0.515625,
      "rewards/rejected": -1.0859375,
      "step": 1180
    },
    {
      "epoch": 0.044294727438536415,
      "grad_norm": 4.6617071681898015,
      "learning_rate": 2.214365463342017e-07,
      "logits/chosen": -1.875,
      "logits/rejected": -1.734375,
      "logps/chosen": -282.0,
      "logps/rejected": -306.0,
      "loss": 0.5535,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.734375,
      "rewards/margins": 0.44140625,
      "rewards/rejected": -1.171875,
      "step": 1190
    },
    {
      "epoch": 0.04466695203886025,
      "grad_norm": 5.166475496771978,
      "learning_rate": 2.2329735764793448e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -1.875,
      "logps/chosen": -266.0,
      "logps/rejected": -320.0,
      "loss": 0.5462,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.69921875,
      "rewards/margins": 0.53125,
      "rewards/rejected": -1.2265625,
      "step": 1200
    },
    {
      "epoch": 0.04503917663918408,
      "grad_norm": 4.525981711504696,
      "learning_rate": 2.2515816896166727e-07,
      "logits/chosen": -1.9765625,
      "logits/rejected": -1.859375,
      "logps/chosen": -284.0,
      "logps/rejected": -312.0,
      "loss": 0.554,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7421875,
      "rewards/margins": 0.44921875,
      "rewards/rejected": -1.1875,
      "step": 1210
    },
    {
      "epoch": 0.04541140123950792,
      "grad_norm": 5.925896363549941,
      "learning_rate": 2.2701898027540007e-07,
      "logits/chosen": -1.90625,
      "logits/rejected": -1.78125,
      "logps/chosen": -274.0,
      "logps/rejected": -318.0,
      "loss": 0.5387,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.68359375,
      "rewards/margins": 0.4296875,
      "rewards/rejected": -1.1171875,
      "step": 1220
    },
    {
      "epoch": 0.045783625839831754,
      "grad_norm": 6.202002627005279,
      "learning_rate": 2.2887979158913283e-07,
      "logits/chosen": -1.953125,
      "logits/rejected": -1.8515625,
      "logps/chosen": -288.0,
      "logps/rejected": -326.0,
      "loss": 0.5415,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.71875,
      "rewards/margins": 0.578125,
      "rewards/rejected": -1.296875,
      "step": 1230
    },
    {
      "epoch": 0.04615585044015559,
      "grad_norm": 4.8170666042847685,
      "learning_rate": 2.3074060290286565e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.7890625,
      "logps/chosen": -304.0,
      "logps/rejected": -328.0,
      "loss": 0.5716,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.75,
      "rewards/margins": 0.392578125,
      "rewards/rejected": -1.140625,
      "step": 1240
    },
    {
      "epoch": 0.04652807504047943,
      "grad_norm": 5.212397653414338,
      "learning_rate": 2.3260141421659842e-07,
      "logits/chosen": -1.9921875,
      "logits/rejected": -1.8515625,
      "logps/chosen": -290.0,
      "logps/rejected": -342.0,
      "loss": 0.5463,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.70703125,
      "rewards/margins": 0.578125,
      "rewards/rejected": -1.2890625,
      "step": 1250
    },
    {
      "epoch": 0.046900299640803264,
      "grad_norm": 5.211063856071835,
      "learning_rate": 2.3446222553033121e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.9296875,
      "logps/chosen": -290.0,
      "logps/rejected": -324.0,
      "loss": 0.5614,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.76171875,
      "rewards/margins": 0.490234375,
      "rewards/rejected": -1.25,
      "step": 1260
    },
    {
      "epoch": 0.047272524241127094,
      "grad_norm": 6.047653977965838,
      "learning_rate": 2.3632303684406398e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.84375,
      "logps/chosen": -304.0,
      "logps/rejected": -316.0,
      "loss": 0.5414,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.83203125,
      "rewards/margins": 0.486328125,
      "rewards/rejected": -1.3203125,
      "step": 1270
    },
    {
      "epoch": 0.04764474884145093,
      "grad_norm": 5.0728471898719665,
      "learning_rate": 2.381838481577968e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.8203125,
      "logps/chosen": -290.0,
      "logps/rejected": -318.0,
      "loss": 0.5248,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.72265625,
      "rewards/margins": 0.5546875,
      "rewards/rejected": -1.2734375,
      "step": 1280
    },
    {
      "epoch": 0.04801697344177477,
      "grad_norm": 5.160176583644704,
      "learning_rate": 2.4004465947152957e-07,
      "logits/chosen": -1.984375,
      "logits/rejected": -1.8046875,
      "logps/chosen": -292.0,
      "logps/rejected": -322.0,
      "loss": 0.535,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.734375,
      "rewards/margins": 0.54296875,
      "rewards/rejected": -1.28125,
      "step": 1290
    },
    {
      "epoch": 0.048389198042098604,
      "grad_norm": 6.0652642687605205,
      "learning_rate": 2.4190547078526233e-07,
      "logits/chosen": -1.984375,
      "logits/rejected": -1.90625,
      "logps/chosen": -298.0,
      "logps/rejected": -346.0,
      "loss": 0.5378,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.75,
      "rewards/margins": 0.6875,
      "rewards/rejected": -1.4375,
      "step": 1300
    },
    {
      "epoch": 0.04876142264242244,
      "grad_norm": 5.0688024094238235,
      "learning_rate": 2.4376628209899515e-07,
      "logits/chosen": -1.859375,
      "logits/rejected": -1.640625,
      "logps/chosen": -280.0,
      "logps/rejected": -306.0,
      "loss": 0.5284,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.75,
      "rewards/margins": 0.60546875,
      "rewards/rejected": -1.3515625,
      "step": 1310
    },
    {
      "epoch": 0.04913364724274627,
      "grad_norm": 7.429927535435445,
      "learning_rate": 2.456270934127279e-07,
      "logits/chosen": -2.046875,
      "logits/rejected": -1.8828125,
      "logps/chosen": -302.0,
      "logps/rejected": -344.0,
      "loss": 0.5351,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.8359375,
      "rewards/margins": 0.58984375,
      "rewards/rejected": -1.4296875,
      "step": 1320
    },
    {
      "epoch": 0.04950587184307011,
      "grad_norm": 9.289116253510421,
      "learning_rate": 2.4748790472646074e-07,
      "logits/chosen": -2.046875,
      "logits/rejected": -1.8671875,
      "logps/chosen": -292.0,
      "logps/rejected": -324.0,
      "loss": 0.5566,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.91015625,
      "rewards/margins": 0.396484375,
      "rewards/rejected": -1.3046875,
      "step": 1330
    },
    {
      "epoch": 0.04987809644339394,
      "grad_norm": 6.535508702717331,
      "learning_rate": 2.493487160401935e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -1.8125,
      "logps/chosen": -282.0,
      "logps/rejected": -356.0,
      "loss": 0.5269,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.80078125,
      "rewards/margins": 0.66796875,
      "rewards/rejected": -1.46875,
      "step": 1340
    },
    {
      "epoch": 0.05025032104371778,
      "grad_norm": 5.263936636627017,
      "learning_rate": 2.5120952735392633e-07,
      "logits/chosen": -1.9453125,
      "logits/rejected": -1.8671875,
      "logps/chosen": -304.0,
      "logps/rejected": -354.0,
      "loss": 0.559,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.9140625,
      "rewards/margins": 0.6484375,
      "rewards/rejected": -1.5625,
      "step": 1350
    },
    {
      "epoch": 0.05062254564404162,
      "grad_norm": 5.121910650454828,
      "learning_rate": 2.530703386676591e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -2.0625,
      "logps/chosen": -320.0,
      "logps/rejected": -348.0,
      "loss": 0.544,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9140625,
      "rewards/margins": 0.498046875,
      "rewards/rejected": -1.4140625,
      "step": 1360
    },
    {
      "epoch": 0.05099477024436545,
      "grad_norm": 6.301558218492348,
      "learning_rate": 2.5493114998139186e-07,
      "logits/chosen": -1.9609375,
      "logits/rejected": -1.6875,
      "logps/chosen": -292.0,
      "logps/rejected": -344.0,
      "loss": 0.5327,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.91796875,
      "rewards/margins": 0.60546875,
      "rewards/rejected": -1.5234375,
      "step": 1370
    },
    {
      "epoch": 0.05136699484468928,
      "grad_norm": 4.441046566163725,
      "learning_rate": 2.5679196129512463e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.9140625,
      "logps/chosen": -294.0,
      "logps/rejected": -326.0,
      "loss": 0.5588,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.76171875,
      "rewards/margins": 0.58984375,
      "rewards/rejected": -1.3515625,
      "step": 1380
    },
    {
      "epoch": 0.05173921944501312,
      "grad_norm": 5.781409043436412,
      "learning_rate": 2.5865277260885745e-07,
      "logits/chosen": -1.9296875,
      "logits/rejected": -1.8125,
      "logps/chosen": -286.0,
      "logps/rejected": -344.0,
      "loss": 0.5492,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.76953125,
      "rewards/margins": 0.63671875,
      "rewards/rejected": -1.40625,
      "step": 1390
    },
    {
      "epoch": 0.052111444045336956,
      "grad_norm": 8.880757900654917,
      "learning_rate": 2.6051358392259027e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.9140625,
      "logps/chosen": -316.0,
      "logps/rejected": -346.0,
      "loss": 0.5266,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.96875,
      "rewards/margins": 0.48046875,
      "rewards/rejected": -1.453125,
      "step": 1400
    },
    {
      "epoch": 0.05248366864566079,
      "grad_norm": 8.007897346142043,
      "learning_rate": 2.6237439523632303e-07,
      "logits/chosen": -1.9296875,
      "logits/rejected": -1.8828125,
      "logps/chosen": -318.0,
      "logps/rejected": -342.0,
      "loss": 0.5289,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9140625,
      "rewards/margins": 0.57421875,
      "rewards/rejected": -1.4921875,
      "step": 1410
    },
    {
      "epoch": 0.05285589324598463,
      "grad_norm": 6.118868418605038,
      "learning_rate": 2.642352065500558e-07,
      "logits/chosen": -1.9921875,
      "logits/rejected": -1.8984375,
      "logps/chosen": -300.0,
      "logps/rejected": -338.0,
      "loss": 0.5264,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.90625,
      "rewards/margins": 0.609375,
      "rewards/rejected": -1.515625,
      "step": 1420
    },
    {
      "epoch": 0.05322811784630846,
      "grad_norm": 6.428728532308711,
      "learning_rate": 2.660960178637886e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.8359375,
      "logps/chosen": -330.0,
      "logps/rejected": -356.0,
      "loss": 0.5228,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.0546875,
      "rewards/margins": 0.578125,
      "rewards/rejected": -1.625,
      "step": 1430
    },
    {
      "epoch": 0.053600342446632296,
      "grad_norm": 5.652932554230279,
      "learning_rate": 2.679568291775214e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.8203125,
      "logps/chosen": -322.0,
      "logps/rejected": -368.0,
      "loss": 0.5143,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1328125,
      "rewards/margins": 0.65625,
      "rewards/rejected": -1.7890625,
      "step": 1440
    },
    {
      "epoch": 0.05397256704695613,
      "grad_norm": 5.719576260298013,
      "learning_rate": 2.6981764049125415e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.8046875,
      "logps/chosen": -304.0,
      "logps/rejected": -358.0,
      "loss": 0.5143,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.984375,
      "rewards/margins": 0.609375,
      "rewards/rejected": -1.59375,
      "step": 1450
    },
    {
      "epoch": 0.05434479164727997,
      "grad_norm": 6.262448083538663,
      "learning_rate": 2.716784518049869e-07,
      "logits/chosen": -1.9375,
      "logits/rejected": -1.8359375,
      "logps/chosen": -324.0,
      "logps/rejected": -364.0,
      "loss": 0.5308,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.0703125,
      "rewards/margins": 0.5859375,
      "rewards/rejected": -1.65625,
      "step": 1460
    },
    {
      "epoch": 0.054717016247603806,
      "grad_norm": 6.800185364814221,
      "learning_rate": 2.735392631187198e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.9375,
      "logps/chosen": -320.0,
      "logps/rejected": -354.0,
      "loss": 0.524,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.015625,
      "rewards/margins": 0.61328125,
      "rewards/rejected": -1.6328125,
      "step": 1470
    },
    {
      "epoch": 0.05508924084792764,
      "grad_norm": 6.262866266124628,
      "learning_rate": 2.7540007443245256e-07,
      "logits/chosen": -1.984375,
      "logits/rejected": -1.90625,
      "logps/chosen": -328.0,
      "logps/rejected": -376.0,
      "loss": 0.5214,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.078125,
      "rewards/margins": 0.61328125,
      "rewards/rejected": -1.6953125,
      "step": 1480
    },
    {
      "epoch": 0.05546146544825147,
      "grad_norm": 6.159194589545578,
      "learning_rate": 2.7726088574618533e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.8828125,
      "logps/chosen": -298.0,
      "logps/rejected": -372.0,
      "loss": 0.5177,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.0703125,
      "rewards/margins": 0.7265625,
      "rewards/rejected": -1.796875,
      "step": 1490
    },
    {
      "epoch": 0.05583369004857531,
      "grad_norm": 6.61255057502829,
      "learning_rate": 2.791216970599181e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -1.84375,
      "logps/chosen": -322.0,
      "logps/rejected": -364.0,
      "loss": 0.529,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.140625,
      "rewards/margins": 0.72265625,
      "rewards/rejected": -1.8671875,
      "step": 1500
    },
    {
      "epoch": 0.056205914648899145,
      "grad_norm": 6.521678037820579,
      "learning_rate": 2.809825083736509e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.828125,
      "logps/chosen": -326.0,
      "logps/rejected": -414.0,
      "loss": 0.5171,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1640625,
      "rewards/margins": 0.78515625,
      "rewards/rejected": -1.9453125,
      "step": 1510
    },
    {
      "epoch": 0.05657813924922298,
      "grad_norm": 6.382389857540072,
      "learning_rate": 2.828433196873837e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.7421875,
      "logps/chosen": -308.0,
      "logps/rejected": -358.0,
      "loss": 0.5295,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.015625,
      "rewards/margins": 0.62890625,
      "rewards/rejected": -1.640625,
      "step": 1520
    },
    {
      "epoch": 0.05695036384954682,
      "grad_norm": 6.6754071966525075,
      "learning_rate": 2.8470413100111645e-07,
      "logits/chosen": -1.9375,
      "logits/rejected": -1.8359375,
      "logps/chosen": -306.0,
      "logps/rejected": -358.0,
      "loss": 0.5067,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.96484375,
      "rewards/margins": 0.58984375,
      "rewards/rejected": -1.5546875,
      "step": 1530
    },
    {
      "epoch": 0.057322588449870655,
      "grad_norm": 5.750582446675947,
      "learning_rate": 2.865649423148492e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -1.9765625,
      "logps/chosen": -328.0,
      "logps/rejected": -388.0,
      "loss": 0.5245,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.171875,
      "rewards/margins": 0.62109375,
      "rewards/rejected": -1.796875,
      "step": 1540
    },
    {
      "epoch": 0.057694813050194485,
      "grad_norm": 5.519401472525661,
      "learning_rate": 2.884257536285821e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -1.90625,
      "logps/chosen": -318.0,
      "logps/rejected": -360.0,
      "loss": 0.5128,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.0,
      "rewards/margins": 0.78515625,
      "rewards/rejected": -1.7890625,
      "step": 1550
    },
    {
      "epoch": 0.05806703765051832,
      "grad_norm": 7.0395643870356155,
      "learning_rate": 2.9028656494231485e-07,
      "logits/chosen": -2.046875,
      "logits/rejected": -1.8046875,
      "logps/chosen": -314.0,
      "logps/rejected": -362.0,
      "loss": 0.5345,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.1328125,
      "rewards/margins": 0.68359375,
      "rewards/rejected": -1.8125,
      "step": 1560
    },
    {
      "epoch": 0.05843926225084216,
      "grad_norm": 7.59324376246413,
      "learning_rate": 2.921473762560476e-07,
      "logits/chosen": -1.96875,
      "logits/rejected": -1.7265625,
      "logps/chosen": -310.0,
      "logps/rejected": -358.0,
      "loss": 0.5125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.0859375,
      "rewards/margins": 0.66796875,
      "rewards/rejected": -1.7578125,
      "step": 1570
    },
    {
      "epoch": 0.058811486851165995,
      "grad_norm": 6.010996470218349,
      "learning_rate": 2.940081875697804e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.8984375,
      "logps/chosen": -348.0,
      "logps/rejected": -386.0,
      "loss": 0.5127,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.15625,
      "rewards/margins": 0.69140625,
      "rewards/rejected": -1.8515625,
      "step": 1580
    },
    {
      "epoch": 0.05918371145148983,
      "grad_norm": 10.675044705323973,
      "learning_rate": 2.958689988835132e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.8671875,
      "logps/chosen": -332.0,
      "logps/rejected": -418.0,
      "loss": 0.503,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.2421875,
      "rewards/margins": 0.72265625,
      "rewards/rejected": -1.96875,
      "step": 1590
    },
    {
      "epoch": 0.05955593605181366,
      "grad_norm": 6.909365129881667,
      "learning_rate": 2.97729810197246e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -1.796875,
      "logps/chosen": -296.0,
      "logps/rejected": -362.0,
      "loss": 0.5093,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.078125,
      "rewards/margins": 0.7734375,
      "rewards/rejected": -1.8515625,
      "step": 1600
    },
    {
      "epoch": 0.0599281606521375,
      "grad_norm": 7.2286689283567025,
      "learning_rate": 2.9959062151097874e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.890625,
      "logps/chosen": -324.0,
      "logps/rejected": -390.0,
      "loss": 0.516,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.1875,
      "rewards/margins": 0.73828125,
      "rewards/rejected": -1.9296875,
      "step": 1610
    },
    {
      "epoch": 0.060300385252461335,
      "grad_norm": 6.389414077783402,
      "learning_rate": 3.0145143282471156e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.90625,
      "logps/chosen": -334.0,
      "logps/rejected": -394.0,
      "loss": 0.5133,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.15625,
      "rewards/margins": 0.68359375,
      "rewards/rejected": -1.8359375,
      "step": 1620
    },
    {
      "epoch": 0.06067260985278517,
      "grad_norm": 7.026427787732427,
      "learning_rate": 3.033122441384444e-07,
      "logits/chosen": -1.9140625,
      "logits/rejected": -1.7421875,
      "logps/chosen": -322.0,
      "logps/rejected": -410.0,
      "loss": 0.509,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.2734375,
      "rewards/margins": 0.78515625,
      "rewards/rejected": -2.0625,
      "step": 1630
    },
    {
      "epoch": 0.06104483445310901,
      "grad_norm": 6.28668005246674,
      "learning_rate": 3.0517305545217715e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.875,
      "logps/chosen": -330.0,
      "logps/rejected": -378.0,
      "loss": 0.5066,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.1796875,
      "rewards/margins": 0.75390625,
      "rewards/rejected": -1.9375,
      "step": 1640
    },
    {
      "epoch": 0.061417059053432844,
      "grad_norm": 7.4624686975863215,
      "learning_rate": 3.070338667659099e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.8984375,
      "logps/chosen": -336.0,
      "logps/rejected": -388.0,
      "loss": 0.5289,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.2109375,
      "rewards/margins": 0.67578125,
      "rewards/rejected": -1.8828125,
      "step": 1650
    },
    {
      "epoch": 0.061789283653756674,
      "grad_norm": 7.17605418888808,
      "learning_rate": 3.088946780796427e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.8671875,
      "logps/chosen": -344.0,
      "logps/rejected": -388.0,
      "loss": 0.4976,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.2578125,
      "rewards/margins": 0.67578125,
      "rewards/rejected": -1.9375,
      "step": 1660
    },
    {
      "epoch": 0.06216150825408051,
      "grad_norm": 6.6429389347340475,
      "learning_rate": 3.107554893933755e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.8828125,
      "logps/chosen": -356.0,
      "logps/rejected": -406.0,
      "loss": 0.5242,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.3046875,
      "rewards/margins": 0.8125,
      "rewards/rejected": -2.125,
      "step": 1670
    },
    {
      "epoch": 0.06253373285440435,
      "grad_norm": 5.648099758022945,
      "learning_rate": 3.1261630070710827e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.90625,
      "logps/chosen": -348.0,
      "logps/rejected": -392.0,
      "loss": 0.5098,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.2890625,
      "rewards/margins": 0.66796875,
      "rewards/rejected": -1.953125,
      "step": 1680
    },
    {
      "epoch": 0.06290595745472818,
      "grad_norm": 6.071529564161894,
      "learning_rate": 3.144771120208411e-07,
      "logits/chosen": -1.984375,
      "logits/rejected": -1.859375,
      "logps/chosen": -340.0,
      "logps/rejected": -392.0,
      "loss": 0.4993,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.1640625,
      "rewards/margins": 0.8515625,
      "rewards/rejected": -2.015625,
      "step": 1690
    },
    {
      "epoch": 0.06327818205505202,
      "grad_norm": 8.250495844267443,
      "learning_rate": 3.1633792333457385e-07,
      "logits/chosen": -1.96875,
      "logits/rejected": -1.8828125,
      "logps/chosen": -328.0,
      "logps/rejected": -400.0,
      "loss": 0.5055,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.171875,
      "rewards/margins": 0.83203125,
      "rewards/rejected": -2.0,
      "step": 1700
    },
    {
      "epoch": 0.06365040665537586,
      "grad_norm": 8.940927620579163,
      "learning_rate": 3.181987346483067e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.953125,
      "logps/chosen": -340.0,
      "logps/rejected": -394.0,
      "loss": 0.5153,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.3125,
      "rewards/margins": 0.69140625,
      "rewards/rejected": -2.0,
      "step": 1710
    },
    {
      "epoch": 0.0640226312556997,
      "grad_norm": 9.396406882964524,
      "learning_rate": 3.2005954596203944e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.875,
      "logps/chosen": -328.0,
      "logps/rejected": -398.0,
      "loss": 0.4935,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.2890625,
      "rewards/margins": 0.81640625,
      "rewards/rejected": -2.109375,
      "step": 1720
    },
    {
      "epoch": 0.06439485585602353,
      "grad_norm": 6.513402618642106,
      "learning_rate": 3.219203572757722e-07,
      "logits/chosen": -1.984375,
      "logits/rejected": -1.78125,
      "logps/chosen": -336.0,
      "logps/rejected": -396.0,
      "loss": 0.5206,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.3828125,
      "rewards/margins": 0.7265625,
      "rewards/rejected": -2.109375,
      "step": 1730
    },
    {
      "epoch": 0.06476708045634735,
      "grad_norm": 7.535877645135211,
      "learning_rate": 3.23781168589505e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -1.875,
      "logps/chosen": -332.0,
      "logps/rejected": -410.0,
      "loss": 0.4987,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.25,
      "rewards/margins": 0.92578125,
      "rewards/rejected": -2.171875,
      "step": 1740
    },
    {
      "epoch": 0.06513930505667119,
      "grad_norm": 7.118568438258204,
      "learning_rate": 3.256419799032378e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -1.90625,
      "logps/chosen": -338.0,
      "logps/rejected": -382.0,
      "loss": 0.5259,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.2109375,
      "rewards/margins": 0.61328125,
      "rewards/rejected": -1.828125,
      "step": 1750
    },
    {
      "epoch": 0.06551152965699503,
      "grad_norm": 6.79821186716964,
      "learning_rate": 3.275027912169706e-07,
      "logits/chosen": -1.984375,
      "logits/rejected": -1.875,
      "logps/chosen": -360.0,
      "logps/rejected": -432.0,
      "loss": 0.4977,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.2734375,
      "rewards/margins": 0.84765625,
      "rewards/rejected": -2.125,
      "step": 1760
    },
    {
      "epoch": 0.06588375425731886,
      "grad_norm": 7.392766204245853,
      "learning_rate": 3.293636025307034e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.96875,
      "logps/chosen": -374.0,
      "logps/rejected": -430.0,
      "loss": 0.5148,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.5625,
      "rewards/margins": 0.76171875,
      "rewards/rejected": -2.328125,
      "step": 1770
    },
    {
      "epoch": 0.0662559788576427,
      "grad_norm": 9.033848286994207,
      "learning_rate": 3.3122441384443615e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.8359375,
      "logps/chosen": -338.0,
      "logps/rejected": -386.0,
      "loss": 0.5182,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.265625,
      "rewards/margins": 0.734375,
      "rewards/rejected": -2.0,
      "step": 1780
    },
    {
      "epoch": 0.06662820345796654,
      "grad_norm": 5.5429144459422535,
      "learning_rate": 3.3308522515816897e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -1.859375,
      "logps/chosen": -328.0,
      "logps/rejected": -370.0,
      "loss": 0.505,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.1953125,
      "rewards/margins": 0.62109375,
      "rewards/rejected": -1.8125,
      "step": 1790
    },
    {
      "epoch": 0.06700042805829037,
      "grad_norm": 6.573471693245076,
      "learning_rate": 3.3494603647190173e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.0,
      "logps/chosen": -362.0,
      "logps/rejected": -406.0,
      "loss": 0.5115,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.390625,
      "rewards/margins": 0.734375,
      "rewards/rejected": -2.125,
      "step": 1800
    },
    {
      "epoch": 0.06737265265861421,
      "grad_norm": 6.396822590192353,
      "learning_rate": 3.368068477856345e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.796875,
      "logps/chosen": -340.0,
      "logps/rejected": -390.0,
      "loss": 0.5014,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.3984375,
      "rewards/margins": 0.6875,
      "rewards/rejected": -2.09375,
      "step": 1810
    },
    {
      "epoch": 0.06774487725893805,
      "grad_norm": 6.645150508941034,
      "learning_rate": 3.3866765909936727e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.015625,
      "logps/chosen": -356.0,
      "logps/rejected": -408.0,
      "loss": 0.5072,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2734375,
      "rewards/margins": 0.875,
      "rewards/rejected": -2.15625,
      "step": 1820
    },
    {
      "epoch": 0.06811710185926188,
      "grad_norm": 5.712015084898278,
      "learning_rate": 3.4052847041310014e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.671875,
      "logps/chosen": -328.0,
      "logps/rejected": -392.0,
      "loss": 0.5117,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.328125,
      "rewards/margins": 0.7734375,
      "rewards/rejected": -2.09375,
      "step": 1830
    },
    {
      "epoch": 0.06848932645958572,
      "grad_norm": 7.762289886406715,
      "learning_rate": 3.423892817268329e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.8515625,
      "logps/chosen": -360.0,
      "logps/rejected": -430.0,
      "loss": 0.5013,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.3203125,
      "rewards/margins": 0.8984375,
      "rewards/rejected": -2.21875,
      "step": 1840
    },
    {
      "epoch": 0.06886155105990954,
      "grad_norm": 6.774807440666504,
      "learning_rate": 3.442500930405657e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -2.09375,
      "logps/chosen": -354.0,
      "logps/rejected": -412.0,
      "loss": 0.4986,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.3828125,
      "rewards/margins": 0.77734375,
      "rewards/rejected": -2.15625,
      "step": 1850
    },
    {
      "epoch": 0.06923377566023338,
      "grad_norm": 7.493077658036802,
      "learning_rate": 3.4611090435429844e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -2.0625,
      "logps/chosen": -344.0,
      "logps/rejected": -408.0,
      "loss": 0.504,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.234375,
      "rewards/margins": 0.77734375,
      "rewards/rejected": -2.015625,
      "step": 1860
    },
    {
      "epoch": 0.06960600026055722,
      "grad_norm": 6.958512350489221,
      "learning_rate": 3.4797171566803126e-07,
      "logits/chosen": -1.953125,
      "logits/rejected": -1.875,
      "logps/chosen": -350.0,
      "logps/rejected": -390.0,
      "loss": 0.4932,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.2578125,
      "rewards/margins": 0.66796875,
      "rewards/rejected": -1.9296875,
      "step": 1870
    },
    {
      "epoch": 0.06997822486088105,
      "grad_norm": 7.885731015057873,
      "learning_rate": 3.4983252698176403e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.875,
      "logps/chosen": -342.0,
      "logps/rejected": -396.0,
      "loss": 0.5122,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.28125,
      "rewards/margins": 0.734375,
      "rewards/rejected": -2.015625,
      "step": 1880
    },
    {
      "epoch": 0.07035044946120489,
      "grad_norm": 6.736941588308517,
      "learning_rate": 3.516933382954968e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -2.09375,
      "logps/chosen": -346.0,
      "logps/rejected": -406.0,
      "loss": 0.4863,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.3125,
      "rewards/margins": 0.73046875,
      "rewards/rejected": -2.046875,
      "step": 1890
    },
    {
      "epoch": 0.07072267406152873,
      "grad_norm": 6.066456394634817,
      "learning_rate": 3.535541496092296e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.140625,
      "logps/chosen": -370.0,
      "logps/rejected": -418.0,
      "loss": 0.4998,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 0.703125,
      "rewards/rejected": -2.140625,
      "step": 1900
    },
    {
      "epoch": 0.07109489866185256,
      "grad_norm": 7.72196189725634,
      "learning_rate": 3.5541496092296243e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.109375,
      "logps/chosen": -360.0,
      "logps/rejected": -430.0,
      "loss": 0.5073,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.484375,
      "rewards/margins": 0.75,
      "rewards/rejected": -2.234375,
      "step": 1910
    },
    {
      "epoch": 0.0714671232621764,
      "grad_norm": 9.603374143414033,
      "learning_rate": 3.572757722366952e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.046875,
      "logps/chosen": -320.0,
      "logps/rejected": -388.0,
      "loss": 0.5206,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.296875,
      "rewards/margins": 0.734375,
      "rewards/rejected": -2.03125,
      "step": 1920
    },
    {
      "epoch": 0.07183934786250024,
      "grad_norm": 6.845397522849816,
      "learning_rate": 3.5913658355042797e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.03125,
      "logps/chosen": -354.0,
      "logps/rejected": -402.0,
      "loss": 0.5157,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 0.6015625,
      "rewards/rejected": -2.046875,
      "step": 1930
    },
    {
      "epoch": 0.07221157246282407,
      "grad_norm": 7.376556301193092,
      "learning_rate": 3.6099739486416073e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -2.015625,
      "logps/chosen": -386.0,
      "logps/rejected": -424.0,
      "loss": 0.5213,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 0.65234375,
      "rewards/rejected": -2.25,
      "step": 1940
    },
    {
      "epoch": 0.07258379706314791,
      "grad_norm": 5.715526417393285,
      "learning_rate": 3.6285820617789355e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.9609375,
      "logps/chosen": -366.0,
      "logps/rejected": -414.0,
      "loss": 0.521,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.46875,
      "rewards/margins": 0.66796875,
      "rewards/rejected": -2.140625,
      "step": 1950
    },
    {
      "epoch": 0.07295602166347175,
      "grad_norm": 7.755405479792618,
      "learning_rate": 3.647190174916263e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -2.109375,
      "logps/chosen": -358.0,
      "logps/rejected": -418.0,
      "loss": 0.5071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.34375,
      "rewards/margins": 0.64453125,
      "rewards/rejected": -1.9921875,
      "step": 1960
    },
    {
      "epoch": 0.07332824626379557,
      "grad_norm": 7.453501236107824,
      "learning_rate": 3.665798288053591e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -1.90625,
      "logps/chosen": -324.0,
      "logps/rejected": -378.0,
      "loss": 0.489,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.296875,
      "rewards/margins": 0.79296875,
      "rewards/rejected": -2.09375,
      "step": 1970
    },
    {
      "epoch": 0.0737004708641194,
      "grad_norm": 6.3798640961693405,
      "learning_rate": 3.684406401190919e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -2.078125,
      "logps/chosen": -344.0,
      "logps/rejected": -408.0,
      "loss": 0.5143,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.328125,
      "rewards/margins": 0.8046875,
      "rewards/rejected": -2.140625,
      "step": 1980
    },
    {
      "epoch": 0.07407269546444324,
      "grad_norm": 7.616930015000822,
      "learning_rate": 3.7030145143282473e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -1.96875,
      "logps/chosen": -332.0,
      "logps/rejected": -398.0,
      "loss": 0.4951,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.3046875,
      "rewards/margins": 0.8359375,
      "rewards/rejected": -2.140625,
      "step": 1990
    },
    {
      "epoch": 0.07444492006476708,
      "grad_norm": 6.436715329645376,
      "learning_rate": 3.721622627465575e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -1.9609375,
      "logps/chosen": -368.0,
      "logps/rejected": -436.0,
      "loss": 0.4886,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.421875,
      "rewards/margins": 0.89453125,
      "rewards/rejected": -2.3125,
      "step": 2000
    },
    {
      "epoch": 0.07481714466509091,
      "grad_norm": 6.436203502140251,
      "learning_rate": 3.7402307406029026e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.96875,
      "logps/chosen": -334.0,
      "logps/rejected": -374.0,
      "loss": 0.513,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.3046875,
      "rewards/margins": 0.671875,
      "rewards/rejected": -1.9765625,
      "step": 2010
    },
    {
      "epoch": 0.07518936926541475,
      "grad_norm": 7.593295569586558,
      "learning_rate": 3.7588388537402303e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -1.9609375,
      "logps/chosen": -332.0,
      "logps/rejected": -380.0,
      "loss": 0.4918,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.2109375,
      "rewards/margins": 0.84765625,
      "rewards/rejected": -2.0625,
      "step": 2020
    },
    {
      "epoch": 0.07556159386573859,
      "grad_norm": 6.438654157649776,
      "learning_rate": 3.7774469668775585e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.859375,
      "logps/chosen": -320.0,
      "logps/rejected": -360.0,
      "loss": 0.5003,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.375,
      "rewards/margins": 0.6484375,
      "rewards/rejected": -2.015625,
      "step": 2030
    },
    {
      "epoch": 0.07593381846606242,
      "grad_norm": 6.694506460978225,
      "learning_rate": 3.796055080014886e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -2.09375,
      "logps/chosen": -372.0,
      "logps/rejected": -414.0,
      "loss": 0.5117,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.609375,
      "rewards/margins": 0.58203125,
      "rewards/rejected": -2.1875,
      "step": 2040
    },
    {
      "epoch": 0.07630604306638626,
      "grad_norm": 7.201760168733986,
      "learning_rate": 3.8146631931522143e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.96875,
      "logps/chosen": -354.0,
      "logps/rejected": -428.0,
      "loss": 0.4829,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.4765625,
      "rewards/margins": 0.8828125,
      "rewards/rejected": -2.359375,
      "step": 2050
    },
    {
      "epoch": 0.0766782676667101,
      "grad_norm": 6.930750435234306,
      "learning_rate": 3.833271306289542e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.828125,
      "logps/chosen": -326.0,
      "logps/rejected": -396.0,
      "loss": 0.4788,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.3671875,
      "rewards/margins": 0.890625,
      "rewards/rejected": -2.265625,
      "step": 2060
    },
    {
      "epoch": 0.07705049226703393,
      "grad_norm": 7.966176376643152,
      "learning_rate": 3.85187941942687e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -1.9609375,
      "logps/chosen": -366.0,
      "logps/rejected": -440.0,
      "loss": 0.4995,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.328125,
      "rewards/margins": 0.90625,
      "rewards/rejected": -2.234375,
      "step": 2070
    },
    {
      "epoch": 0.07742271686735776,
      "grad_norm": 6.330415705912156,
      "learning_rate": 3.870487532564198e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -2.09375,
      "logps/chosen": -352.0,
      "logps/rejected": -410.0,
      "loss": 0.4968,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.484375,
      "rewards/margins": 0.6875,
      "rewards/rejected": -2.171875,
      "step": 2080
    },
    {
      "epoch": 0.0777949414676816,
      "grad_norm": 6.499536448734887,
      "learning_rate": 3.8890956457015255e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -2.078125,
      "logps/chosen": -332.0,
      "logps/rejected": -408.0,
      "loss": 0.5059,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.3984375,
      "rewards/margins": 0.671875,
      "rewards/rejected": -2.0625,
      "step": 2090
    },
    {
      "epoch": 0.07816716606800543,
      "grad_norm": 6.367147078564384,
      "learning_rate": 3.907703758838853e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -2.0625,
      "logps/chosen": -360.0,
      "logps/rejected": -422.0,
      "loss": 0.4711,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.34375,
      "rewards/margins": 0.9296875,
      "rewards/rejected": -2.265625,
      "step": 2100
    },
    {
      "epoch": 0.07853939066832927,
      "grad_norm": 7.499476405373601,
      "learning_rate": 3.9263118719761814e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -2.078125,
      "logps/chosen": -380.0,
      "logps/rejected": -464.0,
      "loss": 0.4835,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 0.8984375,
      "rewards/rejected": -2.609375,
      "step": 2110
    },
    {
      "epoch": 0.0789116152686531,
      "grad_norm": 6.481029011949294,
      "learning_rate": 3.9449199851135096e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.8359375,
      "logps/chosen": -368.0,
      "logps/rejected": -420.0,
      "loss": 0.5125,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.671875,
      "rewards/margins": 0.75390625,
      "rewards/rejected": -2.421875,
      "step": 2120
    },
    {
      "epoch": 0.07928383986897694,
      "grad_norm": 7.415654430429344,
      "learning_rate": 3.9635280982508373e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -2.03125,
      "logps/chosen": -332.0,
      "logps/rejected": -392.0,
      "loss": 0.4869,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.296875,
      "rewards/margins": 0.74609375,
      "rewards/rejected": -2.03125,
      "step": 2130
    },
    {
      "epoch": 0.07965606446930078,
      "grad_norm": 8.390315079240178,
      "learning_rate": 3.982136211388165e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -1.96875,
      "logps/chosen": -360.0,
      "logps/rejected": -400.0,
      "loss": 0.4929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.375,
      "rewards/margins": 0.69921875,
      "rewards/rejected": -2.078125,
      "step": 2140
    },
    {
      "epoch": 0.08002828906962461,
      "grad_norm": 8.183376655157142,
      "learning_rate": 4.000744324525493e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -1.984375,
      "logps/chosen": -354.0,
      "logps/rejected": -442.0,
      "loss": 0.4983,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.46875,
      "rewards/margins": 0.93359375,
      "rewards/rejected": -2.40625,
      "step": 2150
    },
    {
      "epoch": 0.08040051366994845,
      "grad_norm": 7.155326700428032,
      "learning_rate": 4.019352437662821e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.140625,
      "logps/chosen": -348.0,
      "logps/rejected": -398.0,
      "loss": 0.4879,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.3359375,
      "rewards/margins": 0.7109375,
      "rewards/rejected": -2.046875,
      "step": 2160
    },
    {
      "epoch": 0.08077273827027229,
      "grad_norm": 7.7382428841633715,
      "learning_rate": 4.0379605508001485e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -2.015625,
      "logps/chosen": -362.0,
      "logps/rejected": -430.0,
      "loss": 0.4995,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.3671875,
      "rewards/margins": 0.91015625,
      "rewards/rejected": -2.28125,
      "step": 2170
    },
    {
      "epoch": 0.08114496287059612,
      "grad_norm": 5.86745009660174,
      "learning_rate": 4.056568663937476e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -1.8828125,
      "logps/chosen": -358.0,
      "logps/rejected": -430.0,
      "loss": 0.4886,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.4765625,
      "rewards/margins": 0.9375,
      "rewards/rejected": -2.40625,
      "step": 2180
    },
    {
      "epoch": 0.08151718747091995,
      "grad_norm": 6.446855515138818,
      "learning_rate": 4.075176777074805e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -2.125,
      "logps/chosen": -366.0,
      "logps/rejected": -414.0,
      "loss": 0.5006,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.3125,
      "rewards/margins": 0.82421875,
      "rewards/rejected": -2.140625,
      "step": 2190
    },
    {
      "epoch": 0.08188941207124378,
      "grad_norm": 6.466988089368753,
      "learning_rate": 4.0937848902121325e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.046875,
      "logps/chosen": -364.0,
      "logps/rejected": -424.0,
      "loss": 0.4865,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 0.7890625,
      "rewards/rejected": -2.375,
      "step": 2200
    },
    {
      "epoch": 0.08226163667156762,
      "grad_norm": 10.639833688531498,
      "learning_rate": 4.11239300334946e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.0,
      "logps/chosen": -392.0,
      "logps/rejected": -432.0,
      "loss": 0.4876,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 0.66796875,
      "rewards/rejected": -2.484375,
      "step": 2210
    },
    {
      "epoch": 0.08263386127189146,
      "grad_norm": 7.4473486796855255,
      "learning_rate": 4.131001116486788e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -2.046875,
      "logps/chosen": -376.0,
      "logps/rejected": -442.0,
      "loss": 0.4811,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 0.82421875,
      "rewards/rejected": -2.453125,
      "step": 2220
    },
    {
      "epoch": 0.0830060858722153,
      "grad_norm": 6.2487423962299715,
      "learning_rate": 4.149609229624116e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -2.046875,
      "logps/chosen": -364.0,
      "logps/rejected": -410.0,
      "loss": 0.5024,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.453125,
      "rewards/margins": 0.77734375,
      "rewards/rejected": -2.234375,
      "step": 2230
    },
    {
      "epoch": 0.08337831047253913,
      "grad_norm": 6.488938221184718,
      "learning_rate": 4.168217342761444e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.15625,
      "logps/chosen": -352.0,
      "logps/rejected": -404.0,
      "loss": 0.4869,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.390625,
      "rewards/margins": 0.828125,
      "rewards/rejected": -2.21875,
      "step": 2240
    },
    {
      "epoch": 0.08375053507286297,
      "grad_norm": 7.581066712131032,
      "learning_rate": 4.1868254558987714e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.984375,
      "logps/chosen": -374.0,
      "logps/rejected": -412.0,
      "loss": 0.495,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.65625,
      "rewards/margins": 0.6640625,
      "rewards/rejected": -2.328125,
      "step": 2250
    },
    {
      "epoch": 0.0841227596731868,
      "grad_norm": 7.151820813607078,
      "learning_rate": 4.2054335690360996e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.28125,
      "logps/chosen": -350.0,
      "logps/rejected": -442.0,
      "loss": 0.4812,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.4765625,
      "rewards/margins": 0.890625,
      "rewards/rejected": -2.375,
      "step": 2260
    },
    {
      "epoch": 0.08449498427351064,
      "grad_norm": 7.351427522367124,
      "learning_rate": 4.224041682173428e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.203125,
      "logps/chosen": -368.0,
      "logps/rejected": -456.0,
      "loss": 0.5116,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.65625,
      "rewards/margins": 0.7265625,
      "rewards/rejected": -2.375,
      "step": 2270
    },
    {
      "epoch": 0.08486720887383448,
      "grad_norm": 6.4159752050469985,
      "learning_rate": 4.2426497953107555e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -2.203125,
      "logps/chosen": -364.0,
      "logps/rejected": -434.0,
      "loss": 0.4742,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.515625,
      "rewards/margins": 0.828125,
      "rewards/rejected": -2.34375,
      "step": 2280
    },
    {
      "epoch": 0.08523943347415831,
      "grad_norm": 6.6161270229072455,
      "learning_rate": 4.261257908448083e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.25,
      "logps/chosen": -392.0,
      "logps/rejected": -472.0,
      "loss": 0.5011,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -2.65625,
      "step": 2290
    },
    {
      "epoch": 0.08561165807448214,
      "grad_norm": 7.119173360660189,
      "learning_rate": 4.279866021585411e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.125,
      "logps/chosen": -380.0,
      "logps/rejected": -436.0,
      "loss": 0.4959,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 0.76171875,
      "rewards/rejected": -2.359375,
      "step": 2300
    },
    {
      "epoch": 0.08598388267480597,
      "grad_norm": 7.455200617965882,
      "learning_rate": 4.298474134722739e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.15625,
      "logps/chosen": -386.0,
      "logps/rejected": -470.0,
      "loss": 0.468,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6875,
      "rewards/margins": 0.96484375,
      "rewards/rejected": -2.65625,
      "step": 2310
    },
    {
      "epoch": 0.08635610727512981,
      "grad_norm": 7.040193764776661,
      "learning_rate": 4.3170822478600667e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.15625,
      "logps/chosen": -378.0,
      "logps/rejected": -438.0,
      "loss": 0.4878,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.578125,
      "rewards/margins": 0.78125,
      "rewards/rejected": -2.359375,
      "step": 2320
    },
    {
      "epoch": 0.08672833187545365,
      "grad_norm": 7.587904779417955,
      "learning_rate": 4.335690360997395e-07,
      "logits/chosen": -2.25,
      "logits/rejected": -1.9375,
      "logps/chosen": -360.0,
      "logps/rejected": -426.0,
      "loss": 0.475,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.46875,
      "rewards/margins": 0.88671875,
      "rewards/rejected": -2.34375,
      "step": 2330
    },
    {
      "epoch": 0.08710055647577748,
      "grad_norm": 8.097415315524271,
      "learning_rate": 4.3542984741347225e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.09375,
      "logps/chosen": -380.0,
      "logps/rejected": -462.0,
      "loss": 0.484,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.65625,
      "rewards/margins": 0.94140625,
      "rewards/rejected": -2.59375,
      "step": 2340
    },
    {
      "epoch": 0.08747278107610132,
      "grad_norm": 7.498818747137426,
      "learning_rate": 4.372906587272051e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.140625,
      "logps/chosen": -380.0,
      "logps/rejected": -442.0,
      "loss": 0.4855,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 0.92578125,
      "rewards/rejected": -2.625,
      "step": 2350
    },
    {
      "epoch": 0.08784500567642516,
      "grad_norm": 8.33898566724908,
      "learning_rate": 4.3915147004093784e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.09375,
      "logps/chosen": -366.0,
      "logps/rejected": -422.0,
      "loss": 0.4693,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.65625,
      "rewards/margins": 0.80078125,
      "rewards/rejected": -2.46875,
      "step": 2360
    },
    {
      "epoch": 0.08821723027674899,
      "grad_norm": 8.970047546171957,
      "learning_rate": 4.410122813546706e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -2.140625,
      "logps/chosen": -392.0,
      "logps/rejected": -426.0,
      "loss": 0.5046,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.765625,
      "rewards/margins": 0.796875,
      "rewards/rejected": -2.5625,
      "step": 2370
    },
    {
      "epoch": 0.08858945487707283,
      "grad_norm": 6.483608950602399,
      "learning_rate": 4.428730926684034e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.109375,
      "logps/chosen": -388.0,
      "logps/rejected": -430.0,
      "loss": 0.5092,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 0.71484375,
      "rewards/rejected": -2.328125,
      "step": 2380
    },
    {
      "epoch": 0.08896167947739667,
      "grad_norm": 6.872780275045827,
      "learning_rate": 4.447339039821362e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.234375,
      "logps/chosen": -390.0,
      "logps/rejected": -476.0,
      "loss": 0.4741,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 1.0546875,
      "rewards/rejected": -2.796875,
      "step": 2390
    },
    {
      "epoch": 0.0893339040777205,
      "grad_norm": 7.8400998210983275,
      "learning_rate": 4.4659471529586896e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.203125,
      "logps/chosen": -388.0,
      "logps/rejected": -498.0,
      "loss": 0.4936,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 1.0703125,
      "rewards/rejected": -2.796875,
      "step": 2400
    },
    {
      "epoch": 0.08970612867804433,
      "grad_norm": 7.134158151639725,
      "learning_rate": 4.484555266096018e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.0625,
      "logps/chosen": -352.0,
      "logps/rejected": -438.0,
      "loss": 0.4771,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 1.03125,
      "rewards/rejected": -2.703125,
      "step": 2410
    },
    {
      "epoch": 0.09007835327836816,
      "grad_norm": 7.924596755610699,
      "learning_rate": 4.5031633792333455e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.265625,
      "logps/chosen": -412.0,
      "logps/rejected": -468.0,
      "loss": 0.4919,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 0.87890625,
      "rewards/rejected": -2.84375,
      "step": 2420
    },
    {
      "epoch": 0.090450577878692,
      "grad_norm": 9.482778150529827,
      "learning_rate": 4.5217714923706737e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.15625,
      "logps/chosen": -370.0,
      "logps/rejected": -454.0,
      "loss": 0.4812,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.734375,
      "rewards/margins": 0.96875,
      "rewards/rejected": -2.703125,
      "step": 2430
    },
    {
      "epoch": 0.09082280247901584,
      "grad_norm": 7.58431229994053,
      "learning_rate": 4.5403796055080013e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -2.234375,
      "logps/chosen": -388.0,
      "logps/rejected": -440.0,
      "loss": 0.4807,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.515625,
      "rewards/margins": 0.87890625,
      "rewards/rejected": -2.390625,
      "step": 2440
    },
    {
      "epoch": 0.09119502707933967,
      "grad_norm": 6.323247386021342,
      "learning_rate": 4.558987718645329e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.09375,
      "logps/chosen": -364.0,
      "logps/rejected": -460.0,
      "loss": 0.4821,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 1.046875,
      "rewards/rejected": -2.765625,
      "step": 2450
    },
    {
      "epoch": 0.09156725167966351,
      "grad_norm": 7.061331999882923,
      "learning_rate": 4.5775958317826567e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -2.171875,
      "logps/chosen": -408.0,
      "logps/rejected": -444.0,
      "loss": 0.4813,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.734375,
      "rewards/margins": 0.734375,
      "rewards/rejected": -2.46875,
      "step": 2460
    },
    {
      "epoch": 0.09193947627998734,
      "grad_norm": 7.132322325093647,
      "learning_rate": 4.596203944919985e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.109375,
      "logps/chosen": -380.0,
      "logps/rejected": -458.0,
      "loss": 0.4975,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.671875,
      "rewards/margins": 0.875,
      "rewards/rejected": -2.546875,
      "step": 2470
    },
    {
      "epoch": 0.09231170088031118,
      "grad_norm": 7.3789160382495345,
      "learning_rate": 4.614812058057313e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.28125,
      "logps/chosen": -402.0,
      "logps/rejected": -456.0,
      "loss": 0.4661,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.796875,
      "rewards/margins": 0.88671875,
      "rewards/rejected": -2.6875,
      "step": 2480
    },
    {
      "epoch": 0.09268392548063502,
      "grad_norm": 7.132658190420856,
      "learning_rate": 4.633420171194641e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.3125,
      "logps/chosen": -398.0,
      "logps/rejected": -476.0,
      "loss": 0.502,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 0.796875,
      "rewards/rejected": -2.578125,
      "step": 2490
    },
    {
      "epoch": 0.09305615008095885,
      "grad_norm": 7.3364168675638,
      "learning_rate": 4.6520282843319684e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.203125,
      "logps/chosen": -386.0,
      "logps/rejected": -466.0,
      "loss": 0.4942,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 0.87890625,
      "rewards/rejected": -2.6875,
      "step": 2500
    },
    {
      "epoch": 0.09342837468128269,
      "grad_norm": 8.335927298484165,
      "learning_rate": 4.6706363974692966e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.125,
      "logps/chosen": -378.0,
      "logps/rejected": -440.0,
      "loss": 0.5111,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.78125,
      "rewards/margins": 0.89453125,
      "rewards/rejected": -2.671875,
      "step": 2510
    },
    {
      "epoch": 0.09380059928160653,
      "grad_norm": 7.563941548333361,
      "learning_rate": 4.6892445106066243e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.25,
      "logps/chosen": -394.0,
      "logps/rejected": -478.0,
      "loss": 0.4768,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -2.734375,
      "step": 2520
    },
    {
      "epoch": 0.09417282388193035,
      "grad_norm": 6.232884700140862,
      "learning_rate": 4.707852623743952e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.25,
      "logps/chosen": -400.0,
      "logps/rejected": -454.0,
      "loss": 0.5082,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.8125,
      "rewards/margins": 0.828125,
      "rewards/rejected": -2.640625,
      "step": 2530
    },
    {
      "epoch": 0.09454504848225419,
      "grad_norm": 7.800107054547311,
      "learning_rate": 4.7264607368812796e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.09375,
      "logps/chosen": -364.0,
      "logps/rejected": -454.0,
      "loss": 0.4918,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 0.85546875,
      "rewards/rejected": -2.453125,
      "step": 2540
    },
    {
      "epoch": 0.09491727308257802,
      "grad_norm": 7.014735849518465,
      "learning_rate": 4.7450688500186083e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.203125,
      "logps/chosen": -382.0,
      "logps/rejected": -444.0,
      "loss": 0.4649,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.640625,
      "rewards/margins": 0.8359375,
      "rewards/rejected": -2.46875,
      "step": 2550
    },
    {
      "epoch": 0.09528949768290186,
      "grad_norm": 9.143436068684284,
      "learning_rate": 4.763676963155936e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -1.984375,
      "logps/chosen": -374.0,
      "logps/rejected": -470.0,
      "loss": 0.4898,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 1.078125,
      "rewards/rejected": -2.78125,
      "step": 2560
    },
    {
      "epoch": 0.0956617222832257,
      "grad_norm": 8.392494506267354,
      "learning_rate": 4.782285076293264e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.140625,
      "logps/chosen": -358.0,
      "logps/rejected": -438.0,
      "loss": 0.4802,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.4765625,
      "rewards/margins": 0.9609375,
      "rewards/rejected": -2.4375,
      "step": 2570
    },
    {
      "epoch": 0.09603394688354953,
      "grad_norm": 8.887195426407994,
      "learning_rate": 4.800893189430591e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.015625,
      "logps/chosen": -386.0,
      "logps/rejected": -460.0,
      "loss": 0.471,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.765625,
      "rewards/margins": 0.921875,
      "rewards/rejected": -2.6875,
      "step": 2580
    },
    {
      "epoch": 0.09640617148387337,
      "grad_norm": 8.505664680833817,
      "learning_rate": 4.81950130256792e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.140625,
      "logps/chosen": -410.0,
      "logps/rejected": -512.0,
      "loss": 0.4867,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.046875,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -3.265625,
      "step": 2590
    },
    {
      "epoch": 0.09677839608419721,
      "grad_norm": 7.170131667523458,
      "learning_rate": 4.838109415705247e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.078125,
      "logps/chosen": -378.0,
      "logps/rejected": -460.0,
      "loss": 0.4868,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.7734375,
      "rewards/margins": 0.92578125,
      "rewards/rejected": -2.703125,
      "step": 2600
    },
    {
      "epoch": 0.09715062068452104,
      "grad_norm": 9.271156852767316,
      "learning_rate": 4.856717528842575e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.234375,
      "logps/chosen": -358.0,
      "logps/rejected": -424.0,
      "loss": 0.4802,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 0.75,
      "rewards/rejected": -2.359375,
      "step": 2610
    },
    {
      "epoch": 0.09752284528484488,
      "grad_norm": 8.723540021601536,
      "learning_rate": 4.875325641979903e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.125,
      "logps/chosen": -372.0,
      "logps/rejected": -434.0,
      "loss": 0.4858,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.71875,
      "rewards/margins": 0.8671875,
      "rewards/rejected": -2.578125,
      "step": 2620
    },
    {
      "epoch": 0.09789506988516872,
      "grad_norm": 6.4733833408217505,
      "learning_rate": 4.893933755117231e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.296875,
      "logps/chosen": -364.0,
      "logps/rejected": -444.0,
      "loss": 0.4744,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.515625,
      "rewards/margins": 0.9453125,
      "rewards/rejected": -2.453125,
      "step": 2630
    },
    {
      "epoch": 0.09826729448549254,
      "grad_norm": 7.3845877716768324,
      "learning_rate": 4.912541868254558e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.328125,
      "logps/chosen": -388.0,
      "logps/rejected": -448.0,
      "loss": 0.4973,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 0.83203125,
      "rewards/rejected": -2.5,
      "step": 2640
    },
    {
      "epoch": 0.09863951908581638,
      "grad_norm": 6.650764364783044,
      "learning_rate": 4.931149981391887e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.234375,
      "logps/chosen": -384.0,
      "logps/rejected": -456.0,
      "loss": 0.5084,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.734375,
      "rewards/margins": 0.80859375,
      "rewards/rejected": -2.546875,
      "step": 2650
    },
    {
      "epoch": 0.09901174368614021,
      "grad_norm": 8.407761251928182,
      "learning_rate": 4.949758094529215e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -2.109375,
      "logps/chosen": -350.0,
      "logps/rejected": -416.0,
      "loss": 0.4915,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.453125,
      "rewards/margins": 0.93359375,
      "rewards/rejected": -2.375,
      "step": 2660
    },
    {
      "epoch": 0.09938396828646405,
      "grad_norm": 7.715717630327331,
      "learning_rate": 4.968366207666542e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.0625,
      "logps/chosen": -378.0,
      "logps/rejected": -440.0,
      "loss": 0.4857,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.578125,
      "rewards/margins": 0.8828125,
      "rewards/rejected": -2.453125,
      "step": 2670
    },
    {
      "epoch": 0.09975619288678789,
      "grad_norm": 8.314761126466934,
      "learning_rate": 4.98697432080387e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -1.9921875,
      "logps/chosen": -374.0,
      "logps/rejected": -470.0,
      "loss": 0.4768,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -2.703125,
      "step": 2680
    },
    {
      "epoch": 0.10012841748711172,
      "grad_norm": 6.968864811788696,
      "learning_rate": 4.999999810062151e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.28125,
      "logps/chosen": -404.0,
      "logps/rejected": -442.0,
      "loss": 0.4801,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.859375,
      "rewards/margins": 0.6484375,
      "rewards/rejected": -2.515625,
      "step": 2690
    },
    {
      "epoch": 0.10050064208743556,
      "grad_norm": 8.722012236402668,
      "learning_rate": 4.99999643339008e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.109375,
      "logps/chosen": -384.0,
      "logps/rejected": -468.0,
      "loss": 0.4747,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.8125,
      "rewards/margins": 0.94921875,
      "rewards/rejected": -2.765625,
      "step": 2700
    },
    {
      "epoch": 0.1008728666877594,
      "grad_norm": 7.880948774785855,
      "learning_rate": 4.999988835883477e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -2.09375,
      "logps/chosen": -398.0,
      "logps/rejected": -458.0,
      "loss": 0.4933,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 0.8828125,
      "rewards/rejected": -2.71875,
      "step": 2710
    },
    {
      "epoch": 0.10124509128808323,
      "grad_norm": 7.786857858173807,
      "learning_rate": 4.999977017555171e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.25,
      "logps/chosen": -400.0,
      "logps/rejected": -436.0,
      "loss": 0.4847,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 0.73046875,
      "rewards/rejected": -2.4375,
      "step": 2720
    },
    {
      "epoch": 0.10161731588840707,
      "grad_norm": 6.557465577763377,
      "learning_rate": 4.999960978425113e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.28125,
      "logps/chosen": -390.0,
      "logps/rejected": -442.0,
      "loss": 0.4668,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 0.76171875,
      "rewards/rejected": -2.453125,
      "step": 2730
    },
    {
      "epoch": 0.1019895404887309,
      "grad_norm": 7.9202380114331215,
      "learning_rate": 4.999940718520385e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.203125,
      "logps/chosen": -366.0,
      "logps/rejected": -466.0,
      "loss": 0.4794,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -2.671875,
      "step": 2740
    },
    {
      "epoch": 0.10236176508905473,
      "grad_norm": 7.507209078918445,
      "learning_rate": 4.999916237875191e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.171875,
      "logps/chosen": -386.0,
      "logps/rejected": -464.0,
      "loss": 0.4681,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.765625,
      "rewards/margins": 0.984375,
      "rewards/rejected": -2.75,
      "step": 2750
    },
    {
      "epoch": 0.10273398968937857,
      "grad_norm": 8.248787853160541,
      "learning_rate": 4.999887536530864e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.15625,
      "logps/chosen": -378.0,
      "logps/rejected": -454.0,
      "loss": 0.4863,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.78125,
      "rewards/margins": 0.98046875,
      "rewards/rejected": -2.765625,
      "step": 2760
    },
    {
      "epoch": 0.1031062142897024,
      "grad_norm": 7.114173155178757,
      "learning_rate": 4.999854614535859e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -1.9375,
      "logps/chosen": -372.0,
      "logps/rejected": -448.0,
      "loss": 0.4772,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 0.76171875,
      "rewards/rejected": -2.359375,
      "step": 2770
    },
    {
      "epoch": 0.10347843889002624,
      "grad_norm": 6.7720031805269505,
      "learning_rate": 4.999817471945762e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -2.15625,
      "logps/chosen": -362.0,
      "logps/rejected": -424.0,
      "loss": 0.4977,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 0.76171875,
      "rewards/rejected": -2.34375,
      "step": 2780
    },
    {
      "epoch": 0.10385066349035008,
      "grad_norm": 7.856782009922716,
      "learning_rate": 4.99977610882328e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.25,
      "logps/chosen": -414.0,
      "logps/rejected": -490.0,
      "loss": 0.4716,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 0.95703125,
      "rewards/rejected": -2.828125,
      "step": 2790
    },
    {
      "epoch": 0.10422288809067391,
      "grad_norm": 7.169663540643967,
      "learning_rate": 4.99973052523825e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.234375,
      "logps/chosen": -426.0,
      "logps/rejected": -488.0,
      "loss": 0.4777,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.0625,
      "rewards/margins": 0.84375,
      "rewards/rejected": -2.90625,
      "step": 2800
    },
    {
      "epoch": 0.10459511269099775,
      "grad_norm": 8.378333646099144,
      "learning_rate": 4.999680721267631e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.296875,
      "logps/chosen": -390.0,
      "logps/rejected": -480.0,
      "loss": 0.4905,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -2.828125,
      "step": 2810
    },
    {
      "epoch": 0.10496733729132159,
      "grad_norm": 7.650897730555136,
      "learning_rate": 4.999626696995509e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.234375,
      "logps/chosen": -370.0,
      "logps/rejected": -444.0,
      "loss": 0.4689,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 0.984375,
      "rewards/rejected": -2.65625,
      "step": 2820
    },
    {
      "epoch": 0.10533956189164542,
      "grad_norm": 7.579728119589045,
      "learning_rate": 4.999568452513096e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.1875,
      "logps/chosen": -376.0,
      "logps/rejected": -436.0,
      "loss": 0.4831,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.609375,
      "rewards/margins": 0.82421875,
      "rewards/rejected": -2.4375,
      "step": 2830
    },
    {
      "epoch": 0.10571178649196926,
      "grad_norm": 7.696216704407773,
      "learning_rate": 4.999505987918727e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.109375,
      "logps/chosen": -358.0,
      "logps/rejected": -434.0,
      "loss": 0.4871,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 0.8203125,
      "rewards/rejected": -2.46875,
      "step": 2840
    },
    {
      "epoch": 0.1060840110922931,
      "grad_norm": 7.989906725237588,
      "learning_rate": 4.999439303317864e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -1.9296875,
      "logps/chosen": -378.0,
      "logps/rejected": -470.0,
      "loss": 0.4945,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.7734375,
      "rewards/margins": 0.98046875,
      "rewards/rejected": -2.75,
      "step": 2850
    },
    {
      "epoch": 0.10645623569261692,
      "grad_norm": 8.304798344933715,
      "learning_rate": 4.999368398823093e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.34375,
      "logps/chosen": -400.0,
      "logps/rejected": -464.0,
      "loss": 0.4712,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.7578125,
      "rewards/margins": 0.83984375,
      "rewards/rejected": -2.59375,
      "step": 2860
    },
    {
      "epoch": 0.10682846029294076,
      "grad_norm": 9.572061092616801,
      "learning_rate": 4.999293274554124e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.25,
      "logps/chosen": -370.0,
      "logps/rejected": -470.0,
      "loss": 0.4782,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.609375,
      "rewards/margins": 1.109375,
      "rewards/rejected": -2.71875,
      "step": 2870
    },
    {
      "epoch": 0.10720068489326459,
      "grad_norm": 8.901655691217524,
      "learning_rate": 4.999213930637793e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.109375,
      "logps/chosen": -398.0,
      "logps/rejected": -490.0,
      "loss": 0.4721,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 1.078125,
      "rewards/rejected": -2.90625,
      "step": 2880
    },
    {
      "epoch": 0.10757290949358843,
      "grad_norm": 7.512201898688962,
      "learning_rate": 4.999130367208059e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.21875,
      "logps/chosen": -392.0,
      "logps/rejected": -458.0,
      "loss": 0.474,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.78125,
      "rewards/margins": 0.90625,
      "rewards/rejected": -2.6875,
      "step": 2890
    },
    {
      "epoch": 0.10794513409391226,
      "grad_norm": 7.83972347302655,
      "learning_rate": 4.999042584406005e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.1875,
      "logps/chosen": -376.0,
      "logps/rejected": -474.0,
      "loss": 0.4595,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.625,
      "rewards/margins": 1.125,
      "rewards/rejected": -2.75,
      "step": 2900
    },
    {
      "epoch": 0.1083173586942361,
      "grad_norm": 8.227529977446723,
      "learning_rate": 4.998950582379836e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.21875,
      "logps/chosen": -398.0,
      "logps/rejected": -468.0,
      "loss": 0.463,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 0.84375,
      "rewards/rejected": -2.6875,
      "step": 2910
    },
    {
      "epoch": 0.10868958329455994,
      "grad_norm": 7.82017122313443,
      "learning_rate": 4.998854361284885e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -1.9375,
      "logps/chosen": -372.0,
      "logps/rejected": -460.0,
      "loss": 0.472,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 0.9296875,
      "rewards/rejected": -2.484375,
      "step": 2920
    },
    {
      "epoch": 0.10906180789488377,
      "grad_norm": 9.123563265797738,
      "learning_rate": 4.998753921283606e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.3125,
      "logps/chosen": -376.0,
      "logps/rejected": -460.0,
      "loss": 0.4698,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.796875,
      "rewards/margins": 0.87890625,
      "rewards/rejected": -2.671875,
      "step": 2930
    },
    {
      "epoch": 0.10943403249520761,
      "grad_norm": 7.615167752012157,
      "learning_rate": 4.998649262545573e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.34375,
      "logps/chosen": -382.0,
      "logps/rejected": -468.0,
      "loss": 0.4845,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 1.0625,
      "rewards/rejected": -2.890625,
      "step": 2940
    },
    {
      "epoch": 0.10980625709553145,
      "grad_norm": 7.36967531314196,
      "learning_rate": 4.998540385247487e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.203125,
      "logps/chosen": -374.0,
      "logps/rejected": -440.0,
      "loss": 0.5094,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 0.81640625,
      "rewards/rejected": -2.5,
      "step": 2950
    },
    {
      "epoch": 0.11017848169585528,
      "grad_norm": 8.463143887299749,
      "learning_rate": 4.998427289573168e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.375,
      "logps/chosen": -370.0,
      "logps/rejected": -442.0,
      "loss": 0.505,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 0.7578125,
      "rewards/rejected": -2.390625,
      "step": 2960
    },
    {
      "epoch": 0.11055070629617911,
      "grad_norm": 5.943238837158037,
      "learning_rate": 4.998309975713561e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.140625,
      "logps/chosen": -364.0,
      "logps/rejected": -426.0,
      "loss": 0.501,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.65625,
      "rewards/margins": 0.75,
      "rewards/rejected": -2.40625,
      "step": 2970
    },
    {
      "epoch": 0.11092293089650294,
      "grad_norm": 7.223537475507539,
      "learning_rate": 4.99818844386673e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.15625,
      "logps/chosen": -388.0,
      "logps/rejected": -464.0,
      "loss": 0.4725,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.875,
      "rewards/margins": 0.87890625,
      "rewards/rejected": -2.75,
      "step": 2980
    },
    {
      "epoch": 0.11129515549682678,
      "grad_norm": 8.021766195736895,
      "learning_rate": 4.998062694237862e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.453125,
      "logps/chosen": -402.0,
      "logps/rejected": -502.0,
      "loss": 0.4684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 1.0546875,
      "rewards/rejected": -2.90625,
      "step": 2990
    },
    {
      "epoch": 0.11166738009715062,
      "grad_norm": 9.294284211436038,
      "learning_rate": 4.997932727039265e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.265625,
      "logps/chosen": -390.0,
      "logps/rejected": -490.0,
      "loss": 0.4939,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 1.1328125,
      "rewards/rejected": -2.8125,
      "step": 3000
    },
    {
      "epoch": 0.11203960469747445,
      "grad_norm": 6.981409205274026,
      "learning_rate": 4.997798542490368e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.21875,
      "logps/chosen": -370.0,
      "logps/rejected": -474.0,
      "loss": 0.4577,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 0.9921875,
      "rewards/rejected": -2.71875,
      "step": 3010
    },
    {
      "epoch": 0.11241182929779829,
      "grad_norm": 6.897253136266305,
      "learning_rate": 4.997660140817717e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.265625,
      "logps/chosen": -406.0,
      "logps/rejected": -478.0,
      "loss": 0.4504,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 0.8984375,
      "rewards/rejected": -2.8125,
      "step": 3020
    },
    {
      "epoch": 0.11278405389812213,
      "grad_norm": 8.62655311915896,
      "learning_rate": 4.997517522254984e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.15625,
      "logps/chosen": -400.0,
      "logps/rejected": -464.0,
      "loss": 0.4775,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.84375,
      "rewards/margins": 0.82421875,
      "rewards/rejected": -2.671875,
      "step": 3030
    },
    {
      "epoch": 0.11315627849844596,
      "grad_norm": 8.93286896091944,
      "learning_rate": 4.997370687042956e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.296875,
      "logps/chosen": -426.0,
      "logps/rejected": -496.0,
      "loss": 0.4656,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -2.109375,
      "rewards/margins": 0.890625,
      "rewards/rejected": -3.0,
      "step": 3040
    },
    {
      "epoch": 0.1135285030987698,
      "grad_norm": 8.016864560573323,
      "learning_rate": 4.997219635429538e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.421875,
      "logps/chosen": -418.0,
      "logps/rejected": -524.0,
      "loss": 0.4619,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.0625,
      "rewards/margins": 1.09375,
      "rewards/rejected": -3.15625,
      "step": 3050
    },
    {
      "epoch": 0.11390072769909364,
      "grad_norm": 6.729558953662216,
      "learning_rate": 4.997064367669758e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.34375,
      "logps/chosen": -364.0,
      "logps/rejected": -448.0,
      "loss": 0.4818,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.640625,
      "rewards/margins": 0.8203125,
      "rewards/rejected": -2.453125,
      "step": 3060
    },
    {
      "epoch": 0.11427295229941747,
      "grad_norm": 8.670605577460273,
      "learning_rate": 4.996904884025761e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.390625,
      "logps/chosen": -392.0,
      "logps/rejected": -468.0,
      "loss": 0.4846,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -2.796875,
      "step": 3070
    },
    {
      "epoch": 0.11464517689974131,
      "grad_norm": 7.36252668199986,
      "learning_rate": 4.996741184766806e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.328125,
      "logps/chosen": -400.0,
      "logps/rejected": -494.0,
      "loss": 0.4666,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.03125,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -3.0625,
      "step": 3080
    },
    {
      "epoch": 0.11501740150006513,
      "grad_norm": 7.329183383679412,
      "learning_rate": 4.996573270169275e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.421875,
      "logps/chosen": -398.0,
      "logps/rejected": -472.0,
      "loss": 0.4619,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.828125,
      "rewards/margins": 0.97265625,
      "rewards/rejected": -2.796875,
      "step": 3090
    },
    {
      "epoch": 0.11538962610038897,
      "grad_norm": 8.391909491979217,
      "learning_rate": 4.996401140516663e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.4375,
      "logps/chosen": -408.0,
      "logps/rejected": -510.0,
      "loss": 0.4615,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0,
      "rewards/margins": 1.0859375,
      "rewards/rejected": -3.09375,
      "step": 3100
    },
    {
      "epoch": 0.1157618507007128,
      "grad_norm": 7.88600869188835,
      "learning_rate": 4.996224796099584e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.40625,
      "logps/chosen": -410.0,
      "logps/rejected": -504.0,
      "loss": 0.4628,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.015625,
      "rewards/margins": 1.1875,
      "rewards/rejected": -3.203125,
      "step": 3110
    },
    {
      "epoch": 0.11613407530103664,
      "grad_norm": 17.008655350153827,
      "learning_rate": 4.996044237215765e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.328125,
      "logps/chosen": -396.0,
      "logps/rejected": -484.0,
      "loss": 0.4743,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.90625,
      "rewards/margins": 0.9375,
      "rewards/rejected": -2.84375,
      "step": 3120
    },
    {
      "epoch": 0.11650629990136048,
      "grad_norm": 8.641825704990254,
      "learning_rate": 4.995859464170051e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.453125,
      "logps/chosen": -392.0,
      "logps/rejected": -486.0,
      "loss": 0.4576,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 0.98828125,
      "rewards/rejected": -2.859375,
      "step": 3130
    },
    {
      "epoch": 0.11687852450168432,
      "grad_norm": 6.894925182787359,
      "learning_rate": 4.995670477274402e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.375,
      "logps/chosen": -398.0,
      "logps/rejected": -494.0,
      "loss": 0.4769,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 0.96484375,
      "rewards/rejected": -2.9375,
      "step": 3140
    },
    {
      "epoch": 0.11725074910200815,
      "grad_norm": 7.978956949136547,
      "learning_rate": 4.995477276847889e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.265625,
      "logps/chosen": -400.0,
      "logps/rejected": -502.0,
      "loss": 0.4672,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.046875,
      "rewards/margins": 0.9375,
      "rewards/rejected": -2.984375,
      "step": 3150
    },
    {
      "epoch": 0.11762297370233199,
      "grad_norm": 7.347594002744936,
      "learning_rate": 4.995279863216701e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.3125,
      "logps/chosen": -366.0,
      "logps/rejected": -432.0,
      "loss": 0.4568,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.6875,
      "rewards/margins": 0.875,
      "rewards/rejected": -2.5625,
      "step": 3160
    },
    {
      "epoch": 0.11799519830265583,
      "grad_norm": 7.673249877401503,
      "learning_rate": 4.995078236714138e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.140625,
      "logps/chosen": -408.0,
      "logps/rejected": -500.0,
      "loss": 0.463,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.0,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -3.09375,
      "step": 3170
    },
    {
      "epoch": 0.11836742290297966,
      "grad_norm": 8.361137008058112,
      "learning_rate": 4.994872397680615e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.3125,
      "logps/chosen": -402.0,
      "logps/rejected": -504.0,
      "loss": 0.4817,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -2.078125,
      "rewards/margins": 0.97265625,
      "rewards/rejected": -3.046875,
      "step": 3180
    },
    {
      "epoch": 0.1187396475033035,
      "grad_norm": 8.343603011621855,
      "learning_rate": 4.994662346463655e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.5,
      "logps/chosen": -392.0,
      "logps/rejected": -448.0,
      "loss": 0.4889,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.78125,
      "rewards/margins": 0.8515625,
      "rewards/rejected": -2.625,
      "step": 3190
    },
    {
      "epoch": 0.11911187210362732,
      "grad_norm": 9.102554724230833,
      "learning_rate": 4.994448083417896e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.328125,
      "logps/chosen": -382.0,
      "logps/rejected": -484.0,
      "loss": 0.4646,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.78125,
      "rewards/margins": 1.0078125,
      "rewards/rejected": -2.796875,
      "step": 3200
    },
    {
      "epoch": 0.11948409670395116,
      "grad_norm": 8.99245454375693,
      "learning_rate": 4.994229608905087e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.4375,
      "logps/chosen": -410.0,
      "logps/rejected": -536.0,
      "loss": 0.4716,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 1.34375,
      "rewards/rejected": -3.3125,
      "step": 3210
    },
    {
      "epoch": 0.119856321304275,
      "grad_norm": 7.432405865342894,
      "learning_rate": 4.994006923294085e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.4375,
      "logps/chosen": -390.0,
      "logps/rejected": -482.0,
      "loss": 0.4849,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.984375,
      "rewards/margins": 0.94921875,
      "rewards/rejected": -2.9375,
      "step": 3220
    },
    {
      "epoch": 0.12022854590459883,
      "grad_norm": 7.162472408996945,
      "learning_rate": 4.993780026960859e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.453125,
      "logps/chosen": -414.0,
      "logps/rejected": -482.0,
      "loss": 0.4818,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.796875,
      "rewards/margins": 0.984375,
      "rewards/rejected": -2.78125,
      "step": 3230
    },
    {
      "epoch": 0.12060077050492267,
      "grad_norm": 7.465117638714362,
      "learning_rate": 4.993548920288487e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5,
      "logps/chosen": -380.0,
      "logps/rejected": -474.0,
      "loss": 0.4745,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.75,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -2.875,
      "step": 3240
    },
    {
      "epoch": 0.1209729951052465,
      "grad_norm": 7.684449433770762,
      "learning_rate": 4.993313603667152e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.421875,
      "logps/chosen": -378.0,
      "logps/rejected": -472.0,
      "loss": 0.4607,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 0.99609375,
      "rewards/rejected": -2.8125,
      "step": 3250
    },
    {
      "epoch": 0.12134521970557034,
      "grad_norm": 8.054634261872721,
      "learning_rate": 4.993074077494151e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.421875,
      "logps/chosen": -368.0,
      "logps/rejected": -484.0,
      "loss": 0.4766,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.765625,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -2.96875,
      "step": 3260
    },
    {
      "epoch": 0.12171744430589418,
      "grad_norm": 9.158209079484156,
      "learning_rate": 4.992830342173883e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.46875,
      "logps/chosen": -398.0,
      "logps/rejected": -492.0,
      "loss": 0.4342,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 1.203125,
      "rewards/rejected": -2.890625,
      "step": 3270
    },
    {
      "epoch": 0.12208966890621802,
      "grad_norm": 9.9678750837189,
      "learning_rate": 4.992582398117854e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.25,
      "logps/chosen": -398.0,
      "logps/rejected": -512.0,
      "loss": 0.451,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.046875,
      "rewards/margins": 1.390625,
      "rewards/rejected": -3.4375,
      "step": 3280
    },
    {
      "epoch": 0.12246189350654185,
      "grad_norm": 7.964466443950102,
      "learning_rate": 4.992330245744679e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.5625,
      "logps/chosen": -418.0,
      "logps/rejected": -520.0,
      "loss": 0.4597,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.109375,
      "rewards/margins": 1.15625,
      "rewards/rejected": -3.265625,
      "step": 3290
    },
    {
      "epoch": 0.12283411810686569,
      "grad_norm": 6.942468498840152,
      "learning_rate": 4.992073885480076e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.46875,
      "logps/chosen": -396.0,
      "logps/rejected": -438.0,
      "loss": 0.4741,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.734375,
      "rewards/margins": 0.80078125,
      "rewards/rejected": -2.53125,
      "step": 3300
    },
    {
      "epoch": 0.12320634270718951,
      "grad_norm": 8.588346105397495,
      "learning_rate": 4.991813317756866e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.390625,
      "logps/chosen": -384.0,
      "logps/rejected": -464.0,
      "loss": 0.4681,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.78125,
      "rewards/margins": 1.046875,
      "rewards/rejected": -2.828125,
      "step": 3310
    },
    {
      "epoch": 0.12357856730751335,
      "grad_norm": 8.362464228328575,
      "learning_rate": 4.991548543014975e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.375,
      "logps/chosen": -408.0,
      "logps/rejected": -500.0,
      "loss": 0.492,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -3.109375,
      "step": 3320
    },
    {
      "epoch": 0.12395079190783719,
      "grad_norm": 8.943983649632811,
      "learning_rate": 4.991279561701434e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.375,
      "logps/chosen": -394.0,
      "logps/rejected": -502.0,
      "loss": 0.4584,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.84375,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -3.078125,
      "step": 3330
    },
    {
      "epoch": 0.12432301650816102,
      "grad_norm": 7.764582747799325,
      "learning_rate": 4.991006374270371e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.328125,
      "logps/chosen": -422.0,
      "logps/rejected": -508.0,
      "loss": 0.4696,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 1.015625,
      "rewards/rejected": -3.0,
      "step": 3340
    },
    {
      "epoch": 0.12469524110848486,
      "grad_norm": 8.02411744667091,
      "learning_rate": 4.990728981183019e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.484375,
      "logps/chosen": -412.0,
      "logps/rejected": -502.0,
      "loss": 0.4642,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.890625,
      "rewards/margins": 1.1484375,
      "rewards/rejected": -3.03125,
      "step": 3350
    },
    {
      "epoch": 0.1250674657088087,
      "grad_norm": 8.117197519351885,
      "learning_rate": 4.990447382907713e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -404.0,
      "logps/rejected": -494.0,
      "loss": 0.4987,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 1.015625,
      "rewards/rejected": -2.96875,
      "step": 3360
    },
    {
      "epoch": 0.12543969030913252,
      "grad_norm": 7.89849195965691,
      "learning_rate": 4.990161579919882e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.453125,
      "logps/chosen": -402.0,
      "logps/rejected": -492.0,
      "loss": 0.4712,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 1.0859375,
      "rewards/rejected": -2.875,
      "step": 3370
    },
    {
      "epoch": 0.12581191490945637,
      "grad_norm": 9.235912295602464,
      "learning_rate": 4.98987157270206e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.46875,
      "logps/chosen": -420.0,
      "logps/rejected": -496.0,
      "loss": 0.4746,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.0625,
      "rewards/margins": 0.984375,
      "rewards/rejected": -3.046875,
      "step": 3380
    },
    {
      "epoch": 0.1261841395097802,
      "grad_norm": 7.711612260860563,
      "learning_rate": 4.989577361743875e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.4375,
      "logps/chosen": -400.0,
      "logps/rejected": -486.0,
      "loss": 0.459,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 0.9609375,
      "rewards/rejected": -2.8125,
      "step": 3390
    },
    {
      "epoch": 0.12655636411010404,
      "grad_norm": 9.901483910822622,
      "learning_rate": 4.989278947542056e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.359375,
      "logps/chosen": -422.0,
      "logps/rejected": -498.0,
      "loss": 0.4371,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 1.046875,
      "rewards/rejected": -2.9375,
      "step": 3400
    },
    {
      "epoch": 0.12692858871042786,
      "grad_norm": 11.682469810799732,
      "learning_rate": 4.988976330600426e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.421875,
      "logps/chosen": -396.0,
      "logps/rejected": -504.0,
      "loss": 0.4764,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 1.3125,
      "rewards/rejected": -3.28125,
      "step": 3410
    },
    {
      "epoch": 0.12730081331075171,
      "grad_norm": 8.329668176444363,
      "learning_rate": 4.988669511429902e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5,
      "logps/chosen": -396.0,
      "logps/rejected": -478.0,
      "loss": 0.4778,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 0.93359375,
      "rewards/rejected": -2.78125,
      "step": 3420
    },
    {
      "epoch": 0.12767303791107554,
      "grad_norm": 7.61421133583996,
      "learning_rate": 4.988358490548501e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5,
      "logps/chosen": -406.0,
      "logps/rejected": -498.0,
      "loss": 0.4832,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.796875,
      "rewards/margins": 1.1484375,
      "rewards/rejected": -2.9375,
      "step": 3430
    },
    {
      "epoch": 0.1280452625113994,
      "grad_norm": 7.059199901406627,
      "learning_rate": 4.988043268481329e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.4375,
      "logps/chosen": -396.0,
      "logps/rejected": -492.0,
      "loss": 0.4559,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.765625,
      "rewards/margins": 1.15625,
      "rewards/rejected": -2.921875,
      "step": 3440
    },
    {
      "epoch": 0.1284174871117232,
      "grad_norm": 9.392876782952376,
      "learning_rate": 4.987723845760587e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.40625,
      "logps/chosen": -404.0,
      "logps/rejected": -496.0,
      "loss": 0.4682,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.9375,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -2.96875,
      "step": 3450
    },
    {
      "epoch": 0.12878971171204706,
      "grad_norm": 8.538277099676698,
      "learning_rate": 4.987400222925569e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.46875,
      "logps/chosen": -414.0,
      "logps/rejected": -494.0,
      "loss": 0.4707,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.09375,
      "rewards/margins": 0.83203125,
      "rewards/rejected": -2.921875,
      "step": 3460
    },
    {
      "epoch": 0.12916193631237088,
      "grad_norm": 9.855635938290549,
      "learning_rate": 4.987072400522658e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.328125,
      "logps/chosen": -362.0,
      "logps/rejected": -468.0,
      "loss": 0.4583,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.9375,
      "rewards/margins": 1.1328125,
      "rewards/rejected": -3.0625,
      "step": 3470
    },
    {
      "epoch": 0.1295341609126947,
      "grad_norm": 9.692526745509848,
      "learning_rate": 4.986740379105328e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5,
      "logps/chosen": -406.0,
      "logps/rejected": -508.0,
      "loss": 0.4707,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.046875,
      "rewards/margins": 0.97265625,
      "rewards/rejected": -3.015625,
      "step": 3480
    },
    {
      "epoch": 0.12990638551301856,
      "grad_norm": 7.0429305121097,
      "learning_rate": 4.986404159234145e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.40625,
      "logps/chosen": -422.0,
      "logps/rejected": -508.0,
      "loss": 0.4466,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 1.0859375,
      "rewards/rejected": -3.03125,
      "step": 3490
    },
    {
      "epoch": 0.13027861011334238,
      "grad_norm": 7.1598686800960385,
      "learning_rate": 4.986063741476759e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.453125,
      "logps/chosen": -434.0,
      "logps/rejected": -544.0,
      "loss": 0.4758,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.078125,
      "rewards/margins": 1.265625,
      "rewards/rejected": -3.34375,
      "step": 3500
    },
    {
      "epoch": 0.13065083471366623,
      "grad_norm": 9.076592574481234,
      "learning_rate": 4.985719126407912e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -410.0,
      "logps/rejected": -494.0,
      "loss": 0.4536,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.0625,
      "rewards/margins": 1.125,
      "rewards/rejected": -3.1875,
      "step": 3510
    },
    {
      "epoch": 0.13102305931399005,
      "grad_norm": 8.51547003115204,
      "learning_rate": 4.985370314609426e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.5,
      "logps/chosen": -412.0,
      "logps/rejected": -516.0,
      "loss": 0.4544,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.890625,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -3.109375,
      "step": 3520
    },
    {
      "epoch": 0.1313952839143139,
      "grad_norm": 7.764126343546897,
      "learning_rate": 4.985017306670216e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.21875,
      "logps/chosen": -396.0,
      "logps/rejected": -504.0,
      "loss": 0.4712,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 1.296875,
      "rewards/rejected": -3.125,
      "step": 3530
    },
    {
      "epoch": 0.13176750851463773,
      "grad_norm": 9.100448918057232,
      "learning_rate": 4.984660103186278e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.5,
      "logps/chosen": -410.0,
      "logps/rejected": -512.0,
      "loss": 0.4833,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.921875,
      "rewards/margins": 1.125,
      "rewards/rejected": -3.046875,
      "step": 3540
    },
    {
      "epoch": 0.13213973311496158,
      "grad_norm": 7.436429344590512,
      "learning_rate": 4.984298704760689e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.375,
      "logps/chosen": -412.0,
      "logps/rejected": -502.0,
      "loss": 0.4559,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -2.953125,
      "step": 3550
    },
    {
      "epoch": 0.1325119577152854,
      "grad_norm": 8.320224347439524,
      "learning_rate": 4.983933112003615e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.328125,
      "logps/chosen": -370.0,
      "logps/rejected": -470.0,
      "loss": 0.4661,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.65625,
      "rewards/margins": 1.203125,
      "rewards/rejected": -2.859375,
      "step": 3560
    },
    {
      "epoch": 0.13288418231560925,
      "grad_norm": 6.786195093387421,
      "learning_rate": 4.983563325532295e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.390625,
      "logps/chosen": -382.0,
      "logps/rejected": -480.0,
      "loss": 0.4587,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.859375,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -2.953125,
      "step": 3570
    },
    {
      "epoch": 0.13325640691593307,
      "grad_norm": 9.8846334503168,
      "learning_rate": 4.983189345971056e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.515625,
      "logps/chosen": -410.0,
      "logps/rejected": -504.0,
      "loss": 0.4609,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 1.109375,
      "rewards/rejected": -3.078125,
      "step": 3580
    },
    {
      "epoch": 0.1336286315162569,
      "grad_norm": 9.69505232059398,
      "learning_rate": 4.982811173951301e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.328125,
      "logps/chosen": -372.0,
      "logps/rejected": -456.0,
      "loss": 0.4636,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 1.0,
      "rewards/rejected": -2.84375,
      "step": 3590
    },
    {
      "epoch": 0.13400085611658075,
      "grad_norm": 7.971039581446827,
      "learning_rate": 4.982428810111512e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.484375,
      "logps/chosen": -428.0,
      "logps/rejected": -516.0,
      "loss": 0.482,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.03125,
      "rewards/margins": 0.921875,
      "rewards/rejected": -2.953125,
      "step": 3600
    },
    {
      "epoch": 0.13437308071690457,
      "grad_norm": 7.312207960715426,
      "learning_rate": 4.982042255097245e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.328125,
      "logps/chosen": -408.0,
      "logps/rejected": -510.0,
      "loss": 0.4653,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -3.15625,
      "step": 3610
    },
    {
      "epoch": 0.13474530531722842,
      "grad_norm": 8.873088856646056,
      "learning_rate": 4.981651509561137e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.453125,
      "logps/chosen": -398.0,
      "logps/rejected": -466.0,
      "loss": 0.4729,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 0.99609375,
      "rewards/rejected": -2.8125,
      "step": 3620
    },
    {
      "epoch": 0.13511752991755224,
      "grad_norm": 7.569924570699446,
      "learning_rate": 4.981256574162897e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.21875,
      "logps/chosen": -384.0,
      "logps/rejected": -480.0,
      "loss": 0.4674,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -2.96875,
      "step": 3630
    },
    {
      "epoch": 0.1354897545178761,
      "grad_norm": 8.51997933251689,
      "learning_rate": 4.980857449569309e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.4375,
      "logps/chosen": -404.0,
      "logps/rejected": -496.0,
      "loss": 0.4828,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 0.99609375,
      "rewards/rejected": -2.96875,
      "step": 3640
    },
    {
      "epoch": 0.13586197911819992,
      "grad_norm": 9.419546365447536,
      "learning_rate": 4.98045413645423e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.34375,
      "logps/chosen": -426.0,
      "logps/rejected": -532.0,
      "loss": 0.4486,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.953125,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -3.296875,
      "step": 3650
    },
    {
      "epoch": 0.13623420371852377,
      "grad_norm": 8.222058395846904,
      "learning_rate": 4.980046635498589e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.203125,
      "logps/chosen": -400.0,
      "logps/rejected": -512.0,
      "loss": 0.4902,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.109375,
      "rewards/margins": 1.125,
      "rewards/rejected": -3.234375,
      "step": 3660
    },
    {
      "epoch": 0.1366064283188476,
      "grad_norm": 8.362855685326053,
      "learning_rate": 4.979634947390381e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.3125,
      "logps/chosen": -400.0,
      "logps/rejected": -490.0,
      "loss": 0.4682,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 0.98828125,
      "rewards/rejected": -2.875,
      "step": 3670
    },
    {
      "epoch": 0.13697865291917144,
      "grad_norm": 7.890408307852161,
      "learning_rate": 4.979219072824678e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.34375,
      "logps/chosen": -380.0,
      "logps/rejected": -466.0,
      "loss": 0.4663,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 1.0859375,
      "rewards/rejected": -2.921875,
      "step": 3680
    },
    {
      "epoch": 0.13735087751949526,
      "grad_norm": 8.413585275758038,
      "learning_rate": 4.978799012503614e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.3125,
      "logps/chosen": -414.0,
      "logps/rejected": -520.0,
      "loss": 0.457,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 1.1796875,
      "rewards/rejected": -3.15625,
      "step": 3690
    },
    {
      "epoch": 0.13772310211981909,
      "grad_norm": 9.014600892837663,
      "learning_rate": 4.978374767136391e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -408.0,
      "logps/rejected": -492.0,
      "loss": 0.4754,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.015625,
      "rewards/margins": 1.0625,
      "rewards/rejected": -3.078125,
      "step": 3700
    },
    {
      "epoch": 0.13809532672014294,
      "grad_norm": 8.425606149616552,
      "learning_rate": 4.977946337439281e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.53125,
      "logps/chosen": -450.0,
      "logps/rejected": -516.0,
      "loss": 0.4484,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.046875,
      "rewards/margins": 1.078125,
      "rewards/rejected": -3.125,
      "step": 3710
    },
    {
      "epoch": 0.13846755132046676,
      "grad_norm": 7.096379742044035,
      "learning_rate": 4.977513724135615e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.296875,
      "logps/chosen": -406.0,
      "logps/rejected": -524.0,
      "loss": 0.4663,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.984375,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -3.1875,
      "step": 3720
    },
    {
      "epoch": 0.1388397759207906,
      "grad_norm": 7.333259822605454,
      "learning_rate": 4.977076927955792e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.40625,
      "logps/chosen": -410.0,
      "logps/rejected": -496.0,
      "loss": 0.4661,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.96875,
      "rewards/margins": 1.0546875,
      "rewards/rejected": -3.03125,
      "step": 3730
    },
    {
      "epoch": 0.13921200052111443,
      "grad_norm": 8.96355267364442,
      "learning_rate": 4.976635949637268e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.328125,
      "logps/chosen": -400.0,
      "logps/rejected": -494.0,
      "loss": 0.4517,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 1.15625,
      "rewards/rejected": -3.09375,
      "step": 3740
    },
    {
      "epoch": 0.13958422512143828,
      "grad_norm": 7.679811704196777,
      "learning_rate": 4.976190789924563e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.421875,
      "logps/chosen": -386.0,
      "logps/rejected": -478.0,
      "loss": 0.455,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.859375,
      "rewards/margins": 1.03125,
      "rewards/rejected": -2.90625,
      "step": 3750
    },
    {
      "epoch": 0.1399564497217621,
      "grad_norm": 8.39087550476964,
      "learning_rate": 4.975741449569258e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.46875,
      "logps/chosen": -358.0,
      "logps/rejected": -430.0,
      "loss": 0.4806,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 0.81640625,
      "rewards/rejected": -2.53125,
      "step": 3760
    },
    {
      "epoch": 0.14032867432208596,
      "grad_norm": 7.928638771163466,
      "learning_rate": 4.975287929329989e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.5,
      "logps/chosen": -412.0,
      "logps/rejected": -506.0,
      "loss": 0.4709,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 1.0703125,
      "rewards/rejected": -3.0,
      "step": 3770
    },
    {
      "epoch": 0.14070089892240978,
      "grad_norm": 7.5120108216401915,
      "learning_rate": 4.974830229972452e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.421875,
      "logps/chosen": -384.0,
      "logps/rejected": -484.0,
      "loss": 0.4527,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.8984375,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -3.0,
      "step": 3780
    },
    {
      "epoch": 0.14107312352273363,
      "grad_norm": 7.3515781300870495,
      "learning_rate": 4.974368352269397e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.515625,
      "logps/chosen": -392.0,
      "logps/rejected": -496.0,
      "loss": 0.4457,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.953125,
      "rewards/margins": 1.0078125,
      "rewards/rejected": -2.96875,
      "step": 3790
    },
    {
      "epoch": 0.14144534812305745,
      "grad_norm": 9.5175100527338,
      "learning_rate": 4.973902297000628e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.359375,
      "logps/chosen": -410.0,
      "logps/rejected": -482.0,
      "loss": 0.461,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.0625,
      "rewards/margins": 0.9453125,
      "rewards/rejected": -3.015625,
      "step": 3800
    },
    {
      "epoch": 0.14181757272338127,
      "grad_norm": 8.035774435477403,
      "learning_rate": 4.973432064953004e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.40625,
      "logps/chosen": -436.0,
      "logps/rejected": -536.0,
      "loss": 0.4699,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.3125,
      "rewards/margins": 1.046875,
      "rewards/rejected": -3.359375,
      "step": 3810
    },
    {
      "epoch": 0.14218979732370512,
      "grad_norm": 9.626552000891268,
      "learning_rate": 4.972957656920434e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.375,
      "logps/chosen": -406.0,
      "logps/rejected": -524.0,
      "loss": 0.4425,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -3.25,
      "step": 3820
    },
    {
      "epoch": 0.14256202192402895,
      "grad_norm": 7.51875092950856,
      "learning_rate": 4.972479073703879e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -460.0,
      "logps/rejected": -544.0,
      "loss": 0.48,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.25,
      "rewards/margins": 0.9765625,
      "rewards/rejected": -3.234375,
      "step": 3830
    },
    {
      "epoch": 0.1429342465243528,
      "grad_norm": 11.379555801291795,
      "learning_rate": 4.971996316111347e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.40625,
      "logps/chosen": -448.0,
      "logps/rejected": -516.0,
      "loss": 0.4582,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.203125,
      "rewards/margins": 1.09375,
      "rewards/rejected": -3.296875,
      "step": 3840
    },
    {
      "epoch": 0.14330647112467662,
      "grad_norm": 9.355083808581167,
      "learning_rate": 4.971509384957899e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.484375,
      "logps/chosen": -378.0,
      "logps/rejected": -492.0,
      "loss": 0.4513,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 1.0625,
      "rewards/rejected": -3.0,
      "step": 3850
    },
    {
      "epoch": 0.14367869572500047,
      "grad_norm": 7.707778136368054,
      "learning_rate": 4.971018281065632e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.65625,
      "logps/chosen": -430.0,
      "logps/rejected": -524.0,
      "loss": 0.431,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.015625,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -3.234375,
      "step": 3860
    },
    {
      "epoch": 0.1440509203253243,
      "grad_norm": 8.002075733355333,
      "learning_rate": 4.9705230052637e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -444.0,
      "logps/rejected": -532.0,
      "loss": 0.4797,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.078125,
      "rewards/margins": 1.1875,
      "rewards/rejected": -3.265625,
      "step": 3870
    },
    {
      "epoch": 0.14442314492564814,
      "grad_norm": 7.158476221717491,
      "learning_rate": 4.970023558388294e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.46875,
      "logps/chosen": -406.0,
      "logps/rejected": -488.0,
      "loss": 0.4605,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.984375,
      "rewards/margins": 0.98828125,
      "rewards/rejected": -2.984375,
      "step": 3880
    },
    {
      "epoch": 0.14479536952597197,
      "grad_norm": 7.706695730639356,
      "learning_rate": 4.969519941282647e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.53125,
      "logps/chosen": -424.0,
      "logps/rejected": -500.0,
      "loss": 0.4448,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.953125,
      "rewards/margins": 0.984375,
      "rewards/rejected": -2.9375,
      "step": 3890
    },
    {
      "epoch": 0.14516759412629582,
      "grad_norm": 20.896064236066195,
      "learning_rate": 4.969012154797034e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.390625,
      "logps/chosen": -394.0,
      "logps/rejected": -508.0,
      "loss": 0.468,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.875,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -3.1875,
      "step": 3900
    },
    {
      "epoch": 0.14553981872661964,
      "grad_norm": 12.880327151301831,
      "learning_rate": 4.96850019978877e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.3125,
      "logps/chosen": -384.0,
      "logps/rejected": -516.0,
      "loss": 0.4658,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 1.328125,
      "rewards/rejected": -3.21875,
      "step": 3910
    },
    {
      "epoch": 0.1459120433269435,
      "grad_norm": 8.65456679974486,
      "learning_rate": 4.967984077122207e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.546875,
      "logps/chosen": -392.0,
      "logps/rejected": -490.0,
      "loss": 0.472,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 1.0625,
      "rewards/rejected": -2.9375,
      "step": 3920
    },
    {
      "epoch": 0.14628426792726731,
      "grad_norm": 7.8639305086381635,
      "learning_rate": 4.967463787668734e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.625,
      "logps/chosen": -402.0,
      "logps/rejected": -502.0,
      "loss": 0.4454,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 1.1875,
      "rewards/rejected": -2.9375,
      "step": 3930
    },
    {
      "epoch": 0.14665649252759114,
      "grad_norm": 8.857727592734182,
      "learning_rate": 4.966939332306773e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.421875,
      "logps/chosen": -404.0,
      "logps/rejected": -508.0,
      "loss": 0.4616,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.953125,
      "rewards/margins": 0.953125,
      "rewards/rejected": -2.90625,
      "step": 3940
    },
    {
      "epoch": 0.147028717127915,
      "grad_norm": 8.881551045200165,
      "learning_rate": 4.966410711921784e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.375,
      "logps/chosen": -404.0,
      "logps/rejected": -482.0,
      "loss": 0.4564,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.09375,
      "rewards/margins": 0.88671875,
      "rewards/rejected": -2.984375,
      "step": 3950
    },
    {
      "epoch": 0.1474009417282388,
      "grad_norm": 9.352554578123376,
      "learning_rate": 4.965877927406252e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.40625,
      "logps/chosen": -390.0,
      "logps/rejected": -462.0,
      "loss": 0.4508,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 0.921875,
      "rewards/rejected": -2.875,
      "step": 3960
    },
    {
      "epoch": 0.14777316632856266,
      "grad_norm": 9.157476582222554,
      "learning_rate": 4.965340979659699e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.40625,
      "logps/chosen": -392.0,
      "logps/rejected": -496.0,
      "loss": 0.4728,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -2.984375,
      "step": 3970
    },
    {
      "epoch": 0.14814539092888648,
      "grad_norm": 8.886302563727805,
      "learning_rate": 4.964799869588673e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.375,
      "logps/chosen": -386.0,
      "logps/rejected": -508.0,
      "loss": 0.4737,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 1.234375,
      "rewards/rejected": -3.046875,
      "step": 3980
    },
    {
      "epoch": 0.14851761552921033,
      "grad_norm": 7.720707907776766,
      "learning_rate": 4.964254598106751e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -436.0,
      "logps/rejected": -524.0,
      "loss": 0.4585,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.109375,
      "rewards/margins": 1.0546875,
      "rewards/rejected": -3.15625,
      "step": 3990
    },
    {
      "epoch": 0.14888984012953416,
      "grad_norm": 7.984270063940856,
      "learning_rate": 4.96370516613453e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.484375,
      "logps/chosen": -436.0,
      "logps/rejected": -516.0,
      "loss": 0.4747,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.203125,
      "rewards/margins": 0.83984375,
      "rewards/rejected": -3.046875,
      "step": 4000
    },
    {
      "epoch": 0.149262064729858,
      "grad_norm": 7.545085659682554,
      "learning_rate": 4.963151574599641e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.34375,
      "logps/chosen": -370.0,
      "logps/rejected": -488.0,
      "loss": 0.4672,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.7578125,
      "rewards/margins": 1.203125,
      "rewards/rejected": -2.96875,
      "step": 4010
    },
    {
      "epoch": 0.14963428933018183,
      "grad_norm": 8.523065552038032,
      "learning_rate": 4.962593824436731e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.578125,
      "logps/chosen": -414.0,
      "logps/rejected": -536.0,
      "loss": 0.4477,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.0625,
      "rewards/margins": 1.125,
      "rewards/rejected": -3.171875,
      "step": 4020
    },
    {
      "epoch": 0.15000651393050568,
      "grad_norm": 9.015656952915249,
      "learning_rate": 4.962031916587469e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.46875,
      "logps/chosen": -406.0,
      "logps/rejected": -516.0,
      "loss": 0.4611,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.203125,
      "rewards/margins": 1.1875,
      "rewards/rejected": -3.390625,
      "step": 4030
    },
    {
      "epoch": 0.1503787385308295,
      "grad_norm": 7.846613795992775,
      "learning_rate": 4.961465852000545e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.53125,
      "logps/chosen": -434.0,
      "logps/rejected": -536.0,
      "loss": 0.4738,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.21875,
      "rewards/margins": 1.0703125,
      "rewards/rejected": -3.296875,
      "step": 4040
    },
    {
      "epoch": 0.15075096313115333,
      "grad_norm": 9.364722319332515,
      "learning_rate": 4.960895631631665e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.609375,
      "logps/chosen": -424.0,
      "logps/rejected": -502.0,
      "loss": 0.4625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.0,
      "rewards/margins": 1.203125,
      "rewards/rejected": -3.203125,
      "step": 4050
    },
    {
      "epoch": 0.15112318773147718,
      "grad_norm": 7.153001625389234,
      "learning_rate": 4.960321256443555e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.6875,
      "logps/chosen": -462.0,
      "logps/rejected": -556.0,
      "loss": 0.4641,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.140625,
      "rewards/margins": 1.3125,
      "rewards/rejected": -3.4375,
      "step": 4060
    },
    {
      "epoch": 0.151495412331801,
      "grad_norm": 7.957660377101739,
      "learning_rate": 4.959742727405952e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.609375,
      "logps/chosen": -434.0,
      "logps/rejected": -512.0,
      "loss": 0.4467,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.125,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -3.15625,
      "step": 4070
    },
    {
      "epoch": 0.15186763693212485,
      "grad_norm": 8.187993255050667,
      "learning_rate": 4.959160045495607e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.5,
      "logps/chosen": -404.0,
      "logps/rejected": -504.0,
      "loss": 0.4635,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.90625,
      "rewards/margins": 1.046875,
      "rewards/rejected": -2.953125,
      "step": 4080
    },
    {
      "epoch": 0.15223986153244867,
      "grad_norm": 9.024542786878762,
      "learning_rate": 4.958573211696285e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5,
      "logps/chosen": -422.0,
      "logps/rejected": -532.0,
      "loss": 0.4576,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.109375,
      "rewards/margins": 1.1328125,
      "rewards/rejected": -3.25,
      "step": 4090
    },
    {
      "epoch": 0.15261208613277252,
      "grad_norm": 8.773269572079794,
      "learning_rate": 4.957982226998757e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -404.0,
      "logps/rejected": -532.0,
      "loss": 0.4736,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.125,
      "rewards/margins": 1.1875,
      "rewards/rejected": -3.3125,
      "step": 4100
    },
    {
      "epoch": 0.15298431073309635,
      "grad_norm": 8.521177422080484,
      "learning_rate": 4.957387092400805e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.4375,
      "logps/chosen": -408.0,
      "logps/rejected": -500.0,
      "loss": 0.4402,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 1.09375,
      "rewards/rejected": -3.09375,
      "step": 4110
    },
    {
      "epoch": 0.1533565353334202,
      "grad_norm": 8.22488339924465,
      "learning_rate": 4.956787808907215e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -442.0,
      "logps/rejected": -512.0,
      "loss": 0.4712,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -2.328125,
      "rewards/margins": 0.796875,
      "rewards/rejected": -3.125,
      "step": 4120
    },
    {
      "epoch": 0.15372875993374402,
      "grad_norm": 7.758398462435562,
      "learning_rate": 4.95618437752978e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.359375,
      "logps/chosen": -448.0,
      "logps/rejected": -552.0,
      "loss": 0.4693,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.21875,
      "rewards/margins": 1.28125,
      "rewards/rejected": -3.5,
      "step": 4130
    },
    {
      "epoch": 0.15410098453406787,
      "grad_norm": 8.21749223363518,
      "learning_rate": 4.955576799287296e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.46875,
      "logps/chosen": -416.0,
      "logps/rejected": -500.0,
      "loss": 0.4695,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.015625,
      "rewards/margins": 1.0859375,
      "rewards/rejected": -3.109375,
      "step": 4140
    },
    {
      "epoch": 0.1544732091343917,
      "grad_norm": 7.838382736541255,
      "learning_rate": 4.954965075205556e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -402.0,
      "logps/rejected": -468.0,
      "loss": 0.4537,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.015625,
      "rewards/margins": 0.87890625,
      "rewards/rejected": -2.90625,
      "step": 4150
    },
    {
      "epoch": 0.15484543373471552,
      "grad_norm": 9.364725177137592,
      "learning_rate": 4.954349206317359e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -426.0,
      "logps/rejected": -528.0,
      "loss": 0.4509,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.21875,
      "rewards/margins": 1.109375,
      "rewards/rejected": -3.328125,
      "step": 4160
    },
    {
      "epoch": 0.15521765833503937,
      "grad_norm": 8.034510680925932,
      "learning_rate": 4.953729193662498e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -434.0,
      "logps/rejected": -528.0,
      "loss": 0.4415,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.328125,
      "rewards/margins": 1.0859375,
      "rewards/rejected": -3.40625,
      "step": 4170
    },
    {
      "epoch": 0.1555898829353632,
      "grad_norm": 8.821479569925456,
      "learning_rate": 4.953105038287762e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.734375,
      "logps/chosen": -454.0,
      "logps/rejected": -532.0,
      "loss": 0.4611,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.28125,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -3.328125,
      "step": 4180
    },
    {
      "epoch": 0.15596210753568704,
      "grad_norm": 7.655859742543528,
      "learning_rate": 4.952476741246937e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -414.0,
      "logps/rejected": -506.0,
      "loss": 0.4596,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -2.21875,
      "rewards/margins": 0.9375,
      "rewards/rejected": -3.15625,
      "step": 4190
    },
    {
      "epoch": 0.15633433213601086,
      "grad_norm": 8.256390055419596,
      "learning_rate": 4.951844303600798e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.609375,
      "logps/chosen": -428.0,
      "logps/rejected": -524.0,
      "loss": 0.4529,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.140625,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -3.1875,
      "step": 4200
    },
    {
      "epoch": 0.1567065567363347,
      "grad_norm": 8.412317495679707,
      "learning_rate": 4.951207726417113e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5,
      "logps/chosen": -414.0,
      "logps/rejected": -504.0,
      "loss": 0.4763,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.03125,
      "rewards/margins": 1.1484375,
      "rewards/rejected": -3.1875,
      "step": 4210
    },
    {
      "epoch": 0.15707878133665854,
      "grad_norm": 8.21492318733733,
      "learning_rate": 4.950567010770638e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.421875,
      "logps/chosen": -398.0,
      "logps/rejected": -496.0,
      "loss": 0.4518,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.984375,
      "rewards/margins": 1.171875,
      "rewards/rejected": -3.15625,
      "step": 4220
    },
    {
      "epoch": 0.15745100593698239,
      "grad_norm": 8.809952405781551,
      "learning_rate": 4.949922157743116e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.578125,
      "logps/chosen": -434.0,
      "logps/rejected": -540.0,
      "loss": 0.471,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.1875,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -3.234375,
      "step": 4230
    },
    {
      "epoch": 0.1578232305373062,
      "grad_norm": 9.586905428528063,
      "learning_rate": 4.949273168423277e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -452.0,
      "logps/rejected": -536.0,
      "loss": 0.4479,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.265625,
      "rewards/margins": 1.078125,
      "rewards/rejected": -3.34375,
      "step": 4240
    },
    {
      "epoch": 0.15819545513763006,
      "grad_norm": 8.740615411485694,
      "learning_rate": 4.948620043906832e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.359375,
      "logps/chosen": -422.0,
      "logps/rejected": -532.0,
      "loss": 0.456,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.046875,
      "rewards/margins": 1.25,
      "rewards/rejected": -3.3125,
      "step": 4250
    },
    {
      "epoch": 0.15856767973795388,
      "grad_norm": 7.282306049394732,
      "learning_rate": 4.947962785296475e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.671875,
      "logps/chosen": -420.0,
      "logps/rejected": -528.0,
      "loss": 0.4418,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.03125,
      "rewards/margins": 1.234375,
      "rewards/rejected": -3.265625,
      "step": 4260
    },
    {
      "epoch": 0.1589399043382777,
      "grad_norm": 7.617302806519537,
      "learning_rate": 4.947301393701881e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.6875,
      "logps/chosen": -428.0,
      "logps/rejected": -510.0,
      "loss": 0.4317,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.0625,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -3.234375,
      "step": 4270
    },
    {
      "epoch": 0.15931212893860155,
      "grad_norm": 9.943588478839729,
      "learning_rate": 4.946635870239699e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -440.0,
      "logps/rejected": -540.0,
      "loss": 0.4548,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.3125,
      "rewards/margins": 1.140625,
      "rewards/rejected": -3.453125,
      "step": 4280
    },
    {
      "epoch": 0.15968435353892538,
      "grad_norm": 8.243902262954158,
      "learning_rate": 4.945966216033558e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.609375,
      "logps/chosen": -444.0,
      "logps/rejected": -512.0,
      "loss": 0.4497,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.125,
      "rewards/margins": 1.0546875,
      "rewards/rejected": -3.1875,
      "step": 4290
    },
    {
      "epoch": 0.16005657813924923,
      "grad_norm": 7.877219473548476,
      "learning_rate": 4.945292432214059e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.609375,
      "logps/chosen": -416.0,
      "logps/rejected": -512.0,
      "loss": 0.4391,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.1875,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -3.296875,
      "step": 4300
    },
    {
      "epoch": 0.16042880273957305,
      "grad_norm": 9.225930016130667,
      "learning_rate": 4.944614519918775e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.34375,
      "logps/chosen": -422.0,
      "logps/rejected": -548.0,
      "loss": 0.4642,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.34375,
      "rewards/margins": 1.21875,
      "rewards/rejected": -3.5625,
      "step": 4310
    },
    {
      "epoch": 0.1608010273398969,
      "grad_norm": 8.940081252536402,
      "learning_rate": 4.943932480292251e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.640625,
      "logps/chosen": -440.0,
      "logps/rejected": -524.0,
      "loss": 0.4526,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.25,
      "rewards/margins": 1.09375,
      "rewards/rejected": -3.34375,
      "step": 4320
    },
    {
      "epoch": 0.16117325194022072,
      "grad_norm": 9.595103623973435,
      "learning_rate": 4.943246314485999e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.625,
      "logps/chosen": -430.0,
      "logps/rejected": -506.0,
      "loss": 0.4609,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.046875,
      "rewards/margins": 1.015625,
      "rewards/rejected": -3.0625,
      "step": 4330
    },
    {
      "epoch": 0.16154547654054457,
      "grad_norm": 8.329898659063007,
      "learning_rate": 4.942556023658497e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.671875,
      "logps/chosen": -428.0,
      "logps/rejected": -486.0,
      "loss": 0.4503,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.078125,
      "rewards/margins": 1.0,
      "rewards/rejected": -3.078125,
      "step": 4340
    },
    {
      "epoch": 0.1619177011408684,
      "grad_norm": 9.975464930685105,
      "learning_rate": 4.941861608975188e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.578125,
      "logps/chosen": -440.0,
      "logps/rejected": -560.0,
      "loss": 0.4641,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.234375,
      "rewards/margins": 1.34375,
      "rewards/rejected": -3.578125,
      "step": 4350
    },
    {
      "epoch": 0.16228992574119225,
      "grad_norm": 6.463627818041486,
      "learning_rate": 4.941163071608479e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.625,
      "logps/chosen": -412.0,
      "logps/rejected": -528.0,
      "loss": 0.4469,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.0625,
      "rewards/margins": 1.390625,
      "rewards/rejected": -3.453125,
      "step": 4360
    },
    {
      "epoch": 0.16266215034151607,
      "grad_norm": 8.786187518066392,
      "learning_rate": 4.940460412737733e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.75,
      "logps/chosen": -428.0,
      "logps/rejected": -540.0,
      "loss": 0.446,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.203125,
      "rewards/margins": 1.234375,
      "rewards/rejected": -3.4375,
      "step": 4370
    },
    {
      "epoch": 0.1630343749418399,
      "grad_norm": 8.439994711250089,
      "learning_rate": 4.939753633549277e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.515625,
      "logps/chosen": -402.0,
      "logps/rejected": -508.0,
      "loss": 0.4452,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.1875,
      "rewards/margins": 1.015625,
      "rewards/rejected": -3.203125,
      "step": 4380
    },
    {
      "epoch": 0.16340659954216374,
      "grad_norm": 7.905261519338113,
      "learning_rate": 4.93904273523639e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.46875,
      "logps/chosen": -420.0,
      "logps/rejected": -540.0,
      "loss": 0.4699,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.140625,
      "rewards/margins": 1.1875,
      "rewards/rejected": -3.328125,
      "step": 4390
    },
    {
      "epoch": 0.16377882414248757,
      "grad_norm": 7.246996154779962,
      "learning_rate": 4.93832771899931e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.546875,
      "logps/chosen": -406.0,
      "logps/rejected": -478.0,
      "loss": 0.445,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 1.0,
      "rewards/rejected": -2.984375,
      "step": 4400
    },
    {
      "epoch": 0.16415104874281142,
      "grad_norm": 8.913028423590184,
      "learning_rate": 4.937608586045223e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.5625,
      "logps/chosen": -414.0,
      "logps/rejected": -516.0,
      "loss": 0.4513,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1875,
      "rewards/margins": 1.0859375,
      "rewards/rejected": -3.28125,
      "step": 4410
    },
    {
      "epoch": 0.16452327334313524,
      "grad_norm": 8.220569313822812,
      "learning_rate": 4.936885337588266e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -436.0,
      "logps/rejected": -532.0,
      "loss": 0.456,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.21875,
      "rewards/margins": 1.09375,
      "rewards/rejected": -3.3125,
      "step": 4420
    },
    {
      "epoch": 0.1648954979434591,
      "grad_norm": 6.895438311620862,
      "learning_rate": 4.936157974849528e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.515625,
      "logps/chosen": -400.0,
      "logps/rejected": -496.0,
      "loss": 0.4699,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.0,
      "rewards/margins": 1.203125,
      "rewards/rejected": -3.203125,
      "step": 4430
    },
    {
      "epoch": 0.1652677225437829,
      "grad_norm": 7.359690715076742,
      "learning_rate": 4.93542649905704e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.53125,
      "logps/chosen": -414.0,
      "logps/rejected": -502.0,
      "loss": 0.4491,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.15625,
      "rewards/margins": 0.97265625,
      "rewards/rejected": -3.125,
      "step": 4440
    },
    {
      "epoch": 0.16563994714410676,
      "grad_norm": 8.268694230169748,
      "learning_rate": 4.934690911445782e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -424.0,
      "logps/rejected": -536.0,
      "loss": 0.4346,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.234375,
      "rewards/margins": 1.1328125,
      "rewards/rejected": -3.359375,
      "step": 4450
    },
    {
      "epoch": 0.1660121717444306,
      "grad_norm": 8.506306194149925,
      "learning_rate": 4.933951213257669e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -468.0,
      "logps/rejected": -568.0,
      "loss": 0.4696,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.46875,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -3.5,
      "step": 4460
    },
    {
      "epoch": 0.16638439634475444,
      "grad_norm": 10.039995978998382,
      "learning_rate": 4.933207405741563e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.765625,
      "logps/chosen": -428.0,
      "logps/rejected": -516.0,
      "loss": 0.4528,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.109375,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -3.15625,
      "step": 4470
    },
    {
      "epoch": 0.16675662094507826,
      "grad_norm": 8.374175135846649,
      "learning_rate": 4.93245949015326e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.65625,
      "logps/chosen": -420.0,
      "logps/rejected": -508.0,
      "loss": 0.4545,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.03125,
      "rewards/margins": 1.15625,
      "rewards/rejected": -3.1875,
      "step": 4480
    },
    {
      "epoch": 0.16712884554540208,
      "grad_norm": 8.064150750654337,
      "learning_rate": 4.931707467755495e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.65625,
      "logps/chosen": -454.0,
      "logps/rejected": -572.0,
      "loss": 0.4532,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.296875,
      "rewards/margins": 1.3125,
      "rewards/rejected": -3.609375,
      "step": 4490
    },
    {
      "epoch": 0.16750107014572593,
      "grad_norm": 8.852270560058958,
      "learning_rate": 4.930951339817932e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.671875,
      "logps/chosen": -422.0,
      "logps/rejected": -512.0,
      "loss": 0.4539,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.28125,
      "rewards/margins": 1.15625,
      "rewards/rejected": -3.4375,
      "step": 4500
    },
    {
      "epoch": 0.16787329474604976,
      "grad_norm": 9.316238371514368,
      "learning_rate": 4.93019110761717e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.484375,
      "logps/chosen": -402.0,
      "logps/rejected": -504.0,
      "loss": 0.4711,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.03125,
      "rewards/margins": 1.09375,
      "rewards/rejected": -3.140625,
      "step": 4510
    },
    {
      "epoch": 0.1682455193463736,
      "grad_norm": 7.579499763210613,
      "learning_rate": 4.929426772436738e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -414.0,
      "logps/rejected": -512.0,
      "loss": 0.4746,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.1875,
      "rewards/margins": 1.046875,
      "rewards/rejected": -3.234375,
      "step": 4520
    },
    {
      "epoch": 0.16861774394669743,
      "grad_norm": 10.03899288557987,
      "learning_rate": 4.928658335567088e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.71875,
      "logps/chosen": -448.0,
      "logps/rejected": -540.0,
      "loss": 0.4545,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.3125,
      "rewards/margins": 1.125,
      "rewards/rejected": -3.4375,
      "step": 4530
    },
    {
      "epoch": 0.16898996854702128,
      "grad_norm": 7.68097972350197,
      "learning_rate": 4.927885798305603e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -430.0,
      "logps/rejected": -552.0,
      "loss": 0.4389,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.1875,
      "rewards/margins": 1.46875,
      "rewards/rejected": -3.65625,
      "step": 4540
    },
    {
      "epoch": 0.1693621931473451,
      "grad_norm": 7.617017588483837,
      "learning_rate": 4.927109161956584e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.671875,
      "logps/chosen": -444.0,
      "logps/rejected": -524.0,
      "loss": 0.467,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.359375,
      "rewards/margins": 0.90234375,
      "rewards/rejected": -3.265625,
      "step": 4550
    },
    {
      "epoch": 0.16973441774766895,
      "grad_norm": 9.626709871569645,
      "learning_rate": 4.926328427831254e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.703125,
      "logps/chosen": -440.0,
      "logps/rejected": -524.0,
      "loss": 0.4573,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.21875,
      "rewards/margins": 1.125,
      "rewards/rejected": -3.34375,
      "step": 4560
    },
    {
      "epoch": 0.17010664234799278,
      "grad_norm": 7.899708120514527,
      "learning_rate": 4.925543597247756e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.609375,
      "logps/chosen": -424.0,
      "logps/rejected": -580.0,
      "loss": 0.4402,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.21875,
      "rewards/margins": 1.578125,
      "rewards/rejected": -3.796875,
      "step": 4570
    },
    {
      "epoch": 0.17047886694831663,
      "grad_norm": 7.995480614021097,
      "learning_rate": 4.924754671531145e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.4375,
      "logps/chosen": -434.0,
      "logps/rejected": -548.0,
      "loss": 0.4152,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.296875,
      "rewards/margins": 1.265625,
      "rewards/rejected": -3.5625,
      "step": 4580
    },
    {
      "epoch": 0.17085109154864045,
      "grad_norm": 8.278165988474699,
      "learning_rate": 4.923961652013396e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.53125,
      "logps/chosen": -446.0,
      "logps/rejected": -548.0,
      "loss": 0.4513,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.3125,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -3.5,
      "step": 4590
    },
    {
      "epoch": 0.17122331614896427,
      "grad_norm": 8.321535000785424,
      "learning_rate": 4.923164540033392e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.5625,
      "logps/chosen": -392.0,
      "logps/rejected": -476.0,
      "loss": 0.457,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 1.03125,
      "rewards/rejected": -3.015625,
      "step": 4600
    },
    {
      "epoch": 0.17159554074928812,
      "grad_norm": 6.880612596573222,
      "learning_rate": 4.922363336936926e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.53125,
      "logps/chosen": -424.0,
      "logps/rejected": -512.0,
      "loss": 0.45,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.109375,
      "rewards/margins": 1.1484375,
      "rewards/rejected": -3.265625,
      "step": 4610
    },
    {
      "epoch": 0.17196776534961195,
      "grad_norm": 6.724959195308666,
      "learning_rate": 4.921558044076696e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.46875,
      "logps/chosen": -402.0,
      "logps/rejected": -520.0,
      "loss": 0.4627,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.046875,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -3.34375,
      "step": 4620
    },
    {
      "epoch": 0.1723399899499358,
      "grad_norm": 8.932507146656741,
      "learning_rate": 4.920748662812309e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.5,
      "logps/chosen": -446.0,
      "logps/rejected": -516.0,
      "loss": 0.4791,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.1875,
      "rewards/margins": 0.8828125,
      "rewards/rejected": -3.078125,
      "step": 4630
    },
    {
      "epoch": 0.17271221455025962,
      "grad_norm": 10.01825523269522,
      "learning_rate": 4.919935194510274e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.515625,
      "logps/chosen": -424.0,
      "logps/rejected": -516.0,
      "loss": 0.4579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.109375,
      "rewards/margins": 1.1328125,
      "rewards/rejected": -3.25,
      "step": 4640
    },
    {
      "epoch": 0.17308443915058347,
      "grad_norm": 7.425761617929371,
      "learning_rate": 4.919117640543996e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.4375,
      "logps/chosen": -442.0,
      "logps/rejected": -524.0,
      "loss": 0.4582,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.125,
      "rewards/margins": 1.1484375,
      "rewards/rejected": -3.265625,
      "step": 4650
    },
    {
      "epoch": 0.1734566637509073,
      "grad_norm": 8.252841608425232,
      "learning_rate": 4.918296002293782e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5625,
      "logps/chosen": -422.0,
      "logps/rejected": -500.0,
      "loss": 0.4434,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -2.09375,
      "rewards/margins": 0.859375,
      "rewards/rejected": -2.953125,
      "step": 4660
    },
    {
      "epoch": 0.17382888835123114,
      "grad_norm": 7.054513710378779,
      "learning_rate": 4.917470281146836e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.453125,
      "logps/chosen": -400.0,
      "logps/rejected": -510.0,
      "loss": 0.4463,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.109375,
      "rewards/margins": 1.203125,
      "rewards/rejected": -3.3125,
      "step": 4670
    },
    {
      "epoch": 0.17420111295155497,
      "grad_norm": 7.976999345778815,
      "learning_rate": 4.916640478497249e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5625,
      "logps/chosen": -434.0,
      "logps/rejected": -528.0,
      "loss": 0.4305,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.109375,
      "rewards/margins": 1.234375,
      "rewards/rejected": -3.34375,
      "step": 4680
    },
    {
      "epoch": 0.17457333755187882,
      "grad_norm": 8.834577919207028,
      "learning_rate": 4.91580659574601e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -454.0,
      "logps/rejected": -564.0,
      "loss": 0.4546,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.40625,
      "rewards/margins": 1.28125,
      "rewards/rejected": -3.6875,
      "step": 4690
    },
    {
      "epoch": 0.17494556215220264,
      "grad_norm": 8.265686785716541,
      "learning_rate": 4.914968634300993e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.421875,
      "logps/chosen": -416.0,
      "logps/rejected": -572.0,
      "loss": 0.4652,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.171875,
      "rewards/margins": 1.546875,
      "rewards/rejected": -3.71875,
      "step": 4700
    },
    {
      "epoch": 0.17531778675252646,
      "grad_norm": 9.117463653624723,
      "learning_rate": 4.914126595576957e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.640625,
      "logps/chosen": -428.0,
      "logps/rejected": -516.0,
      "loss": 0.4432,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 1.140625,
      "rewards/rejected": -3.125,
      "step": 4710
    },
    {
      "epoch": 0.1756900113528503,
      "grad_norm": 9.349225023329247,
      "learning_rate": 4.913280480995547e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.65625,
      "logps/chosen": -448.0,
      "logps/rejected": -528.0,
      "loss": 0.4632,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.21875,
      "rewards/margins": 0.98046875,
      "rewards/rejected": -3.203125,
      "step": 4720
    },
    {
      "epoch": 0.17606223595317413,
      "grad_norm": 10.635275705601062,
      "learning_rate": 4.912430291985291e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.546875,
      "logps/chosen": -414.0,
      "logps/rejected": -500.0,
      "loss": 0.4484,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.046875,
      "rewards/margins": 1.1875,
      "rewards/rejected": -3.234375,
      "step": 4730
    },
    {
      "epoch": 0.17643446055349798,
      "grad_norm": 13.400407677823374,
      "learning_rate": 4.911576029981591e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.46875,
      "logps/chosen": -426.0,
      "logps/rejected": -552.0,
      "loss": 0.4375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.25,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -3.5,
      "step": 4740
    },
    {
      "epoch": 0.1768066851538218,
      "grad_norm": 9.484186833270678,
      "learning_rate": 4.91071769642673e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.484375,
      "logps/chosen": -432.0,
      "logps/rejected": -544.0,
      "loss": 0.4486,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.15625,
      "rewards/margins": 1.234375,
      "rewards/rejected": -3.40625,
      "step": 4750
    },
    {
      "epoch": 0.17717890975414566,
      "grad_norm": 8.834066691884978,
      "learning_rate": 4.909855292769863e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -434.0,
      "logps/rejected": -552.0,
      "loss": 0.4695,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.171875,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -3.5,
      "step": 4760
    },
    {
      "epoch": 0.17755113435446948,
      "grad_norm": 9.03600654623299,
      "learning_rate": 4.908988820467018e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -430.0,
      "logps/rejected": -510.0,
      "loss": 0.4555,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.09375,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -3.203125,
      "step": 4770
    },
    {
      "epoch": 0.17792335895479333,
      "grad_norm": 9.445239142118064,
      "learning_rate": 4.908118280981091e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.453125,
      "logps/chosen": -422.0,
      "logps/rejected": -520.0,
      "loss": 0.4583,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.078125,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -3.28125,
      "step": 4780
    },
    {
      "epoch": 0.17829558355511715,
      "grad_norm": 7.93980004804038,
      "learning_rate": 4.907243675781847e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.375,
      "logps/chosen": -412.0,
      "logps/rejected": -492.0,
      "loss": 0.4474,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.140625,
      "rewards/margins": 1.0,
      "rewards/rejected": -3.140625,
      "step": 4790
    },
    {
      "epoch": 0.178667808155441,
      "grad_norm": 8.302955024210911,
      "learning_rate": 4.90636500634591e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.671875,
      "logps/chosen": -442.0,
      "logps/rejected": -544.0,
      "loss": 0.4457,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.28125,
      "rewards/margins": 1.0625,
      "rewards/rejected": -3.34375,
      "step": 4800
    },
    {
      "epoch": 0.17904003275576483,
      "grad_norm": 9.052021621133953,
      "learning_rate": 4.905482274156773e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.4375,
      "logps/chosen": -414.0,
      "logps/rejected": -524.0,
      "loss": 0.4548,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.0625,
      "rewards/margins": 1.125,
      "rewards/rejected": -3.1875,
      "step": 4810
    },
    {
      "epoch": 0.17941225735608865,
      "grad_norm": 9.168597952540626,
      "learning_rate": 4.904595480704784e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.484375,
      "logps/chosen": -416.0,
      "logps/rejected": -536.0,
      "loss": 0.4592,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.1875,
      "rewards/margins": 1.140625,
      "rewards/rejected": -3.328125,
      "step": 4820
    },
    {
      "epoch": 0.1797844819564125,
      "grad_norm": 8.441528931238581,
      "learning_rate": 4.903704627487148e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.703125,
      "logps/chosen": -434.0,
      "logps/rejected": -520.0,
      "loss": 0.4358,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.328125,
      "rewards/margins": 0.8828125,
      "rewards/rejected": -3.21875,
      "step": 4830
    },
    {
      "epoch": 0.18015670655673632,
      "grad_norm": 8.200574067260103,
      "learning_rate": 4.902809716007923e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.640625,
      "logps/chosen": -406.0,
      "logps/rejected": -500.0,
      "loss": 0.4581,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.140625,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -3.25,
      "step": 4840
    },
    {
      "epoch": 0.18052893115706017,
      "grad_norm": 8.08568493304088,
      "learning_rate": 4.901910747778023e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.65625,
      "logps/chosen": -414.0,
      "logps/rejected": -512.0,
      "loss": 0.4619,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.078125,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -3.171875,
      "step": 4850
    },
    {
      "epoch": 0.180901155757384,
      "grad_norm": 8.356822228293808,
      "learning_rate": 4.901007724315209e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.546875,
      "logps/chosen": -416.0,
      "logps/rejected": -528.0,
      "loss": 0.4288,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.90625,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -3.15625,
      "step": 4860
    },
    {
      "epoch": 0.18127338035770785,
      "grad_norm": 8.732334253947649,
      "learning_rate": 4.900100647144085e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -442.0,
      "logps/rejected": -536.0,
      "loss": 0.4554,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.203125,
      "rewards/margins": 1.109375,
      "rewards/rejected": -3.3125,
      "step": 4870
    },
    {
      "epoch": 0.18164560495803167,
      "grad_norm": 8.898655594842568,
      "learning_rate": 4.899189517796105e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.421875,
      "logps/chosen": -418.0,
      "logps/rejected": -524.0,
      "loss": 0.4559,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.25,
      "rewards/margins": 1.171875,
      "rewards/rejected": -3.421875,
      "step": 4880
    },
    {
      "epoch": 0.18201782955835552,
      "grad_norm": 7.3102675135991015,
      "learning_rate": 4.898274337809562e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5,
      "logps/chosen": -444.0,
      "logps/rejected": -552.0,
      "loss": 0.4462,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.234375,
      "rewards/margins": 1.28125,
      "rewards/rejected": -3.515625,
      "step": 4890
    },
    {
      "epoch": 0.18239005415867934,
      "grad_norm": 7.455906995616752,
      "learning_rate": 4.897355108729585e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -426.0,
      "logps/rejected": -536.0,
      "loss": 0.4398,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.171875,
      "rewards/margins": 1.21875,
      "rewards/rejected": -3.390625,
      "step": 4900
    },
    {
      "epoch": 0.1827622787590032,
      "grad_norm": 9.751868199604877,
      "learning_rate": 4.896431832108143e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.328125,
      "logps/chosen": -402.0,
      "logps/rejected": -492.0,
      "loss": 0.4641,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.03125,
      "rewards/margins": 1.015625,
      "rewards/rejected": -3.046875,
      "step": 4910
    },
    {
      "epoch": 0.18313450335932702,
      "grad_norm": 7.328176443695507,
      "learning_rate": 4.895504509504038e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.53125,
      "logps/chosen": -424.0,
      "logps/rejected": -516.0,
      "loss": 0.4307,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.015625,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -3.171875,
      "step": 4920
    },
    {
      "epoch": 0.18350672795965087,
      "grad_norm": 7.126514646870822,
      "learning_rate": 4.894573142482902e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.46875,
      "logps/chosen": -426.0,
      "logps/rejected": -548.0,
      "loss": 0.4468,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.1875,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -3.46875,
      "step": 4930
    },
    {
      "epoch": 0.1838789525599747,
      "grad_norm": 7.379678528607842,
      "learning_rate": 4.893637732617196e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.4375,
      "logps/chosen": -404.0,
      "logps/rejected": -512.0,
      "loss": 0.4448,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.921875,
      "rewards/margins": 1.15625,
      "rewards/rejected": -3.078125,
      "step": 4940
    },
    {
      "epoch": 0.1842511771602985,
      "grad_norm": 8.155729804972074,
      "learning_rate": 4.892698281486206e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.640625,
      "logps/chosen": -440.0,
      "logps/rejected": -548.0,
      "loss": 0.4331,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.140625,
      "rewards/margins": 1.359375,
      "rewards/rejected": -3.5,
      "step": 4950
    },
    {
      "epoch": 0.18462340176062236,
      "grad_norm": 7.796376183268067,
      "learning_rate": 4.89175479067604e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -406.0,
      "logps/rejected": -524.0,
      "loss": 0.4502,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.03125,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -3.28125,
      "step": 4960
    },
    {
      "epoch": 0.18499562636094619,
      "grad_norm": 8.602085800667219,
      "learning_rate": 4.890807261779631e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.6875,
      "logps/chosen": -448.0,
      "logps/rejected": -576.0,
      "loss": 0.4582,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.359375,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -3.75,
      "step": 4970
    },
    {
      "epoch": 0.18536785096127004,
      "grad_norm": 10.098464849155265,
      "learning_rate": 4.889855696396722e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5625,
      "logps/chosen": -434.0,
      "logps/rejected": -544.0,
      "loss": 0.4519,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -2.390625,
      "rewards/margins": 1.0625,
      "rewards/rejected": -3.453125,
      "step": 4980
    },
    {
      "epoch": 0.18574007556159386,
      "grad_norm": 8.987987817796718,
      "learning_rate": 4.88890009613388e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.703125,
      "logps/chosen": -402.0,
      "logps/rejected": -504.0,
      "loss": 0.4341,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.03125,
      "rewards/margins": 1.1328125,
      "rewards/rejected": -3.171875,
      "step": 4990
    },
    {
      "epoch": 0.1861123001619177,
      "grad_norm": 9.437912666642232,
      "learning_rate": 4.887940462604475e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.34375,
      "logps/chosen": -456.0,
      "logps/rejected": -576.0,
      "loss": 0.4508,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.375,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -3.703125,
      "step": 5000
    },
    {
      "epoch": 0.18648452476224153,
      "grad_norm": 8.358090475753432,
      "learning_rate": 4.886976797428694e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.546875,
      "logps/chosen": -446.0,
      "logps/rejected": -540.0,
      "loss": 0.4524,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.328125,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -3.421875,
      "step": 5010
    },
    {
      "epoch": 0.18685674936256538,
      "grad_norm": 10.279659986763178,
      "learning_rate": 4.886009102233528e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.4375,
      "logps/chosen": -440.0,
      "logps/rejected": -544.0,
      "loss": 0.4565,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.21875,
      "rewards/margins": 1.3125,
      "rewards/rejected": -3.53125,
      "step": 5020
    },
    {
      "epoch": 0.1872289739628892,
      "grad_norm": 8.967010974221918,
      "learning_rate": 4.88503737865277e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.53125,
      "logps/chosen": -424.0,
      "logps/rejected": -540.0,
      "loss": 0.4369,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.28125,
      "rewards/margins": 1.1796875,
      "rewards/rejected": -3.46875,
      "step": 5030
    },
    {
      "epoch": 0.18760119856321306,
      "grad_norm": 9.275399109294542,
      "learning_rate": 4.884061628327018e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.578125,
      "logps/chosen": -454.0,
      "logps/rejected": -564.0,
      "loss": 0.45,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.484375,
      "rewards/margins": 1.25,
      "rewards/rejected": -3.734375,
      "step": 5040
    },
    {
      "epoch": 0.18797342316353688,
      "grad_norm": 8.510625839435587,
      "learning_rate": 4.883081852903665e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.671875,
      "logps/chosen": -436.0,
      "logps/rejected": -552.0,
      "loss": 0.4134,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.21875,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -3.53125,
      "step": 5050
    },
    {
      "epoch": 0.1883456477638607,
      "grad_norm": 9.974270904159324,
      "learning_rate": 4.882098054036901e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -432.0,
      "logps/rejected": -528.0,
      "loss": 0.4674,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.171875,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -3.40625,
      "step": 5060
    },
    {
      "epoch": 0.18871787236418455,
      "grad_norm": 8.198094210688621,
      "learning_rate": 4.881110233387711e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5,
      "logps/chosen": -420.0,
      "logps/rejected": -536.0,
      "loss": 0.4322,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.09375,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -3.40625,
      "step": 5070
    },
    {
      "epoch": 0.18909009696450838,
      "grad_norm": 9.405602048229078,
      "learning_rate": 4.880118392623869e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -412.0,
      "logps/rejected": -532.0,
      "loss": 0.4635,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.09375,
      "rewards/margins": 1.421875,
      "rewards/rejected": -3.515625,
      "step": 5080
    },
    {
      "epoch": 0.18946232156483223,
      "grad_norm": 7.873473505416543,
      "learning_rate": 4.879122533419933e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.484375,
      "logps/chosen": -424.0,
      "logps/rejected": -524.0,
      "loss": 0.4291,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.1875,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -3.40625,
      "step": 5090
    },
    {
      "epoch": 0.18983454616515605,
      "grad_norm": 9.751527644057527,
      "learning_rate": 4.87812265745725e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.546875,
      "logps/chosen": -426.0,
      "logps/rejected": -520.0,
      "loss": 0.4431,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.21875,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -3.4375,
      "step": 5100
    },
    {
      "epoch": 0.1902067707654799,
      "grad_norm": 10.356505767819806,
      "learning_rate": 4.877118766423945e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.484375,
      "logps/chosen": -448.0,
      "logps/rejected": -576.0,
      "loss": 0.4506,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.40625,
      "rewards/margins": 1.234375,
      "rewards/rejected": -3.65625,
      "step": 5110
    },
    {
      "epoch": 0.19057899536580372,
      "grad_norm": 9.28393405673084,
      "learning_rate": 4.876110862014926e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.59375,
      "logps/chosen": -464.0,
      "logps/rejected": -580.0,
      "loss": 0.4546,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.328125,
      "rewards/rejected": -3.875,
      "step": 5120
    },
    {
      "epoch": 0.19095121996612757,
      "grad_norm": 7.830275704052257,
      "learning_rate": 4.875098945931873e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.609375,
      "logps/chosen": -428.0,
      "logps/rejected": -536.0,
      "loss": 0.439,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.265625,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -3.578125,
      "step": 5130
    },
    {
      "epoch": 0.1913234445664514,
      "grad_norm": 10.170065498047265,
      "learning_rate": 4.874083019883242e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -412.0,
      "logps/rejected": -508.0,
      "loss": 0.467,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.25,
      "rewards/margins": 0.9921875,
      "rewards/rejected": -3.25,
      "step": 5140
    },
    {
      "epoch": 0.19169566916677525,
      "grad_norm": 8.18467474872402,
      "learning_rate": 4.873063085584256e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.53125,
      "logps/chosen": -426.0,
      "logps/rejected": -520.0,
      "loss": 0.456,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.296875,
      "rewards/margins": 0.9921875,
      "rewards/rejected": -3.28125,
      "step": 5150
    },
    {
      "epoch": 0.19206789376709907,
      "grad_norm": 9.934138788027518,
      "learning_rate": 4.872039144756907e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.40625,
      "logps/chosen": -406.0,
      "logps/rejected": -510.0,
      "loss": 0.4388,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.078125,
      "rewards/margins": 1.203125,
      "rewards/rejected": -3.265625,
      "step": 5160
    },
    {
      "epoch": 0.1924401183674229,
      "grad_norm": 8.612639453529853,
      "learning_rate": 4.871011199129953e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.53125,
      "logps/chosen": -426.0,
      "logps/rejected": -532.0,
      "loss": 0.4548,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.3125,
      "rewards/margins": 1.1484375,
      "rewards/rejected": -3.453125,
      "step": 5170
    },
    {
      "epoch": 0.19281234296774674,
      "grad_norm": 10.632191943238523,
      "learning_rate": 4.869979250438911e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.3125,
      "logps/chosen": -404.0,
      "logps/rejected": -536.0,
      "loss": 0.4378,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.171875,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -3.40625,
      "step": 5180
    },
    {
      "epoch": 0.19318456756807056,
      "grad_norm": 8.5331104909626,
      "learning_rate": 4.868943300426057e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -456.0,
      "logps/rejected": -540.0,
      "loss": 0.4603,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -2.546875,
      "rewards/margins": 0.984375,
      "rewards/rejected": -3.53125,
      "step": 5190
    },
    {
      "epoch": 0.19355679216839441,
      "grad_norm": 10.58390870959992,
      "learning_rate": 4.867903350840423e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.546875,
      "logps/chosen": -438.0,
      "logps/rejected": -506.0,
      "loss": 0.4482,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.296875,
      "rewards/margins": 0.984375,
      "rewards/rejected": -3.28125,
      "step": 5200
    },
    {
      "epoch": 0.19392901676871824,
      "grad_norm": 8.565591805459936,
      "learning_rate": 4.866859403437795e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.546875,
      "logps/chosen": -460.0,
      "logps/rejected": -564.0,
      "loss": 0.4186,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.359375,
      "rewards/margins": 1.234375,
      "rewards/rejected": -3.578125,
      "step": 5210
    },
    {
      "epoch": 0.1943012413690421,
      "grad_norm": 8.604089403049617,
      "learning_rate": 4.865811459980705e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.5,
      "logps/chosen": -432.0,
      "logps/rejected": -512.0,
      "loss": 0.4442,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.078125,
      "rewards/margins": 1.109375,
      "rewards/rejected": -3.1875,
      "step": 5220
    },
    {
      "epoch": 0.1946734659693659,
      "grad_norm": 9.197180867595893,
      "learning_rate": 4.864759522238435e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.453125,
      "logps/chosen": -398.0,
      "logps/rejected": -478.0,
      "loss": 0.4639,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.078125,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -3.09375,
      "step": 5230
    },
    {
      "epoch": 0.19504569056968976,
      "grad_norm": 8.060535480991017,
      "learning_rate": 4.86370359198701e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.53125,
      "logps/chosen": -424.0,
      "logps/rejected": -536.0,
      "loss": 0.4519,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.015625,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -3.4375,
      "step": 5240
    },
    {
      "epoch": 0.19541791517001358,
      "grad_norm": 7.763914041684228,
      "learning_rate": 4.862643671009193e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5625,
      "logps/chosen": -420.0,
      "logps/rejected": -512.0,
      "loss": 0.4461,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.3125,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -3.328125,
      "step": 5250
    },
    {
      "epoch": 0.19579013977033743,
      "grad_norm": 10.4419400256207,
      "learning_rate": 4.861579761094491e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.546875,
      "logps/chosen": -412.0,
      "logps/rejected": -516.0,
      "loss": 0.4405,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.15625,
      "rewards/margins": 1.265625,
      "rewards/rejected": -3.421875,
      "step": 5260
    },
    {
      "epoch": 0.19616236437066126,
      "grad_norm": 8.93188743925281,
      "learning_rate": 4.860511864039139e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.453125,
      "logps/chosen": -450.0,
      "logps/rejected": -564.0,
      "loss": 0.4596,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.359375,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -3.625,
      "step": 5270
    },
    {
      "epoch": 0.19653458897098508,
      "grad_norm": 10.16041854105076,
      "learning_rate": 4.859439981646106e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.515625,
      "logps/chosen": -408.0,
      "logps/rejected": -520.0,
      "loss": 0.4276,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 1.5,
      "rewards/rejected": -3.421875,
      "step": 5280
    },
    {
      "epoch": 0.19690681357130893,
      "grad_norm": 9.64662963712854,
      "learning_rate": 4.858364115725091e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.484375,
      "logps/chosen": -416.0,
      "logps/rejected": -516.0,
      "loss": 0.4446,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.09375,
      "rewards/margins": 1.0703125,
      "rewards/rejected": -3.171875,
      "step": 5290
    },
    {
      "epoch": 0.19727903817163275,
      "grad_norm": 8.699547412383628,
      "learning_rate": 4.857284268092516e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.546875,
      "logps/chosen": -408.0,
      "logps/rejected": -516.0,
      "loss": 0.4508,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.234375,
      "rewards/margins": 1.1875,
      "rewards/rejected": -3.421875,
      "step": 5300
    },
    {
      "epoch": 0.1976512627719566,
      "grad_norm": 7.969688625655919,
      "learning_rate": 4.856200440571529e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.4375,
      "logps/chosen": -418.0,
      "logps/rejected": -532.0,
      "loss": 0.4307,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.234375,
      "rewards/margins": 1.40625,
      "rewards/rejected": -3.640625,
      "step": 5310
    },
    {
      "epoch": 0.19802348737228043,
      "grad_norm": 10.326464923146393,
      "learning_rate": 4.855112634991994e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.40625,
      "logps/chosen": -440.0,
      "logps/rejected": -544.0,
      "loss": 0.4358,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.34375,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -3.546875,
      "step": 5320
    },
    {
      "epoch": 0.19839571197260428,
      "grad_norm": 8.463028765476123,
      "learning_rate": 4.854020853190492e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5,
      "logps/chosen": -442.0,
      "logps/rejected": -552.0,
      "loss": 0.4521,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.296875,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -3.578125,
      "step": 5330
    },
    {
      "epoch": 0.1987679365729281,
      "grad_norm": 9.268451015551165,
      "learning_rate": 4.85292509701032e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.46875,
      "logps/chosen": -442.0,
      "logps/rejected": -528.0,
      "loss": 0.439,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.265625,
      "rewards/margins": 1.03125,
      "rewards/rejected": -3.296875,
      "step": 5340
    },
    {
      "epoch": 0.19914016117325195,
      "grad_norm": 8.663753577826954,
      "learning_rate": 4.85182536830148e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.40625,
      "logps/chosen": -416.0,
      "logps/rejected": -552.0,
      "loss": 0.4403,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.21875,
      "rewards/margins": 1.359375,
      "rewards/rejected": -3.578125,
      "step": 5350
    },
    {
      "epoch": 0.19951238577357577,
      "grad_norm": 10.554587753500426,
      "learning_rate": 4.850721668920684e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.4375,
      "logps/chosen": -438.0,
      "logps/rejected": -536.0,
      "loss": 0.4398,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.3125,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -3.53125,
      "step": 5360
    },
    {
      "epoch": 0.19988461037389962,
      "grad_norm": 8.564519897201302,
      "learning_rate": 4.84961400073135e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.40625,
      "logps/chosen": -416.0,
      "logps/rejected": -516.0,
      "loss": 0.4412,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.15625,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -3.359375,
      "step": 5370
    },
    {
      "epoch": 0.20025683497422345,
      "grad_norm": 9.704992110304065,
      "learning_rate": 4.848502365603593e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.515625,
      "logps/chosen": -418.0,
      "logps/rejected": -510.0,
      "loss": 0.45,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0625,
      "rewards/margins": 1.0859375,
      "rewards/rejected": -3.140625,
      "step": 5380
    },
    {
      "epoch": 0.20062905957454727,
      "grad_norm": 9.16640399324903,
      "learning_rate": 4.847386765414227e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.4375,
      "logps/chosen": -460.0,
      "logps/rejected": -536.0,
      "loss": 0.4275,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.328125,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -3.5625,
      "step": 5390
    },
    {
      "epoch": 0.20100128417487112,
      "grad_norm": 9.965982905057228,
      "learning_rate": 4.84626720204676e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.40625,
      "logps/chosen": -438.0,
      "logps/rejected": -532.0,
      "loss": 0.4242,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.21875,
      "rewards/margins": 1.078125,
      "rewards/rejected": -3.296875,
      "step": 5400
    },
    {
      "epoch": 0.20137350877519494,
      "grad_norm": 10.060602673968209,
      "learning_rate": 4.845143677391392e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.609375,
      "logps/chosen": -432.0,
      "logps/rejected": -548.0,
      "loss": 0.4126,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.125,
      "rewards/margins": 1.28125,
      "rewards/rejected": -3.40625,
      "step": 5410
    },
    {
      "epoch": 0.2017457333755188,
      "grad_norm": 11.39696289059235,
      "learning_rate": 4.84401619334501e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -450.0,
      "logps/rejected": -548.0,
      "loss": 0.4606,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.421875,
      "rewards/margins": 1.0625,
      "rewards/rejected": -3.484375,
      "step": 5420
    },
    {
      "epoch": 0.20211795797584262,
      "grad_norm": 12.777668477119674,
      "learning_rate": 4.842884751811185e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.703125,
      "logps/chosen": -428.0,
      "logps/rejected": -532.0,
      "loss": 0.4404,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.265625,
      "rewards/margins": 1.21875,
      "rewards/rejected": -3.484375,
      "step": 5430
    },
    {
      "epoch": 0.20249018257616647,
      "grad_norm": 9.580332137204472,
      "learning_rate": 4.841749354700172e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.625,
      "logps/chosen": -442.0,
      "logps/rejected": -528.0,
      "loss": 0.4449,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.125,
      "rewards/margins": 1.203125,
      "rewards/rejected": -3.328125,
      "step": 5440
    },
    {
      "epoch": 0.2028624071764903,
      "grad_norm": 10.823840287121717,
      "learning_rate": 4.840610003928902e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.640625,
      "logps/chosen": -460.0,
      "logps/rejected": -560.0,
      "loss": 0.4387,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.328125,
      "rewards/margins": 1.203125,
      "rewards/rejected": -3.53125,
      "step": 5450
    },
    {
      "epoch": 0.20323463177681414,
      "grad_norm": 11.534566127715584,
      "learning_rate": 4.839466701420985e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.515625,
      "logps/chosen": -480.0,
      "logps/rejected": -564.0,
      "loss": 0.4793,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.5,
      "rewards/margins": 1.140625,
      "rewards/rejected": -3.640625,
      "step": 5460
    },
    {
      "epoch": 0.20360685637713796,
      "grad_norm": 8.267417611470218,
      "learning_rate": 4.838319449106697e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.4375,
      "logps/chosen": -442.0,
      "logps/rejected": -532.0,
      "loss": 0.4416,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.375,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -3.40625,
      "step": 5470
    },
    {
      "epoch": 0.2039790809774618,
      "grad_norm": 8.87793453313825,
      "learning_rate": 4.837168248922986e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.484375,
      "logps/chosen": -470.0,
      "logps/rejected": -564.0,
      "loss": 0.4265,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.4375,
      "rewards/margins": 1.09375,
      "rewards/rejected": -3.53125,
      "step": 5480
    },
    {
      "epoch": 0.20435130557778564,
      "grad_norm": 10.216483020922649,
      "learning_rate": 4.836013102813467e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.53125,
      "logps/chosen": -464.0,
      "logps/rejected": -572.0,
      "loss": 0.4214,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.4375,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -3.8125,
      "step": 5490
    },
    {
      "epoch": 0.20472353017810946,
      "grad_norm": 9.513592713390551,
      "learning_rate": 4.834854012728412e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.546875,
      "logps/chosen": -466.0,
      "logps/rejected": -572.0,
      "loss": 0.4581,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.515625,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -3.75,
      "step": 5500
    },
    {
      "epoch": 0.2050957547784333,
      "grad_norm": 9.158131064300202,
      "learning_rate": 4.833690980624758e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5,
      "logps/chosen": -462.0,
      "logps/rejected": -536.0,
      "loss": 0.4677,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.5,
      "rewards/margins": 0.94921875,
      "rewards/rejected": -3.453125,
      "step": 5510
    },
    {
      "epoch": 0.20546797937875713,
      "grad_norm": 8.716203046226425,
      "learning_rate": 4.832524008466091e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.546875,
      "logps/chosen": -452.0,
      "logps/rejected": -540.0,
      "loss": 0.4352,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.359375,
      "rewards/margins": 1.09375,
      "rewards/rejected": -3.453125,
      "step": 5520
    },
    {
      "epoch": 0.20584020397908098,
      "grad_norm": 8.644685727763132,
      "learning_rate": 4.831353098222655e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.515625,
      "logps/chosen": -456.0,
      "logps/rejected": -584.0,
      "loss": 0.4531,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.453125,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -3.890625,
      "step": 5530
    },
    {
      "epoch": 0.2062124285794048,
      "grad_norm": 7.961561740293185,
      "learning_rate": 4.83017825187134e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.46875,
      "logps/chosen": -444.0,
      "logps/rejected": -564.0,
      "loss": 0.4434,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.265625,
      "rewards/margins": 1.265625,
      "rewards/rejected": -3.53125,
      "step": 5540
    },
    {
      "epoch": 0.20658465317972866,
      "grad_norm": 9.057026504224842,
      "learning_rate": 4.828999471395679e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -446.0,
      "logps/rejected": -544.0,
      "loss": 0.4357,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.296875,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -3.515625,
      "step": 5550
    },
    {
      "epoch": 0.20695687778005248,
      "grad_norm": 9.132320603586045,
      "learning_rate": 4.827816758785853e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.453125,
      "logps/chosen": -448.0,
      "logps/rejected": -536.0,
      "loss": 0.4369,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.359375,
      "rewards/margins": 1.046875,
      "rewards/rejected": -3.40625,
      "step": 5560
    },
    {
      "epoch": 0.20732910238037633,
      "grad_norm": 8.454814771385099,
      "learning_rate": 4.826630116038677e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.765625,
      "logps/chosen": -452.0,
      "logps/rejected": -548.0,
      "loss": 0.4331,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.40625,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -3.65625,
      "step": 5570
    },
    {
      "epoch": 0.20770132698070015,
      "grad_norm": 8.864499531729393,
      "learning_rate": 4.825439545157603e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.6875,
      "logps/chosen": -458.0,
      "logps/rejected": -568.0,
      "loss": 0.4622,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.375,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -3.5625,
      "step": 5580
    },
    {
      "epoch": 0.208073551581024,
      "grad_norm": 9.151180087478288,
      "learning_rate": 4.824245048152715e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.609375,
      "logps/chosen": -438.0,
      "logps/rejected": -532.0,
      "loss": 0.4561,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.25,
      "rewards/margins": 1.0546875,
      "rewards/rejected": -3.296875,
      "step": 5590
    },
    {
      "epoch": 0.20844577618134782,
      "grad_norm": 9.123245524732672,
      "learning_rate": 4.823046627040725e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.46875,
      "logps/chosen": -436.0,
      "logps/rejected": -524.0,
      "loss": 0.4626,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.328125,
      "rewards/margins": 1.109375,
      "rewards/rejected": -3.4375,
      "step": 5600
    },
    {
      "epoch": 0.20881800078167165,
      "grad_norm": 8.591594605468881,
      "learning_rate": 4.821844283844972e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -438.0,
      "logps/rejected": -548.0,
      "loss": 0.443,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.1875,
      "rewards/margins": 1.125,
      "rewards/rejected": -3.3125,
      "step": 5610
    },
    {
      "epoch": 0.2091902253819955,
      "grad_norm": 9.767603577203115,
      "learning_rate": 4.820638020595414e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.484375,
      "logps/chosen": -432.0,
      "logps/rejected": -520.0,
      "loss": 0.4529,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.1875,
      "rewards/margins": 1.0,
      "rewards/rejected": -3.1875,
      "step": 5620
    },
    {
      "epoch": 0.20956244998231932,
      "grad_norm": 8.432020406350393,
      "learning_rate": 4.819427839328632e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.578125,
      "logps/chosen": -424.0,
      "logps/rejected": -512.0,
      "loss": 0.4488,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.234375,
      "rewards/margins": 1.171875,
      "rewards/rejected": -3.40625,
      "step": 5630
    },
    {
      "epoch": 0.20993467458264317,
      "grad_norm": 8.60340776484446,
      "learning_rate": 4.818213742087815e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.53125,
      "logps/chosen": -452.0,
      "logps/rejected": -572.0,
      "loss": 0.4365,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.40625,
      "rewards/margins": 1.40625,
      "rewards/rejected": -3.8125,
      "step": 5640
    },
    {
      "epoch": 0.210306899182967,
      "grad_norm": 9.328773728109747,
      "learning_rate": 4.81699573092277e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.703125,
      "logps/chosen": -450.0,
      "logps/rejected": -560.0,
      "loss": 0.4247,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.46875,
      "rewards/margins": 1.328125,
      "rewards/rejected": -3.796875,
      "step": 5650
    },
    {
      "epoch": 0.21067912378329084,
      "grad_norm": 9.151670189597018,
      "learning_rate": 4.815773807889907e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.71875,
      "logps/chosen": -420.0,
      "logps/rejected": -536.0,
      "loss": 0.4353,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.375,
      "rewards/margins": 1.1796875,
      "rewards/rejected": -3.5625,
      "step": 5660
    },
    {
      "epoch": 0.21105134838361467,
      "grad_norm": 7.54142996999106,
      "learning_rate": 4.814547975052244e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.734375,
      "logps/chosen": -452.0,
      "logps/rejected": -544.0,
      "loss": 0.4691,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.4375,
      "rewards/margins": 0.9140625,
      "rewards/rejected": -3.34375,
      "step": 5670
    },
    {
      "epoch": 0.21142357298393852,
      "grad_norm": 9.149933624289034,
      "learning_rate": 4.813318234479401e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.703125,
      "logps/chosen": -434.0,
      "logps/rejected": -544.0,
      "loss": 0.4478,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.265625,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -3.484375,
      "step": 5680
    },
    {
      "epoch": 0.21179579758426234,
      "grad_norm": 9.237332516553877,
      "learning_rate": 4.812084588247592e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.734375,
      "logps/chosen": -436.0,
      "logps/rejected": -532.0,
      "loss": 0.4391,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.296875,
      "rewards/margins": 1.0546875,
      "rewards/rejected": -3.359375,
      "step": 5690
    },
    {
      "epoch": 0.2121680221845862,
      "grad_norm": 8.140285219303877,
      "learning_rate": 4.810847038439626e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.40625,
      "logps/chosen": -400.0,
      "logps/rejected": -504.0,
      "loss": 0.4636,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.09375,
      "rewards/margins": 1.046875,
      "rewards/rejected": -3.140625,
      "step": 5700
    },
    {
      "epoch": 0.21254024678491001,
      "grad_norm": 8.637677868120612,
      "learning_rate": 4.809605587144904e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -416.0,
      "logps/rejected": -528.0,
      "loss": 0.4511,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.203125,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -3.390625,
      "step": 5710
    },
    {
      "epoch": 0.21291247138523384,
      "grad_norm": 8.384357243097014,
      "learning_rate": 4.808360236459412e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.75,
      "logps/chosen": -428.0,
      "logps/rejected": -548.0,
      "loss": 0.4346,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.1875,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -3.59375,
      "step": 5720
    },
    {
      "epoch": 0.2132846959855577,
      "grad_norm": 9.15115139964247,
      "learning_rate": 4.807110988485721e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.78125,
      "logps/chosen": -448.0,
      "logps/rejected": -588.0,
      "loss": 0.4425,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.390625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.0,
      "step": 5730
    },
    {
      "epoch": 0.2136569205858815,
      "grad_norm": 9.096128313768538,
      "learning_rate": 4.805857845332983e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.875,
      "logps/chosen": -480.0,
      "logps/rejected": -580.0,
      "loss": 0.4165,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.1328125,
      "rewards/rejected": -3.765625,
      "step": 5740
    },
    {
      "epoch": 0.21402914518620536,
      "grad_norm": 7.487069095643862,
      "learning_rate": 4.804600809116925e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.734375,
      "logps/chosen": -468.0,
      "logps/rejected": -580.0,
      "loss": 0.4377,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.5,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -3.8125,
      "step": 5750
    },
    {
      "epoch": 0.21440136978652918,
      "grad_norm": 11.43859875043155,
      "learning_rate": 4.803339881959845e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -432.0,
      "logps/rejected": -516.0,
      "loss": 0.4488,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.171875,
      "rewards/margins": 1.125,
      "rewards/rejected": -3.296875,
      "step": 5760
    },
    {
      "epoch": 0.21477359438685303,
      "grad_norm": 8.789466200907084,
      "learning_rate": 4.802075065990613e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.75,
      "logps/chosen": -456.0,
      "logps/rejected": -528.0,
      "loss": 0.4378,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.390625,
      "rewards/margins": 0.96875,
      "rewards/rejected": -3.359375,
      "step": 5770
    },
    {
      "epoch": 0.21514581898717686,
      "grad_norm": 9.350829696718824,
      "learning_rate": 4.800806363344663e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.6875,
      "logps/chosen": -428.0,
      "logps/rejected": -536.0,
      "loss": 0.4559,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.3125,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -3.34375,
      "step": 5780
    },
    {
      "epoch": 0.2155180435875007,
      "grad_norm": 8.882634191355207,
      "learning_rate": 4.799533776163993e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.65625,
      "logps/chosen": -436.0,
      "logps/rejected": -552.0,
      "loss": 0.4332,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.421875,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -3.625,
      "step": 5790
    },
    {
      "epoch": 0.21589026818782453,
      "grad_norm": 8.440996843820145,
      "learning_rate": 4.798257306597156e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -472.0,
      "logps/rejected": -580.0,
      "loss": 0.4188,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.578125,
      "rewards/margins": 1.09375,
      "rewards/rejected": -3.671875,
      "step": 5800
    },
    {
      "epoch": 0.21626249278814838,
      "grad_norm": 11.088629532522978,
      "learning_rate": 4.796976956799264e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.828125,
      "logps/chosen": -456.0,
      "logps/rejected": -556.0,
      "loss": 0.4503,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.5,
      "rewards/margins": 1.109375,
      "rewards/rejected": -3.609375,
      "step": 5810
    },
    {
      "epoch": 0.2166347173884722,
      "grad_norm": 9.653626745650296,
      "learning_rate": 4.795692728931977e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.828125,
      "logps/chosen": -442.0,
      "logps/rejected": -560.0,
      "loss": 0.4455,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.40625,
      "rewards/margins": 1.3125,
      "rewards/rejected": -3.71875,
      "step": 5820
    },
    {
      "epoch": 0.21700694198879603,
      "grad_norm": 8.599182415105444,
      "learning_rate": 4.794404625163503e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.859375,
      "logps/chosen": -452.0,
      "logps/rejected": -556.0,
      "loss": 0.4216,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.390625,
      "rewards/margins": 1.1875,
      "rewards/rejected": -3.578125,
      "step": 5830
    },
    {
      "epoch": 0.21737916658911988,
      "grad_norm": 8.196900954052591,
      "learning_rate": 4.793112647668594e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.734375,
      "logps/chosen": -418.0,
      "logps/rejected": -548.0,
      "loss": 0.4355,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.203125,
      "rewards/margins": 1.375,
      "rewards/rejected": -3.59375,
      "step": 5840
    },
    {
      "epoch": 0.2177513911894437,
      "grad_norm": 10.228206224851787,
      "learning_rate": 4.791816798628544e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.671875,
      "logps/chosen": -414.0,
      "logps/rejected": -536.0,
      "loss": 0.4486,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.109375,
      "rewards/margins": 1.375,
      "rewards/rejected": -3.484375,
      "step": 5850
    },
    {
      "epoch": 0.21812361578976755,
      "grad_norm": 9.314484775006418,
      "learning_rate": 4.790517080231179e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.671875,
      "logps/chosen": -462.0,
      "logps/rejected": -552.0,
      "loss": 0.4669,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.359375,
      "rewards/margins": 1.21875,
      "rewards/rejected": -3.59375,
      "step": 5860
    },
    {
      "epoch": 0.21849584039009137,
      "grad_norm": 11.249457547521407,
      "learning_rate": 4.789213494670864e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.625,
      "logps/chosen": -460.0,
      "logps/rejected": -528.0,
      "loss": 0.4512,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.3125,
      "rewards/margins": 0.97265625,
      "rewards/rejected": -3.28125,
      "step": 5870
    },
    {
      "epoch": 0.21886806499041522,
      "grad_norm": 7.597797755929162,
      "learning_rate": 4.787906044148489e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -422.0,
      "logps/rejected": -544.0,
      "loss": 0.4383,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 1.46875,
      "rewards/rejected": -3.421875,
      "step": 5880
    },
    {
      "epoch": 0.21924028959073905,
      "grad_norm": 9.043373461043474,
      "learning_rate": 4.786594730871467e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.671875,
      "logps/chosen": -472.0,
      "logps/rejected": -564.0,
      "loss": 0.4386,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.46875,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -3.671875,
      "step": 5890
    },
    {
      "epoch": 0.2196125141910629,
      "grad_norm": 9.874322022877863,
      "learning_rate": 4.785279557053739e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.734375,
      "logps/chosen": -432.0,
      "logps/rejected": -548.0,
      "loss": 0.4594,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.34375,
      "rewards/margins": 1.140625,
      "rewards/rejected": -3.484375,
      "step": 5900
    },
    {
      "epoch": 0.21998473879138672,
      "grad_norm": 10.119446228131016,
      "learning_rate": 4.78396052491576e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -428.0,
      "logps/rejected": -556.0,
      "loss": 0.4345,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.3125,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -3.6875,
      "step": 5910
    },
    {
      "epoch": 0.22035696339171057,
      "grad_norm": 8.308482203456336,
      "learning_rate": 4.782637636684499e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.546875,
      "logps/chosen": -444.0,
      "logps/rejected": -548.0,
      "loss": 0.4314,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.40625,
      "rewards/margins": 1.21875,
      "rewards/rejected": -3.640625,
      "step": 5920
    },
    {
      "epoch": 0.2207291879920344,
      "grad_norm": 9.791667031945384,
      "learning_rate": 4.781310894593437e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.734375,
      "logps/chosen": -468.0,
      "logps/rejected": -560.0,
      "loss": 0.4465,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.59375,
      "rewards/margins": 0.94921875,
      "rewards/rejected": -3.546875,
      "step": 5930
    },
    {
      "epoch": 0.22110141259235822,
      "grad_norm": 10.013040162176994,
      "learning_rate": 4.779980300882559e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5625,
      "logps/chosen": -440.0,
      "logps/rejected": -588.0,
      "loss": 0.4252,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.453125,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -3.96875,
      "step": 5940
    },
    {
      "epoch": 0.22147363719268207,
      "grad_norm": 10.66929806092116,
      "learning_rate": 4.778645857798357e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.671875,
      "logps/chosen": -418.0,
      "logps/rejected": -516.0,
      "loss": 0.4326,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.46875,
      "rewards/margins": 0.91796875,
      "rewards/rejected": -3.390625,
      "step": 5950
    },
    {
      "epoch": 0.2218458617930059,
      "grad_norm": 8.465241517435018,
      "learning_rate": 4.777307567593818e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -448.0,
      "logps/rejected": -564.0,
      "loss": 0.4475,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.40625,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -3.71875,
      "step": 5960
    },
    {
      "epoch": 0.22221808639332974,
      "grad_norm": 8.075516582095707,
      "learning_rate": 4.775965432528426e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.6875,
      "logps/chosen": -468.0,
      "logps/rejected": -588.0,
      "loss": 0.4384,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.609375,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.0,
      "step": 5970
    },
    {
      "epoch": 0.22259031099365356,
      "grad_norm": 9.349920720450532,
      "learning_rate": 4.774619454868156e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.71875,
      "logps/chosen": -434.0,
      "logps/rejected": -548.0,
      "loss": 0.4388,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.421875,
      "rewards/margins": 1.3125,
      "rewards/rejected": -3.734375,
      "step": 5980
    },
    {
      "epoch": 0.2229625355939774,
      "grad_norm": 7.3713400459853045,
      "learning_rate": 4.773269636885471e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -482.0,
      "logps/rejected": -592.0,
      "loss": 0.4306,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.609375,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -3.828125,
      "step": 5990
    },
    {
      "epoch": 0.22333476019430124,
      "grad_norm": 8.672445961950231,
      "learning_rate": 4.771915980859318e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.796875,
      "logps/chosen": -486.0,
      "logps/rejected": -592.0,
      "loss": 0.4374,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.21875,
      "rewards/rejected": -3.859375,
      "step": 6000
    },
    {
      "epoch": 0.22370698479462509,
      "grad_norm": 9.620100393449702,
      "learning_rate": 4.770558489075124e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.828125,
      "logps/chosen": -476.0,
      "logps/rejected": -584.0,
      "loss": 0.4337,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.5,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -3.875,
      "step": 6010
    },
    {
      "epoch": 0.2240792093949489,
      "grad_norm": 10.283115688269113,
      "learning_rate": 4.76919716382479e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.765625,
      "logps/chosen": -480.0,
      "logps/rejected": -564.0,
      "loss": 0.4468,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.578125,
      "rewards/margins": 1.25,
      "rewards/rejected": -3.828125,
      "step": 6020
    },
    {
      "epoch": 0.22445143399527276,
      "grad_norm": 7.815458749401708,
      "learning_rate": 4.7678320074066925e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.640625,
      "logps/chosen": -462.0,
      "logps/rejected": -568.0,
      "loss": 0.4277,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.453125,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -3.796875,
      "step": 6030
    },
    {
      "epoch": 0.22482365859559658,
      "grad_norm": 9.887225268019792,
      "learning_rate": 4.766463022125673e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.890625,
      "logps/chosen": -442.0,
      "logps/rejected": -536.0,
      "loss": 0.4402,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.25,
      "rewards/margins": 1.34375,
      "rewards/rejected": -3.609375,
      "step": 6040
    },
    {
      "epoch": 0.22519588319592043,
      "grad_norm": 13.052975785438106,
      "learning_rate": 4.765090210293039e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.84375,
      "logps/chosen": -444.0,
      "logps/rejected": -548.0,
      "loss": 0.4458,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.421875,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -3.640625,
      "step": 6050
    },
    {
      "epoch": 0.22556810779624425,
      "grad_norm": 8.420912157923407,
      "learning_rate": 4.76371357422656e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.8125,
      "logps/chosen": -440.0,
      "logps/rejected": -560.0,
      "loss": 0.4239,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.375,
      "rewards/margins": 1.234375,
      "rewards/rejected": -3.59375,
      "step": 6060
    },
    {
      "epoch": 0.22594033239656808,
      "grad_norm": 8.70164368497188,
      "learning_rate": 4.7623331162504585e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.78125,
      "logps/chosen": -416.0,
      "logps/rejected": -544.0,
      "loss": 0.4394,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.265625,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -3.59375,
      "step": 6070
    },
    {
      "epoch": 0.22631255699689193,
      "grad_norm": 7.895116952701407,
      "learning_rate": 4.7609488386954137e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.734375,
      "logps/chosen": -424.0,
      "logps/rejected": -548.0,
      "loss": 0.4439,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.234375,
      "rewards/margins": 1.21875,
      "rewards/rejected": -3.453125,
      "step": 6080
    },
    {
      "epoch": 0.22668478159721575,
      "grad_norm": 8.149254648700698,
      "learning_rate": 4.759560743898551e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.828125,
      "logps/chosen": -492.0,
      "logps/rejected": -568.0,
      "loss": 0.4343,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.484375,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -3.6875,
      "step": 6090
    },
    {
      "epoch": 0.2270570061975396,
      "grad_norm": 8.143848371435306,
      "learning_rate": 4.758168834203439e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.796875,
      "logps/chosen": -450.0,
      "logps/rejected": -544.0,
      "loss": 0.4274,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.40625,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -3.609375,
      "step": 6100
    },
    {
      "epoch": 0.22742923079786342,
      "grad_norm": 10.39752324588755,
      "learning_rate": 4.7567731119600916e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.625,
      "logps/chosen": -424.0,
      "logps/rejected": -572.0,
      "loss": 0.4371,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.25,
      "rewards/margins": 1.484375,
      "rewards/rejected": -3.734375,
      "step": 6110
    },
    {
      "epoch": 0.22780145539818727,
      "grad_norm": 9.814166553285158,
      "learning_rate": 4.755373579524957e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.71875,
      "logps/chosen": -454.0,
      "logps/rejected": -560.0,
      "loss": 0.4572,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.5625,
      "rewards/margins": 1.109375,
      "rewards/rejected": -3.671875,
      "step": 6120
    },
    {
      "epoch": 0.2281736799985111,
      "grad_norm": 8.60413971772251,
      "learning_rate": 4.753970239260916e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.703125,
      "logps/chosen": -466.0,
      "logps/rejected": -568.0,
      "loss": 0.4595,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.59375,
      "rewards/margins": 1.1796875,
      "rewards/rejected": -3.78125,
      "step": 6130
    },
    {
      "epoch": 0.22854590459883495,
      "grad_norm": 8.39007550718725,
      "learning_rate": 4.752563093537279e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.78125,
      "logps/chosen": -430.0,
      "logps/rejected": -548.0,
      "loss": 0.4368,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.28125,
      "rewards/margins": 1.328125,
      "rewards/rejected": -3.609375,
      "step": 6140
    },
    {
      "epoch": 0.22891812919915877,
      "grad_norm": 9.023921620472663,
      "learning_rate": 4.751152144729781e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.796875,
      "logps/chosen": -452.0,
      "logps/rejected": -540.0,
      "loss": 0.44,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.40625,
      "rewards/margins": 1.1328125,
      "rewards/rejected": -3.546875,
      "step": 6150
    },
    {
      "epoch": 0.22929035379948262,
      "grad_norm": 9.18127066259618,
      "learning_rate": 4.749737395220578e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.859375,
      "logps/chosen": -462.0,
      "logps/rejected": -584.0,
      "loss": 0.4181,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.40625,
      "rewards/margins": 1.390625,
      "rewards/rejected": -3.796875,
      "step": 6160
    },
    {
      "epoch": 0.22966257839980644,
      "grad_norm": 9.569705753208357,
      "learning_rate": 4.748318847398242e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.671875,
      "logps/chosen": -446.0,
      "logps/rejected": -564.0,
      "loss": 0.429,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.3125,
      "rewards/margins": 1.375,
      "rewards/rejected": -3.6875,
      "step": 6170
    },
    {
      "epoch": 0.23003480300013027,
      "grad_norm": 8.540821446351094,
      "learning_rate": 4.746896503657759e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -418.0,
      "logps/rejected": -548.0,
      "loss": 0.4331,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.1875,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -3.625,
      "step": 6180
    },
    {
      "epoch": 0.23040702760045412,
      "grad_norm": 8.927667583200886,
      "learning_rate": 4.745470366400525e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.890625,
      "logps/chosen": -476.0,
      "logps/rejected": -596.0,
      "loss": 0.4236,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.421875,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -3.9375,
      "step": 6190
    },
    {
      "epoch": 0.23077925220077794,
      "grad_norm": 9.85893361959906,
      "learning_rate": 4.744040438034338e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.921875,
      "logps/chosen": -474.0,
      "logps/rejected": -564.0,
      "loss": 0.4489,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.59375,
      "rewards/margins": 1.171875,
      "rewards/rejected": -3.78125,
      "step": 6200
    },
    {
      "epoch": 0.2311514768011018,
      "grad_norm": 9.157719626679256,
      "learning_rate": 4.7426067209733977e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.734375,
      "logps/chosen": -440.0,
      "logps/rejected": -532.0,
      "loss": 0.4422,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.15625,
      "rewards/margins": 1.125,
      "rewards/rejected": -3.28125,
      "step": 6210
    },
    {
      "epoch": 0.2315237014014256,
      "grad_norm": 9.016525623506558,
      "learning_rate": 4.7411692176383013e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.640625,
      "logps/chosen": -444.0,
      "logps/rejected": -548.0,
      "loss": 0.4254,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.296875,
      "rewards/margins": 1.375,
      "rewards/rejected": -3.671875,
      "step": 6220
    },
    {
      "epoch": 0.23189592600174946,
      "grad_norm": 9.050440597659447,
      "learning_rate": 4.739727930456037e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.921875,
      "logps/chosen": -468.0,
      "logps/rejected": -576.0,
      "loss": 0.4364,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.484375,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -3.78125,
      "step": 6230
    },
    {
      "epoch": 0.2322681506020733,
      "grad_norm": 9.14355538026599,
      "learning_rate": 4.738282861859982e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.75,
      "logps/chosen": -448.0,
      "logps/rejected": -592.0,
      "loss": 0.4497,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.4375,
      "rewards/margins": 1.484375,
      "rewards/rejected": -3.921875,
      "step": 6240
    },
    {
      "epoch": 0.23264037520239714,
      "grad_norm": 8.779054223069888,
      "learning_rate": 4.7368340142898983e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.734375,
      "logps/chosen": -420.0,
      "logps/rejected": -506.0,
      "loss": 0.4454,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.28125,
      "rewards/margins": 1.0078125,
      "rewards/rejected": -3.296875,
      "step": 6250
    },
    {
      "epoch": 0.23301259980272096,
      "grad_norm": 9.545364256357699,
      "learning_rate": 4.7353813901919283e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -2.984375,
      "logps/chosen": -430.0,
      "logps/rejected": -540.0,
      "loss": 0.4327,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.359375,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -3.59375,
      "step": 6260
    },
    {
      "epoch": 0.2333848244030448,
      "grad_norm": 8.306653143526962,
      "learning_rate": 4.7339249920185885e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.796875,
      "logps/chosen": -458.0,
      "logps/rejected": -588.0,
      "loss": 0.4384,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.4375,
      "rewards/margins": 1.203125,
      "rewards/rejected": -3.640625,
      "step": 6270
    },
    {
      "epoch": 0.23375704900336863,
      "grad_norm": 9.205856158749267,
      "learning_rate": 4.73246482222877e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.859375,
      "logps/chosen": -442.0,
      "logps/rejected": -568.0,
      "loss": 0.4115,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.453125,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -3.796875,
      "step": 6280
    },
    {
      "epoch": 0.23412927360369246,
      "grad_norm": 10.743900858522482,
      "learning_rate": 4.73100088328773e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.828125,
      "logps/chosen": -498.0,
      "logps/rejected": -608.0,
      "loss": 0.4417,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -4.0,
      "step": 6290
    },
    {
      "epoch": 0.2345014982040163,
      "grad_norm": 7.6688129491876165,
      "learning_rate": 4.72953317766709e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.78125,
      "logps/chosen": -470.0,
      "logps/rejected": -600.0,
      "loss": 0.4479,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.359375,
      "rewards/rejected": -3.90625,
      "step": 6300
    },
    {
      "epoch": 0.23487372280434013,
      "grad_norm": 9.367994204115433,
      "learning_rate": 4.7280617078448294e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.875,
      "logps/chosen": -460.0,
      "logps/rejected": -572.0,
      "loss": 0.4426,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.15625,
      "rewards/rejected": -3.703125,
      "step": 6310
    },
    {
      "epoch": 0.23524594740466398,
      "grad_norm": 8.345137021127687,
      "learning_rate": 4.726586476305285e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.78125,
      "logps/chosen": -444.0,
      "logps/rejected": -548.0,
      "loss": 0.4509,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.359375,
      "rewards/margins": 1.34375,
      "rewards/rejected": -3.703125,
      "step": 6320
    },
    {
      "epoch": 0.2356181720049878,
      "grad_norm": 8.564442880802357,
      "learning_rate": 4.725107485539143e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -488.0,
      "logps/rejected": -596.0,
      "loss": 0.441,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -4.03125,
      "step": 6330
    },
    {
      "epoch": 0.23599039660531165,
      "grad_norm": 7.828896506352744,
      "learning_rate": 4.723624738043438e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.796875,
      "logps/chosen": -478.0,
      "logps/rejected": -576.0,
      "loss": 0.4353,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.125,
      "rewards/rejected": -3.875,
      "step": 6340
    },
    {
      "epoch": 0.23636262120563548,
      "grad_norm": 9.925235581117606,
      "learning_rate": 4.7221382363215447e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.875,
      "logps/chosen": -470.0,
      "logps/rejected": -588.0,
      "loss": 0.4487,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -4.0625,
      "step": 6350
    },
    {
      "epoch": 0.23673484580595933,
      "grad_norm": 9.189043899736243,
      "learning_rate": 4.72064798288318e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.890625,
      "logps/chosen": -476.0,
      "logps/rejected": -556.0,
      "loss": 0.431,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.65625,
      "rewards/margins": 1.03125,
      "rewards/rejected": -3.6875,
      "step": 6360
    },
    {
      "epoch": 0.23710707040628315,
      "grad_norm": 9.776012022406416,
      "learning_rate": 4.7191539802443906e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.59375,
      "logps/chosen": -448.0,
      "logps/rejected": -576.0,
      "loss": 0.4213,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.4375,
      "rewards/margins": 1.359375,
      "rewards/rejected": -3.796875,
      "step": 6370
    },
    {
      "epoch": 0.237479295006607,
      "grad_norm": 9.438069865154143,
      "learning_rate": 4.717656230927557e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.734375,
      "logps/chosen": -464.0,
      "logps/rejected": -572.0,
      "loss": 0.4467,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.5625,
      "rewards/margins": 1.296875,
      "rewards/rejected": -3.859375,
      "step": 6380
    },
    {
      "epoch": 0.23785151960693082,
      "grad_norm": 8.664909530223989,
      "learning_rate": 4.7161547374613817e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.609375,
      "logps/chosen": -464.0,
      "logps/rejected": -552.0,
      "loss": 0.4325,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.46875,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -3.65625,
      "step": 6390
    },
    {
      "epoch": 0.23822374420725465,
      "grad_norm": 9.451400279230453,
      "learning_rate": 4.714649502380893e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.671875,
      "logps/chosen": -478.0,
      "logps/rejected": -592.0,
      "loss": 0.4442,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.65625,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -3.90625,
      "step": 6400
    },
    {
      "epoch": 0.2385959688075785,
      "grad_norm": 12.064228886202951,
      "learning_rate": 4.7131405282274315e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.859375,
      "logps/chosen": -496.0,
      "logps/rejected": -560.0,
      "loss": 0.4515,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.84375,
      "rewards/margins": 0.94921875,
      "rewards/rejected": -3.796875,
      "step": 6410
    },
    {
      "epoch": 0.23896819340790232,
      "grad_norm": 7.357332380905109,
      "learning_rate": 4.7116278175486546e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -426.0,
      "logps/rejected": -540.0,
      "loss": 0.4384,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.40625,
      "rewards/margins": 1.234375,
      "rewards/rejected": -3.640625,
      "step": 6420
    },
    {
      "epoch": 0.23934041800822617,
      "grad_norm": 7.70353830833083,
      "learning_rate": 4.7101113728985253e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.890625,
      "logps/chosen": -456.0,
      "logps/rejected": -576.0,
      "loss": 0.4351,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.390625,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -3.703125,
      "step": 6430
    },
    {
      "epoch": 0.23971264260855,
      "grad_norm": 9.162382669941785,
      "learning_rate": 4.7085911968373135e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.765625,
      "logps/chosen": -442.0,
      "logps/rejected": -544.0,
      "loss": 0.4183,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.421875,
      "rewards/margins": 1.203125,
      "rewards/rejected": -3.625,
      "step": 6440
    },
    {
      "epoch": 0.24008486720887384,
      "grad_norm": 8.729466042261377,
      "learning_rate": 4.7070672919315856e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.65625,
      "logps/chosen": -460.0,
      "logps/rejected": -548.0,
      "loss": 0.4373,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.4375,
      "rewards/margins": 1.0859375,
      "rewards/rejected": -3.515625,
      "step": 6450
    },
    {
      "epoch": 0.24045709180919767,
      "grad_norm": 8.822533177888092,
      "learning_rate": 4.705539660754207e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -462.0,
      "logps/rejected": -572.0,
      "loss": 0.4444,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.59375,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -3.796875,
      "step": 6460
    },
    {
      "epoch": 0.24082931640952152,
      "grad_norm": 8.116707730462094,
      "learning_rate": 4.704008305884332e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.609375,
      "logps/chosen": -484.0,
      "logps/rejected": -564.0,
      "loss": 0.4558,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.46875,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -3.75,
      "step": 6470
    },
    {
      "epoch": 0.24120154100984534,
      "grad_norm": 9.535781750475286,
      "learning_rate": 4.7024732299074023e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.625,
      "logps/chosen": -438.0,
      "logps/rejected": -556.0,
      "loss": 0.4347,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.328125,
      "rewards/margins": 1.359375,
      "rewards/rejected": -3.6875,
      "step": 6480
    },
    {
      "epoch": 0.2415737656101692,
      "grad_norm": 9.964415820359594,
      "learning_rate": 4.7009344354151424e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.78125,
      "logps/chosen": -462.0,
      "logps/rejected": -564.0,
      "loss": 0.4333,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -3.828125,
      "step": 6490
    },
    {
      "epoch": 0.241945990210493,
      "grad_norm": 9.92980142548488,
      "learning_rate": 4.6993919250055557e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.65625,
      "logps/chosen": -472.0,
      "logps/rejected": -576.0,
      "loss": 0.4503,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.484375,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -3.6875,
      "step": 6500
    },
    {
      "epoch": 0.24231821481081683,
      "grad_norm": 9.164209782883436,
      "learning_rate": 4.6978457012829174e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.890625,
      "logps/chosen": -472.0,
      "logps/rejected": -564.0,
      "loss": 0.4198,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -3.765625,
      "step": 6510
    },
    {
      "epoch": 0.24269043941114068,
      "grad_norm": 8.793942633166495,
      "learning_rate": 4.6962957668577736e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.90625,
      "logps/chosen": -480.0,
      "logps/rejected": -572.0,
      "loss": 0.4491,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.1875,
      "rewards/rejected": -3.734375,
      "step": 6520
    },
    {
      "epoch": 0.2430626640114645,
      "grad_norm": 10.17461532383169,
      "learning_rate": 4.694742124346934e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.828125,
      "logps/chosen": -460.0,
      "logps/rejected": -564.0,
      "loss": 0.449,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.15625,
      "rewards/rejected": -3.703125,
      "step": 6530
    },
    {
      "epoch": 0.24343488861178836,
      "grad_norm": 9.184032005775975,
      "learning_rate": 4.6931847763734703e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.828125,
      "logps/chosen": -458.0,
      "logps/rejected": -568.0,
      "loss": 0.447,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.453125,
      "rewards/margins": 1.1796875,
      "rewards/rejected": -3.640625,
      "step": 6540
    },
    {
      "epoch": 0.24380711321211218,
      "grad_norm": 9.16570246550826,
      "learning_rate": 4.6916237255667093e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.640625,
      "logps/chosen": -446.0,
      "logps/rejected": -572.0,
      "loss": 0.4437,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.375,
      "rewards/margins": 1.390625,
      "rewards/rejected": -3.765625,
      "step": 6550
    },
    {
      "epoch": 0.24417933781243603,
      "grad_norm": 8.370531974353044,
      "learning_rate": 4.6900589745622294e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.703125,
      "logps/chosen": -446.0,
      "logps/rejected": -560.0,
      "loss": 0.4343,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.4375,
      "rewards/margins": 1.15625,
      "rewards/rejected": -3.59375,
      "step": 6560
    },
    {
      "epoch": 0.24455156241275985,
      "grad_norm": 9.619193757148652,
      "learning_rate": 4.688490526001856e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.828125,
      "logps/chosen": -470.0,
      "logps/rejected": -572.0,
      "loss": 0.4283,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.125,
      "rewards/rejected": -3.859375,
      "step": 6570
    },
    {
      "epoch": 0.2449237870130837,
      "grad_norm": 10.084347577589655,
      "learning_rate": 4.6869183825336587e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.625,
      "logps/chosen": -452.0,
      "logps/rejected": -588.0,
      "loss": 0.4601,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.578125,
      "rewards/margins": 1.25,
      "rewards/rejected": -3.828125,
      "step": 6580
    },
    {
      "epoch": 0.24529601161340753,
      "grad_norm": 9.217694571321292,
      "learning_rate": 4.685342546811943e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.71875,
      "logps/chosen": -406.0,
      "logps/rejected": -552.0,
      "loss": 0.4425,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.1875,
      "rewards/margins": 1.46875,
      "rewards/rejected": -3.640625,
      "step": 6590
    },
    {
      "epoch": 0.24566823621373138,
      "grad_norm": 8.459201899021275,
      "learning_rate": 4.6837630214972514e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.8125,
      "logps/chosen": -486.0,
      "logps/rejected": -564.0,
      "loss": 0.4625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -2.546875,
      "rewards/margins": 0.94140625,
      "rewards/rejected": -3.5,
      "step": 6600
    },
    {
      "epoch": 0.2460404608140552,
      "grad_norm": 7.345996039141722,
      "learning_rate": 4.6821798092563514e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.75,
      "logps/chosen": -454.0,
      "logps/rejected": -536.0,
      "loss": 0.4271,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.3125,
      "rewards/margins": 1.0703125,
      "rewards/rejected": -3.375,
      "step": 6610
    },
    {
      "epoch": 0.24641268541437902,
      "grad_norm": 9.435267538244602,
      "learning_rate": 4.680592912762238e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.96875,
      "logps/chosen": -460.0,
      "logps/rejected": -588.0,
      "loss": 0.4192,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.5,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -3.921875,
      "step": 6620
    },
    {
      "epoch": 0.24678491001470287,
      "grad_norm": 11.759487612629192,
      "learning_rate": 4.6790023346941274e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.8125,
      "logps/chosen": -476.0,
      "logps/rejected": -580.0,
      "loss": 0.4226,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.3125,
      "rewards/rejected": -3.859375,
      "step": 6630
    },
    {
      "epoch": 0.2471571346150267,
      "grad_norm": 11.551666225551017,
      "learning_rate": 4.677408077737449e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.859375,
      "logps/chosen": -450.0,
      "logps/rejected": -576.0,
      "loss": 0.4562,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.5,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -3.84375,
      "step": 6640
    },
    {
      "epoch": 0.24752935921535055,
      "grad_norm": 8.601964562921653,
      "learning_rate": 4.6758101445838457e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.78125,
      "logps/chosen": -484.0,
      "logps/rejected": -580.0,
      "loss": 0.4393,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.671875,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -3.921875,
      "step": 6650
    },
    {
      "epoch": 0.24790158381567437,
      "grad_norm": 10.887547515783835,
      "learning_rate": 4.6742085379311645e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.84375,
      "logps/chosen": -464.0,
      "logps/rejected": -568.0,
      "loss": 0.4304,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.53125,
      "rewards/margins": 1.171875,
      "rewards/rejected": -3.703125,
      "step": 6660
    },
    {
      "epoch": 0.24827380841599822,
      "grad_norm": 9.609280867960655,
      "learning_rate": 4.6726032604834566e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.8125,
      "logps/chosen": -478.0,
      "logps/rejected": -596.0,
      "loss": 0.4368,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.65625,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.0,
      "step": 6670
    },
    {
      "epoch": 0.24864603301632204,
      "grad_norm": 10.201066908688517,
      "learning_rate": 4.67099431495097e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.8125,
      "logps/chosen": -466.0,
      "logps/rejected": -556.0,
      "loss": 0.4543,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.546875,
      "rewards/margins": 0.96484375,
      "rewards/rejected": -3.5,
      "step": 6680
    },
    {
      "epoch": 0.2490182576166459,
      "grad_norm": 9.982194233020216,
      "learning_rate": 4.669381704050146e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.984375,
      "logps/chosen": -466.0,
      "logps/rejected": -548.0,
      "loss": 0.4357,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.65625,
      "rewards/margins": 1.078125,
      "rewards/rejected": -3.734375,
      "step": 6690
    },
    {
      "epoch": 0.24939048221696972,
      "grad_norm": 8.433517413531952,
      "learning_rate": 4.6677654305036136e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.71875,
      "logps/chosen": -436.0,
      "logps/rejected": -564.0,
      "loss": 0.4433,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.40625,
      "rewards/margins": 1.3125,
      "rewards/rejected": -3.71875,
      "step": 6700
    },
    {
      "epoch": 0.24976270681729357,
      "grad_norm": 9.437867672018813,
      "learning_rate": 4.666145497040186e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -462.0,
      "logps/rejected": -576.0,
      "loss": 0.4404,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.359375,
      "rewards/rejected": -3.90625,
      "step": 6710
    },
    {
      "epoch": 0.2501349314176174,
      "grad_norm": 10.800555059414286,
      "learning_rate": 4.664521906394856e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.703125,
      "logps/chosen": -472.0,
      "logps/rejected": -596.0,
      "loss": 0.4277,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.59375,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.0625,
      "step": 6720
    },
    {
      "epoch": 0.2505071560179412,
      "grad_norm": 18.53084733998349,
      "learning_rate": 4.662894661308789e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.6875,
      "logps/chosen": -450.0,
      "logps/rejected": -524.0,
      "loss": 0.4708,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.046875,
      "rewards/rejected": -3.59375,
      "step": 6730
    },
    {
      "epoch": 0.25087938061826504,
      "grad_norm": 8.913495521187285,
      "learning_rate": 4.6612637645293243e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.59375,
      "logps/chosen": -440.0,
      "logps/rejected": -580.0,
      "loss": 0.4441,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.34375,
      "rewards/margins": 1.359375,
      "rewards/rejected": -3.703125,
      "step": 6740
    },
    {
      "epoch": 0.2512516052185889,
      "grad_norm": 9.483013120269376,
      "learning_rate": 4.659629218809963e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.53125,
      "logps/chosen": -438.0,
      "logps/rejected": -564.0,
      "loss": 0.4229,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.328125,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -3.78125,
      "step": 6750
    },
    {
      "epoch": 0.25162382981891274,
      "grad_norm": 8.500166614201406,
      "learning_rate": 4.657991026910366e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.546875,
      "logps/chosen": -472.0,
      "logps/rejected": -572.0,
      "loss": 0.433,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.265625,
      "rewards/rejected": -3.90625,
      "step": 6760
    },
    {
      "epoch": 0.25199605441923656,
      "grad_norm": 9.886645233097864,
      "learning_rate": 4.656349191596355e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.671875,
      "logps/chosen": -464.0,
      "logps/rejected": -608.0,
      "loss": 0.4111,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.59375,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.125,
      "step": 6770
    },
    {
      "epoch": 0.2523682790195604,
      "grad_norm": 9.721464224310834,
      "learning_rate": 4.6547037156398976e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.484375,
      "logps/chosen": -422.0,
      "logps/rejected": -548.0,
      "loss": 0.4107,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.359375,
      "rewards/margins": 1.40625,
      "rewards/rejected": -3.765625,
      "step": 6780
    },
    {
      "epoch": 0.25274050361988426,
      "grad_norm": 10.62033742846061,
      "learning_rate": 4.653054601819112e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.71875,
      "logps/chosen": -446.0,
      "logps/rejected": -588.0,
      "loss": 0.429,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.421875,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.0,
      "step": 6790
    },
    {
      "epoch": 0.2531127282202081,
      "grad_norm": 8.860584380083635,
      "learning_rate": 4.651401852918256e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.625,
      "logps/chosen": -466.0,
      "logps/rejected": -612.0,
      "loss": 0.4349,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.546875,
      "rewards/rejected": -4.09375,
      "step": 6800
    },
    {
      "epoch": 0.2534849528205319,
      "grad_norm": 8.996625342326046,
      "learning_rate": 4.6497454717277253e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.6875,
      "logps/chosen": -448.0,
      "logps/rejected": -560.0,
      "loss": 0.4266,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.421875,
      "rewards/margins": 1.234375,
      "rewards/rejected": -3.65625,
      "step": 6810
    },
    {
      "epoch": 0.25385717742085573,
      "grad_norm": 9.789815696649658,
      "learning_rate": 4.648085461044049e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -446.0,
      "logps/rejected": -548.0,
      "loss": 0.4327,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.25,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -3.5,
      "step": 6820
    },
    {
      "epoch": 0.2542294020211796,
      "grad_norm": 8.77488215193305,
      "learning_rate": 4.646421823669883e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -468.0,
      "logps/rejected": -580.0,
      "loss": 0.4423,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.53125,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -3.734375,
      "step": 6830
    },
    {
      "epoch": 0.25460162662150343,
      "grad_norm": 11.239970436792927,
      "learning_rate": 4.644754562414006e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -426.0,
      "logps/rejected": -528.0,
      "loss": 0.4382,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.265625,
      "rewards/margins": 1.125,
      "rewards/rejected": -3.390625,
      "step": 6840
    },
    {
      "epoch": 0.25497385122182725,
      "grad_norm": 8.103429049831506,
      "learning_rate": 4.6430836800913167e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -452.0,
      "logps/rejected": -552.0,
      "loss": 0.4173,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.265625,
      "rewards/margins": 1.453125,
      "rewards/rejected": -3.71875,
      "step": 6850
    },
    {
      "epoch": 0.2553460758221511,
      "grad_norm": 9.613661225490846,
      "learning_rate": 4.6414091795228247e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.515625,
      "logps/chosen": -472.0,
      "logps/rejected": -576.0,
      "loss": 0.4256,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.453125,
      "rewards/margins": 1.234375,
      "rewards/rejected": -3.6875,
      "step": 6860
    },
    {
      "epoch": 0.2557183004224749,
      "grad_norm": 11.814745805994798,
      "learning_rate": 4.6397310635356514e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.734375,
      "logps/chosen": -456.0,
      "logps/rejected": -556.0,
      "loss": 0.4386,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.5625,
      "rewards/margins": 1.1875,
      "rewards/rejected": -3.75,
      "step": 6870
    },
    {
      "epoch": 0.2560905250227988,
      "grad_norm": 8.715661908273438,
      "learning_rate": 4.63804933496302e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.5625,
      "logps/chosen": -424.0,
      "logps/rejected": -528.0,
      "loss": 0.4426,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.296875,
      "rewards/margins": 1.265625,
      "rewards/rejected": -3.5625,
      "step": 6880
    },
    {
      "epoch": 0.2564627496231226,
      "grad_norm": 7.575911055222431,
      "learning_rate": 4.6363639966442535e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.515625,
      "logps/chosen": -464.0,
      "logps/rejected": -584.0,
      "loss": 0.4432,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.34375,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -3.65625,
      "step": 6890
    },
    {
      "epoch": 0.2568349742234464,
      "grad_norm": 10.51609223645884,
      "learning_rate": 4.634675051424771e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -462.0,
      "logps/rejected": -588.0,
      "loss": 0.4471,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.484375,
      "rewards/margins": 1.390625,
      "rewards/rejected": -3.875,
      "step": 6900
    },
    {
      "epoch": 0.25720719882377024,
      "grad_norm": 7.83513638873318,
      "learning_rate": 4.632982502156078e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.6875,
      "logps/chosen": -458.0,
      "logps/rejected": -552.0,
      "loss": 0.4389,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.421875,
      "rewards/margins": 1.21875,
      "rewards/rejected": -3.640625,
      "step": 6910
    },
    {
      "epoch": 0.2575794234240941,
      "grad_norm": 10.507423960659995,
      "learning_rate": 4.6312863516957686e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.6875,
      "logps/chosen": -466.0,
      "logps/rejected": -576.0,
      "loss": 0.4355,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.515625,
      "rewards/margins": 1.296875,
      "rewards/rejected": -3.8125,
      "step": 6920
    },
    {
      "epoch": 0.25795164802441795,
      "grad_norm": 8.131644298731901,
      "learning_rate": 4.629586602907514e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.859375,
      "logps/chosen": -430.0,
      "logps/rejected": -536.0,
      "loss": 0.4624,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.34375,
      "rewards/margins": 1.171875,
      "rewards/rejected": -3.515625,
      "step": 6930
    },
    {
      "epoch": 0.25832387262474177,
      "grad_norm": 9.14079080789668,
      "learning_rate": 4.627883258661061e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -438.0,
      "logps/rejected": -560.0,
      "loss": 0.4273,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.171875,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -3.625,
      "step": 6940
    },
    {
      "epoch": 0.2586960972250656,
      "grad_norm": 8.919421973650937,
      "learning_rate": 4.626176321832229e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -444.0,
      "logps/rejected": -556.0,
      "loss": 0.4293,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.421875,
      "rewards/margins": 1.140625,
      "rewards/rejected": -3.5625,
      "step": 6950
    },
    {
      "epoch": 0.2590683218253894,
      "grad_norm": 10.168868511736289,
      "learning_rate": 4.6244657953029005e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.734375,
      "logps/chosen": -484.0,
      "logps/rejected": -592.0,
      "loss": 0.4337,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.140625,
      "rewards/rejected": -3.84375,
      "step": 6960
    },
    {
      "epoch": 0.2594405464257133,
      "grad_norm": 9.017688536130056,
      "learning_rate": 4.622751681961019e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.71875,
      "logps/chosen": -472.0,
      "logps/rejected": -572.0,
      "loss": 0.4312,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.40625,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -3.75,
      "step": 6970
    },
    {
      "epoch": 0.2598127710260371,
      "grad_norm": 8.518544456520402,
      "learning_rate": 4.621033984700585e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.59375,
      "logps/chosen": -440.0,
      "logps/rejected": -532.0,
      "loss": 0.4263,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.34375,
      "rewards/margins": 1.1328125,
      "rewards/rejected": -3.46875,
      "step": 6980
    },
    {
      "epoch": 0.26018499562636094,
      "grad_norm": 8.13945512872423,
      "learning_rate": 4.6193127064216486e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.8125,
      "logps/chosen": -458.0,
      "logps/rejected": -576.0,
      "loss": 0.4,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.4375,
      "rewards/margins": 1.40625,
      "rewards/rejected": -3.859375,
      "step": 6990
    },
    {
      "epoch": 0.26055722022668476,
      "grad_norm": 8.854579649104878,
      "learning_rate": 4.6175878500303054e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.6875,
      "logps/chosen": -446.0,
      "logps/rejected": -580.0,
      "loss": 0.4358,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.53125,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -3.984375,
      "step": 7000
    },
    {
      "epoch": 0.26092944482700864,
      "grad_norm": 9.77046137363716,
      "learning_rate": 4.615859418438695e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.6875,
      "logps/chosen": -486.0,
      "logps/rejected": -588.0,
      "loss": 0.4283,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.578125,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -3.90625,
      "step": 7010
    },
    {
      "epoch": 0.26130166942733246,
      "grad_norm": 6.8932037029253515,
      "learning_rate": 4.6141274145649876e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.796875,
      "logps/chosen": -466.0,
      "logps/rejected": -580.0,
      "loss": 0.4322,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.671875,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -3.796875,
      "step": 7020
    },
    {
      "epoch": 0.2616738940276563,
      "grad_norm": 9.433524020779906,
      "learning_rate": 4.612391841333392e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5625,
      "logps/chosen": -468.0,
      "logps/rejected": -588.0,
      "loss": 0.4237,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.390625,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -3.828125,
      "step": 7030
    },
    {
      "epoch": 0.2620461186279801,
      "grad_norm": 8.057626804589518,
      "learning_rate": 4.6106527016741377e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.671875,
      "logps/chosen": -502.0,
      "logps/rejected": -600.0,
      "loss": 0.4299,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.1796875,
      "rewards/rejected": -3.890625,
      "step": 7040
    },
    {
      "epoch": 0.262418343228304,
      "grad_norm": 8.984675603160092,
      "learning_rate": 4.608909998523476e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.828125,
      "logps/chosen": -480.0,
      "logps/rejected": -592.0,
      "loss": 0.4339,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.03125,
      "step": 7050
    },
    {
      "epoch": 0.2627905678286278,
      "grad_norm": 9.111858853610967,
      "learning_rate": 4.607163734823678e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.65625,
      "logps/chosen": -478.0,
      "logps/rejected": -588.0,
      "loss": 0.4292,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.328125,
      "rewards/rejected": -3.875,
      "step": 7060
    },
    {
      "epoch": 0.26316279242895163,
      "grad_norm": 8.791696766295743,
      "learning_rate": 4.605413913523022e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.828125,
      "logps/chosen": -484.0,
      "logps/rejected": -592.0,
      "loss": 0.4262,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.234375,
      "rewards/rejected": -3.96875,
      "step": 7070
    },
    {
      "epoch": 0.26353501702927545,
      "grad_norm": 9.056891980567295,
      "learning_rate": 4.603660537575796e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.75,
      "logps/chosen": -472.0,
      "logps/rejected": -568.0,
      "loss": 0.4253,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.578125,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -3.8125,
      "step": 7080
    },
    {
      "epoch": 0.2639072416295993,
      "grad_norm": 9.76995832756375,
      "learning_rate": 4.601903609942287e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.75,
      "logps/chosen": -492.0,
      "logps/rejected": -624.0,
      "loss": 0.415,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.125,
      "step": 7090
    },
    {
      "epoch": 0.26427946622992315,
      "grad_norm": 10.315057524583661,
      "learning_rate": 4.600143133588781e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.703125,
      "logps/chosen": -458.0,
      "logps/rejected": -592.0,
      "loss": 0.4033,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.578125,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.0,
      "step": 7100
    },
    {
      "epoch": 0.264651690830247,
      "grad_norm": 10.389485477769302,
      "learning_rate": 4.598379111487552e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.453125,
      "logps/chosen": -454.0,
      "logps/rejected": -632.0,
      "loss": 0.442,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.5,
      "rewards/margins": 1.546875,
      "rewards/rejected": -4.03125,
      "step": 7110
    },
    {
      "epoch": 0.2650239154305708,
      "grad_norm": 10.407405223821284,
      "learning_rate": 4.5966115466168645e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.46875,
      "logps/chosen": -510.0,
      "logps/rejected": -612.0,
      "loss": 0.4295,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.234375,
      "rewards/rejected": -4.0625,
      "step": 7120
    },
    {
      "epoch": 0.2653961400308946,
      "grad_norm": 10.81606273534761,
      "learning_rate": 4.594840441960961e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5,
      "logps/chosen": -474.0,
      "logps/rejected": -620.0,
      "loss": 0.4381,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.65625,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.1875,
      "step": 7130
    },
    {
      "epoch": 0.2657683646312185,
      "grad_norm": 8.48960924243069,
      "learning_rate": 4.593065800510062e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.546875,
      "logps/chosen": -450.0,
      "logps/rejected": -560.0,
      "loss": 0.4207,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.5625,
      "rewards/margins": 1.171875,
      "rewards/rejected": -3.734375,
      "step": 7140
    },
    {
      "epoch": 0.2661405892315423,
      "grad_norm": 9.373736432458722,
      "learning_rate": 4.5912876252603585e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.453125,
      "logps/chosen": -462.0,
      "logps/rejected": -592.0,
      "loss": 0.4193,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.65625,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -4.0,
      "step": 7150
    },
    {
      "epoch": 0.26651281383186615,
      "grad_norm": 8.751990899391556,
      "learning_rate": 4.5895059192140095e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.65625,
      "logps/chosen": -486.0,
      "logps/rejected": -592.0,
      "loss": 0.4532,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -4.03125,
      "step": 7160
    },
    {
      "epoch": 0.26688503843218997,
      "grad_norm": 9.737576232353794,
      "learning_rate": 4.587720685379132e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.8125,
      "logps/chosen": -478.0,
      "logps/rejected": -596.0,
      "loss": 0.4436,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.0625,
      "step": 7170
    },
    {
      "epoch": 0.2672572630325138,
      "grad_norm": 11.105763047454959,
      "learning_rate": 4.5859319267698025e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.546875,
      "logps/chosen": -484.0,
      "logps/rejected": -552.0,
      "loss": 0.4484,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -3.734375,
      "step": 7180
    },
    {
      "epoch": 0.26762948763283767,
      "grad_norm": 9.315733063644277,
      "learning_rate": 4.584139646406047e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.578125,
      "logps/chosen": -430.0,
      "logps/rejected": -564.0,
      "loss": 0.416,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.34375,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -3.625,
      "step": 7190
    },
    {
      "epoch": 0.2680017122331615,
      "grad_norm": 9.73273608602203,
      "learning_rate": 4.5823438473138353e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.625,
      "logps/chosen": -432.0,
      "logps/rejected": -548.0,
      "loss": 0.4419,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.390625,
      "rewards/margins": 1.171875,
      "rewards/rejected": -3.578125,
      "step": 7200
    },
    {
      "epoch": 0.2683739368334853,
      "grad_norm": 9.963624463643777,
      "learning_rate": 4.5805445325250826e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.65625,
      "logps/chosen": -432.0,
      "logps/rejected": -572.0,
      "loss": 0.4251,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.25,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -3.6875,
      "step": 7210
    },
    {
      "epoch": 0.26874616143380914,
      "grad_norm": 9.891816396608306,
      "learning_rate": 4.578741705077636e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.609375,
      "logps/chosen": -484.0,
      "logps/rejected": -608.0,
      "loss": 0.4447,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.03125,
      "step": 7220
    },
    {
      "epoch": 0.269118386034133,
      "grad_norm": 9.21440161811792,
      "learning_rate": 4.5769353680152735e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.6875,
      "logps/chosen": -470.0,
      "logps/rejected": -588.0,
      "loss": 0.4414,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.515625,
      "rewards/margins": 1.359375,
      "rewards/rejected": -3.875,
      "step": 7230
    },
    {
      "epoch": 0.26949061063445684,
      "grad_norm": 9.133279908856897,
      "learning_rate": 4.575125524387701e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.671875,
      "logps/chosen": -452.0,
      "logps/rejected": -544.0,
      "loss": 0.4438,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.328125,
      "rewards/margins": 1.140625,
      "rewards/rejected": -3.46875,
      "step": 7240
    },
    {
      "epoch": 0.26986283523478066,
      "grad_norm": 8.760256145446002,
      "learning_rate": 4.573312177250543e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -456.0,
      "logps/rejected": -540.0,
      "loss": 0.4231,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.46875,
      "rewards/margins": 1.109375,
      "rewards/rejected": -3.578125,
      "step": 7250
    },
    {
      "epoch": 0.2702350598351045,
      "grad_norm": 10.851653624795185,
      "learning_rate": 4.571495329665338e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.640625,
      "logps/chosen": -454.0,
      "logps/rejected": -564.0,
      "loss": 0.4501,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.625,
      "rewards/margins": 1.171875,
      "rewards/rejected": -3.796875,
      "step": 7260
    },
    {
      "epoch": 0.27060728443542836,
      "grad_norm": 8.806138053875072,
      "learning_rate": 4.5696749846995365e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -472.0,
      "logps/rejected": -568.0,
      "loss": 0.444,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.484375,
      "rewards/margins": 1.296875,
      "rewards/rejected": -3.78125,
      "step": 7270
    },
    {
      "epoch": 0.2709795090357522,
      "grad_norm": 9.918139231342344,
      "learning_rate": 4.5678511454264924e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -450.0,
      "logps/rejected": -584.0,
      "loss": 0.4314,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.28125,
      "rewards/margins": 1.390625,
      "rewards/rejected": -3.671875,
      "step": 7280
    },
    {
      "epoch": 0.271351733636076,
      "grad_norm": 9.503170357790024,
      "learning_rate": 4.566023814925459e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.71875,
      "logps/chosen": -462.0,
      "logps/rejected": -568.0,
      "loss": 0.4293,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.53125,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -3.796875,
      "step": 7290
    },
    {
      "epoch": 0.27172395823639983,
      "grad_norm": 8.484825658152953,
      "learning_rate": 4.5641929962815863e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.515625,
      "logps/chosen": -464.0,
      "logps/rejected": -584.0,
      "loss": 0.4488,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.65625,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -3.890625,
      "step": 7300
    },
    {
      "epoch": 0.27209618283672365,
      "grad_norm": 9.13812060603436,
      "learning_rate": 4.56235869258591e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.578125,
      "logps/chosen": -468.0,
      "logps/rejected": -564.0,
      "loss": 0.448,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.65625,
      "rewards/margins": 1.25,
      "rewards/rejected": -3.90625,
      "step": 7310
    },
    {
      "epoch": 0.27246840743704753,
      "grad_norm": 9.617210960051262,
      "learning_rate": 4.5605209069353525e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.625,
      "logps/chosen": -498.0,
      "logps/rejected": -640.0,
      "loss": 0.4024,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.421875,
      "rewards/rejected": -4.28125,
      "step": 7320
    },
    {
      "epoch": 0.27284063203737136,
      "grad_norm": 8.648262999367194,
      "learning_rate": 4.5586796424327135e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.765625,
      "logps/chosen": -506.0,
      "logps/rejected": -608.0,
      "loss": 0.4375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -4.15625,
      "step": 7330
    },
    {
      "epoch": 0.2732128566376952,
      "grad_norm": 9.409351921949998,
      "learning_rate": 4.5568349021866666e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.546875,
      "logps/chosen": -496.0,
      "logps/rejected": -640.0,
      "loss": 0.426,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.28125,
      "step": 7340
    },
    {
      "epoch": 0.273585081238019,
      "grad_norm": 9.087899684864666,
      "learning_rate": 4.554986689311754e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.46875,
      "logps/chosen": -458.0,
      "logps/rejected": -580.0,
      "loss": 0.464,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.359375,
      "rewards/rejected": -4.0625,
      "step": 7350
    },
    {
      "epoch": 0.2739573058383429,
      "grad_norm": 9.338649534975936,
      "learning_rate": 4.553135006928379e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.65625,
      "logps/chosen": -470.0,
      "logps/rejected": -620.0,
      "loss": 0.4398,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.1875,
      "step": 7360
    },
    {
      "epoch": 0.2743295304386667,
      "grad_norm": 10.460411863045259,
      "learning_rate": 4.551279858162806e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.796875,
      "logps/chosen": -482.0,
      "logps/rejected": -620.0,
      "loss": 0.4196,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.1875,
      "step": 7370
    },
    {
      "epoch": 0.2747017550389905,
      "grad_norm": 8.914197891651543,
      "learning_rate": 4.549421246147149e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.734375,
      "logps/chosen": -470.0,
      "logps/rejected": -576.0,
      "loss": 0.4309,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.671875,
      "rewards/margins": 1.25,
      "rewards/rejected": -3.921875,
      "step": 7380
    },
    {
      "epoch": 0.27507397963931435,
      "grad_norm": 9.218531598769285,
      "learning_rate": 4.547559174019369e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.578125,
      "logps/chosen": -460.0,
      "logps/rejected": -616.0,
      "loss": 0.411,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.578125,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.125,
      "step": 7390
    },
    {
      "epoch": 0.27544620423963817,
      "grad_norm": 10.74423858798588,
      "learning_rate": 4.5456936449232715e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5625,
      "logps/chosen": -438.0,
      "logps/rejected": -592.0,
      "loss": 0.4187,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.53125,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.03125,
      "step": 7400
    },
    {
      "epoch": 0.27581842883996205,
      "grad_norm": 8.642040444172453,
      "learning_rate": 4.543824662008496e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.671875,
      "logps/chosen": -504.0,
      "logps/rejected": -624.0,
      "loss": 0.4155,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.21875,
      "step": 7410
    },
    {
      "epoch": 0.27619065344028587,
      "grad_norm": 8.575418327177706,
      "learning_rate": 4.541952228430514e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.734375,
      "logps/chosen": -490.0,
      "logps/rejected": -576.0,
      "loss": 0.4431,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.203125,
      "rewards/rejected": -3.96875,
      "step": 7420
    },
    {
      "epoch": 0.2765628780406097,
      "grad_norm": 12.218337953263015,
      "learning_rate": 4.540076347350623e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.4375,
      "logps/chosen": -520.0,
      "logps/rejected": -632.0,
      "loss": 0.4089,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.34375,
      "step": 7430
    },
    {
      "epoch": 0.2769351026409335,
      "grad_norm": 13.724993061529444,
      "learning_rate": 4.538197021935941e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.734375,
      "logps/chosen": -502.0,
      "logps/rejected": -604.0,
      "loss": 0.4176,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.0625,
      "step": 7440
    },
    {
      "epoch": 0.2773073272412574,
      "grad_norm": 8.240593746778485,
      "learning_rate": 4.5363142553594014e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.65625,
      "logps/chosen": -468.0,
      "logps/rejected": -596.0,
      "loss": 0.4224,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.46875,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -3.953125,
      "step": 7450
    },
    {
      "epoch": 0.2776795518415812,
      "grad_norm": 9.161658338903422,
      "learning_rate": 4.534428050799747e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.71875,
      "logps/chosen": -478.0,
      "logps/rejected": -604.0,
      "loss": 0.4205,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.65625,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.15625,
      "step": 7460
    },
    {
      "epoch": 0.27805177644190504,
      "grad_norm": 8.453790251809252,
      "learning_rate": 4.5325384114415254e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.75,
      "logps/chosen": -472.0,
      "logps/rejected": -588.0,
      "loss": 0.4364,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.578125,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.0625,
      "step": 7470
    },
    {
      "epoch": 0.27842400104222886,
      "grad_norm": 9.509368467159304,
      "learning_rate": 4.530645340475083e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.703125,
      "logps/chosen": -466.0,
      "logps/rejected": -592.0,
      "loss": 0.4192,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.46875,
      "rewards/margins": 1.515625,
      "rewards/rejected": -3.984375,
      "step": 7480
    },
    {
      "epoch": 0.27879622564255274,
      "grad_norm": 10.76265479469647,
      "learning_rate": 4.52874884109656e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.703125,
      "logps/chosen": -466.0,
      "logps/rejected": -636.0,
      "loss": 0.4283,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.625,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.25,
      "step": 7490
    },
    {
      "epoch": 0.27916845024287656,
      "grad_norm": 10.213159548556993,
      "learning_rate": 4.5268489165078855e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.84375,
      "logps/chosen": -488.0,
      "logps/rejected": -592.0,
      "loss": 0.4624,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.59375,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -3.921875,
      "step": 7500
    },
    {
      "epoch": 0.2795406748432004,
      "grad_norm": 8.856338678405313,
      "learning_rate": 4.5249455699167706e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.828125,
      "logps/chosen": -452.0,
      "logps/rejected": -552.0,
      "loss": 0.4242,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.53125,
      "rewards/margins": 1.1796875,
      "rewards/rejected": -3.703125,
      "step": 7510
    },
    {
      "epoch": 0.2799128994435242,
      "grad_norm": 9.970300732053747,
      "learning_rate": 4.523038804536704e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.671875,
      "logps/chosen": -476.0,
      "logps/rejected": -592.0,
      "loss": 0.4226,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.03125,
      "step": 7520
    },
    {
      "epoch": 0.28028512404384803,
      "grad_norm": 10.34812992220721,
      "learning_rate": 4.521128623586947e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.640625,
      "logps/chosen": -488.0,
      "logps/rejected": -616.0,
      "loss": 0.4444,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.1875,
      "step": 7530
    },
    {
      "epoch": 0.2806573486441719,
      "grad_norm": 10.194555807147292,
      "learning_rate": 4.519215030292527e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.78125,
      "logps/chosen": -476.0,
      "logps/rejected": -584.0,
      "loss": 0.4551,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.296875,
      "rewards/rejected": -3.984375,
      "step": 7540
    },
    {
      "epoch": 0.28102957324449573,
      "grad_norm": 8.36996248403759,
      "learning_rate": 4.517298027884234e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.75,
      "logps/chosen": -478.0,
      "logps/rejected": -568.0,
      "loss": 0.4363,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.359375,
      "rewards/margins": 1.203125,
      "rewards/rejected": -3.578125,
      "step": 7550
    },
    {
      "epoch": 0.28140179784481956,
      "grad_norm": 10.725301632401019,
      "learning_rate": 4.5153776195986113e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -506.0,
      "logps/rejected": -564.0,
      "loss": 0.4409,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.765625,
      "rewards/margins": 0.92578125,
      "rewards/rejected": -3.703125,
      "step": 7560
    },
    {
      "epoch": 0.2817740224451434,
      "grad_norm": 9.175718841863716,
      "learning_rate": 4.513453808677955e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -490.0,
      "logps/rejected": -600.0,
      "loss": 0.3965,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.609375,
      "rewards/margins": 1.265625,
      "rewards/rejected": -3.875,
      "step": 7570
    },
    {
      "epoch": 0.28214624704546726,
      "grad_norm": 9.875717738271442,
      "learning_rate": 4.5115265983703036e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5,
      "logps/chosen": -464.0,
      "logps/rejected": -588.0,
      "loss": 0.4206,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.125,
      "step": 7580
    },
    {
      "epoch": 0.2825184716457911,
      "grad_norm": 8.08008980392857,
      "learning_rate": 4.5095959919294366e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.765625,
      "logps/chosen": -468.0,
      "logps/rejected": -580.0,
      "loss": 0.4215,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.34375,
      "rewards/rejected": -3.984375,
      "step": 7590
    },
    {
      "epoch": 0.2828906962461149,
      "grad_norm": 9.3727655517383,
      "learning_rate": 4.5076619926148673e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.71875,
      "logps/chosen": -504.0,
      "logps/rejected": -636.0,
      "loss": 0.4369,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.21875,
      "step": 7600
    },
    {
      "epoch": 0.2832629208464387,
      "grad_norm": 9.419274069705734,
      "learning_rate": 4.5057246036918357e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.8125,
      "logps/chosen": -510.0,
      "logps/rejected": -652.0,
      "loss": 0.4253,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.3125,
      "step": 7610
    },
    {
      "epoch": 0.28363514544676255,
      "grad_norm": 8.536239914529924,
      "learning_rate": 4.503783828431306e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.796875,
      "logps/chosen": -456.0,
      "logps/rejected": -576.0,
      "loss": 0.437,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.5625,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -3.78125,
      "step": 7620
    },
    {
      "epoch": 0.2840073700470864,
      "grad_norm": 11.938479017755395,
      "learning_rate": 4.50183967010996e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.65625,
      "logps/chosen": -486.0,
      "logps/rejected": -596.0,
      "loss": 0.434,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -4.0625,
      "step": 7630
    },
    {
      "epoch": 0.28437959464741025,
      "grad_norm": 9.156125970325142,
      "learning_rate": 4.49989213201019e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.65625,
      "logps/chosen": -470.0,
      "logps/rejected": -584.0,
      "loss": 0.4417,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.296875,
      "rewards/rejected": -3.984375,
      "step": 7640
    },
    {
      "epoch": 0.2847518192477341,
      "grad_norm": 9.787843995519967,
      "learning_rate": 4.497941217420095e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5,
      "logps/chosen": -486.0,
      "logps/rejected": -592.0,
      "loss": 0.4142,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.65625,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -3.984375,
      "step": 7650
    },
    {
      "epoch": 0.2851240438480579,
      "grad_norm": 10.75056224535599,
      "learning_rate": 4.495986929633476e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.546875,
      "logps/chosen": -480.0,
      "logps/rejected": -588.0,
      "loss": 0.448,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.125,
      "rewards/rejected": -3.90625,
      "step": 7660
    },
    {
      "epoch": 0.2854962684483818,
      "grad_norm": 11.403104723093547,
      "learning_rate": 4.494029271949826e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -478.0,
      "logps/rejected": -584.0,
      "loss": 0.4106,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.1875,
      "rewards/rejected": -3.921875,
      "step": 7670
    },
    {
      "epoch": 0.2858684930487056,
      "grad_norm": 11.431081383567781,
      "learning_rate": 4.492068247674331e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.515625,
      "logps/chosen": -472.0,
      "logps/rejected": -592.0,
      "loss": 0.4549,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.15625,
      "step": 7680
    },
    {
      "epoch": 0.2862407176490294,
      "grad_norm": 12.95278551518027,
      "learning_rate": 4.490103860117858e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -472.0,
      "logps/rejected": -612.0,
      "loss": 0.438,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.09375,
      "step": 7690
    },
    {
      "epoch": 0.28661294224935324,
      "grad_norm": 8.751232259927255,
      "learning_rate": 4.4881361125969546e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.703125,
      "logps/chosen": -468.0,
      "logps/rejected": -584.0,
      "loss": 0.4164,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.0625,
      "step": 7700
    },
    {
      "epoch": 0.2869851668496771,
      "grad_norm": 8.677440798975486,
      "learning_rate": 4.48616500843384e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.65625,
      "logps/chosen": -488.0,
      "logps/rejected": -584.0,
      "loss": 0.4307,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.671875,
      "rewards/margins": 1.25,
      "rewards/rejected": -3.90625,
      "step": 7710
    },
    {
      "epoch": 0.28735739145000094,
      "grad_norm": 9.621854374706956,
      "learning_rate": 4.4841905509564e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.78125,
      "logps/chosen": -496.0,
      "logps/rejected": -584.0,
      "loss": 0.413,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.25,
      "rewards/rejected": -3.984375,
      "step": 7720
    },
    {
      "epoch": 0.28772961605032477,
      "grad_norm": 9.863773088692668,
      "learning_rate": 4.4822127434981845e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.515625,
      "logps/chosen": -502.0,
      "logps/rejected": -640.0,
      "loss": 0.419,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.015625,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.53125,
      "step": 7730
    },
    {
      "epoch": 0.2881018406506486,
      "grad_norm": 9.629006241828245,
      "learning_rate": 4.4802315893983957e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.6875,
      "logps/chosen": -482.0,
      "logps/rejected": -616.0,
      "loss": 0.4154,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.21875,
      "step": 7740
    },
    {
      "epoch": 0.2884740652509724,
      "grad_norm": 8.399658045271744,
      "learning_rate": 4.4782470920018875e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.640625,
      "logps/chosen": -476.0,
      "logps/rejected": -604.0,
      "loss": 0.4289,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.21875,
      "step": 7750
    },
    {
      "epoch": 0.2888462898512963,
      "grad_norm": 9.017159699371572,
      "learning_rate": 4.4762592546591617e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.46875,
      "logps/chosen": -480.0,
      "logps/rejected": -604.0,
      "loss": 0.4496,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.28125,
      "rewards/rejected": -4.125,
      "step": 7760
    },
    {
      "epoch": 0.2892185144516201,
      "grad_norm": 9.408658397572184,
      "learning_rate": 4.4742680807263524e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.515625,
      "logps/chosen": -460.0,
      "logps/rejected": -604.0,
      "loss": 0.4354,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.59375,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.0625,
      "step": 7770
    },
    {
      "epoch": 0.28959073905194394,
      "grad_norm": 9.627428221974103,
      "learning_rate": 4.4722735735652327e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.34375,
      "logps/chosen": -498.0,
      "logps/rejected": -628.0,
      "loss": 0.4451,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.375,
      "step": 7780
    },
    {
      "epoch": 0.28996296365226776,
      "grad_norm": 9.26640264714739,
      "learning_rate": 4.4702757365432007e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.65625,
      "logps/chosen": -490.0,
      "logps/rejected": -624.0,
      "loss": 0.4136,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.1875,
      "step": 7790
    },
    {
      "epoch": 0.29033518825259164,
      "grad_norm": 10.643869942850532,
      "learning_rate": 4.468274573033278e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.453125,
      "logps/chosen": -506.0,
      "logps/rejected": -608.0,
      "loss": 0.4557,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -4.15625,
      "step": 7800
    },
    {
      "epoch": 0.29070741285291546,
      "grad_norm": 8.787682928008328,
      "learning_rate": 4.4662700864141e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.6875,
      "logps/chosen": -482.0,
      "logps/rejected": -580.0,
      "loss": 0.4303,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.609375,
      "rewards/margins": 1.078125,
      "rewards/rejected": -3.6875,
      "step": 7810
    },
    {
      "epoch": 0.2910796374532393,
      "grad_norm": 9.554848855267512,
      "learning_rate": 4.4642622800699155e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.75,
      "logps/chosen": -492.0,
      "logps/rejected": -608.0,
      "loss": 0.4315,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.171875,
      "rewards/rejected": -4.03125,
      "step": 7820
    },
    {
      "epoch": 0.2914518620535631,
      "grad_norm": 8.06280449059316,
      "learning_rate": 4.4622511573905754e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.65625,
      "logps/chosen": -470.0,
      "logps/rejected": -592.0,
      "loss": 0.425,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.625,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -4.03125,
      "step": 7830
    },
    {
      "epoch": 0.291824086653887,
      "grad_norm": 8.589068284877582,
      "learning_rate": 4.460236721771531e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.859375,
      "logps/chosen": -466.0,
      "logps/rejected": -560.0,
      "loss": 0.4227,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -3.71875,
      "step": 7840
    },
    {
      "epoch": 0.2921963112542108,
      "grad_norm": 9.991572915165573,
      "learning_rate": 4.458218976613828e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.6875,
      "logps/chosen": -470.0,
      "logps/rejected": -580.0,
      "loss": 0.4329,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.65625,
      "rewards/margins": 1.25,
      "rewards/rejected": -3.90625,
      "step": 7850
    },
    {
      "epoch": 0.29256853585453463,
      "grad_norm": 8.641830768744516,
      "learning_rate": 4.456197925324098e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.71875,
      "logps/chosen": -500.0,
      "logps/rejected": -612.0,
      "loss": 0.43,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.3125,
      "rewards/rejected": -4.25,
      "step": 7860
    },
    {
      "epoch": 0.29294076045485845,
      "grad_norm": 9.514452977760614,
      "learning_rate": 4.4541735713145546e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.71875,
      "logps/chosen": -480.0,
      "logps/rejected": -604.0,
      "loss": 0.4309,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.25,
      "step": 7870
    },
    {
      "epoch": 0.2933129850551823,
      "grad_norm": 8.488063710651373,
      "learning_rate": 4.4521459180029884e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.671875,
      "logps/chosen": -482.0,
      "logps/rejected": -596.0,
      "loss": 0.4316,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.25,
      "rewards/rejected": -4.0,
      "step": 7880
    },
    {
      "epoch": 0.29368520965550615,
      "grad_norm": 8.9022521909282,
      "learning_rate": 4.45011496881276e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.71875,
      "logps/chosen": -494.0,
      "logps/rejected": -592.0,
      "loss": 0.4201,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.296875,
      "rewards/rejected": -4.125,
      "step": 7890
    },
    {
      "epoch": 0.29405743425583,
      "grad_norm": 10.665920937738507,
      "learning_rate": 4.4480807271727954e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.65625,
      "logps/chosen": -504.0,
      "logps/rejected": -616.0,
      "loss": 0.4182,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.25,
      "step": 7900
    },
    {
      "epoch": 0.2944296588561538,
      "grad_norm": 9.225704005255015,
      "learning_rate": 4.446043196517577e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.703125,
      "logps/chosen": -464.0,
      "logps/rejected": -572.0,
      "loss": 0.4397,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.15625,
      "rewards/rejected": -3.859375,
      "step": 7910
    },
    {
      "epoch": 0.2948018834564776,
      "grad_norm": 9.249579456061367,
      "learning_rate": 4.444002380287143e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -482.0,
      "logps/rejected": -588.0,
      "loss": 0.4295,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -4.03125,
      "step": 7920
    },
    {
      "epoch": 0.2951741080568015,
      "grad_norm": 9.948610964490316,
      "learning_rate": 4.441958281927075e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.578125,
      "logps/chosen": -488.0,
      "logps/rejected": -596.0,
      "loss": 0.427,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.1328125,
      "rewards/rejected": -3.890625,
      "step": 7930
    },
    {
      "epoch": 0.2955463326571253,
      "grad_norm": 9.542757933316402,
      "learning_rate": 4.4399109048885006e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.8125,
      "logps/chosen": -498.0,
      "logps/rejected": -616.0,
      "loss": 0.4355,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -4.3125,
      "step": 7940
    },
    {
      "epoch": 0.29591855725744914,
      "grad_norm": 11.181562954986296,
      "learning_rate": 4.437860252628081e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.75,
      "logps/chosen": -476.0,
      "logps/rejected": -584.0,
      "loss": 0.4081,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.265625,
      "rewards/rejected": -3.96875,
      "step": 7950
    },
    {
      "epoch": 0.29629078185777297,
      "grad_norm": 10.77257374268559,
      "learning_rate": 4.435806328608004e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.59375,
      "logps/chosen": -484.0,
      "logps/rejected": -612.0,
      "loss": 0.4159,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.125,
      "step": 7960
    },
    {
      "epoch": 0.2966630064580968,
      "grad_norm": 10.238547137760142,
      "learning_rate": 4.4337491362959854e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.65625,
      "logps/chosen": -476.0,
      "logps/rejected": -620.0,
      "loss": 0.4084,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -4.40625,
      "step": 7970
    },
    {
      "epoch": 0.29703523105842067,
      "grad_norm": 11.491308847801792,
      "learning_rate": 4.4316886791652584e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.71875,
      "logps/chosen": -500.0,
      "logps/rejected": -608.0,
      "loss": 0.4173,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -4.0625,
      "step": 7980
    },
    {
      "epoch": 0.2974074556587445,
      "grad_norm": 11.325926858080472,
      "learning_rate": 4.429624960694566e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.78125,
      "logps/chosen": -488.0,
      "logps/rejected": -620.0,
      "loss": 0.436,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.421875,
      "rewards/rejected": -4.1875,
      "step": 7990
    },
    {
      "epoch": 0.2977796802590683,
      "grad_norm": 10.566281429882185,
      "learning_rate": 4.42755798436816e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.59375,
      "logps/chosen": -470.0,
      "logps/rejected": -576.0,
      "loss": 0.4181,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.3125,
      "rewards/rejected": -3.953125,
      "step": 8000
    },
    {
      "epoch": 0.29815190485939214,
      "grad_norm": 8.934541551933057,
      "learning_rate": 4.42548775367579e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.53125,
      "logps/chosen": -452.0,
      "logps/rejected": -584.0,
      "loss": 0.407,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.515625,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.0,
      "step": 8010
    },
    {
      "epoch": 0.298524129459716,
      "grad_norm": 11.312319431068998,
      "learning_rate": 4.4234142721127026e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.8125,
      "logps/chosen": -488.0,
      "logps/rejected": -624.0,
      "loss": 0.4207,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.28125,
      "step": 8020
    },
    {
      "epoch": 0.29889635406003984,
      "grad_norm": 8.588099793248118,
      "learning_rate": 4.4213375431796316e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.6875,
      "logps/chosen": -456.0,
      "logps/rejected": -580.0,
      "loss": 0.4027,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.0625,
      "step": 8030
    },
    {
      "epoch": 0.29926857866036366,
      "grad_norm": 11.109176478049163,
      "learning_rate": 4.419257570382793e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.671875,
      "logps/chosen": -474.0,
      "logps/rejected": -620.0,
      "loss": 0.4319,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.375,
      "step": 8040
    },
    {
      "epoch": 0.2996408032606875,
      "grad_norm": 9.893467655840137,
      "learning_rate": 4.4171743572338813e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.75,
      "logps/chosen": -510.0,
      "logps/rejected": -612.0,
      "loss": 0.4497,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.203125,
      "rewards/rejected": -4.15625,
      "step": 8050
    },
    {
      "epoch": 0.30001302786101136,
      "grad_norm": 9.040783265563508,
      "learning_rate": 4.4150879072500604e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.78125,
      "logps/chosen": -476.0,
      "logps/rejected": -588.0,
      "loss": 0.4035,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.625,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -3.953125,
      "step": 8060
    },
    {
      "epoch": 0.3003852524613352,
      "grad_norm": 8.69534002128894,
      "learning_rate": 4.4129982239539594e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.765625,
      "logps/chosen": -498.0,
      "logps/rejected": -608.0,
      "loss": 0.4269,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.15625,
      "step": 8070
    },
    {
      "epoch": 0.300757477061659,
      "grad_norm": 9.219787803005481,
      "learning_rate": 4.410905310873665e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.734375,
      "logps/chosen": -454.0,
      "logps/rejected": -556.0,
      "loss": 0.449,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.5625,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -3.765625,
      "step": 8080
    },
    {
      "epoch": 0.30112970166198283,
      "grad_norm": 9.925145447229985,
      "learning_rate": 4.40880917154272e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -504.0,
      "logps/rejected": -640.0,
      "loss": 0.4149,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.125,
      "step": 8090
    },
    {
      "epoch": 0.30150192626230665,
      "grad_norm": 12.879118849159674,
      "learning_rate": 4.4067098095001113e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.734375,
      "logps/chosen": -472.0,
      "logps/rejected": -592.0,
      "loss": 0.4365,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.328125,
      "rewards/rejected": -4.125,
      "step": 8100
    },
    {
      "epoch": 0.30187415086263053,
      "grad_norm": 9.219043135070907,
      "learning_rate": 4.4046072282902687e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.65625,
      "logps/chosen": -448.0,
      "logps/rejected": -580.0,
      "loss": 0.4271,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.40625,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -3.890625,
      "step": 8110
    },
    {
      "epoch": 0.30224637546295435,
      "grad_norm": 8.647665171578879,
      "learning_rate": 4.402501431463055e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.71875,
      "logps/chosen": -480.0,
      "logps/rejected": -564.0,
      "loss": 0.4461,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.578125,
      "rewards/margins": 1.1796875,
      "rewards/rejected": -3.765625,
      "step": 8120
    },
    {
      "epoch": 0.3026186000632782,
      "grad_norm": 10.971964645043624,
      "learning_rate": 4.4003924225737643e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.6875,
      "logps/chosen": -404.0,
      "logps/rejected": -568.0,
      "loss": 0.4137,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.171875,
      "rewards/margins": 1.671875,
      "rewards/rejected": -3.84375,
      "step": 8130
    },
    {
      "epoch": 0.302990824663602,
      "grad_norm": 9.906048410852744,
      "learning_rate": 4.3982802051831127e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.734375,
      "logps/chosen": -488.0,
      "logps/rejected": -612.0,
      "loss": 0.4021,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.15625,
      "step": 8140
    },
    {
      "epoch": 0.3033630492639259,
      "grad_norm": 10.227347410967264,
      "learning_rate": 4.396164782857232e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.59375,
      "logps/chosen": -500.0,
      "logps/rejected": -648.0,
      "loss": 0.419,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.5,
      "step": 8150
    },
    {
      "epoch": 0.3037352738642497,
      "grad_norm": 8.713554085241258,
      "learning_rate": 4.3940461591676683e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.625,
      "logps/chosen": -524.0,
      "logps/rejected": -632.0,
      "loss": 0.4069,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.25,
      "step": 8160
    },
    {
      "epoch": 0.3041074984645735,
      "grad_norm": 9.459182302839464,
      "learning_rate": 4.391924337691368e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.734375,
      "logps/chosen": -492.0,
      "logps/rejected": -616.0,
      "loss": 0.4046,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -4.21875,
      "step": 8170
    },
    {
      "epoch": 0.30447972306489735,
      "grad_norm": 12.182930880811446,
      "learning_rate": 4.3897993220106825e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.625,
      "logps/chosen": -500.0,
      "logps/rejected": -648.0,
      "loss": 0.4151,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.46875,
      "step": 8180
    },
    {
      "epoch": 0.30485194766522117,
      "grad_norm": 9.51262957439446,
      "learning_rate": 4.3876711157133506e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.640625,
      "logps/chosen": -480.0,
      "logps/rejected": -584.0,
      "loss": 0.4183,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.09375,
      "step": 8190
    },
    {
      "epoch": 0.30522417226554505,
      "grad_norm": 8.837068186403384,
      "learning_rate": 4.385539722392501e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5,
      "logps/chosen": -498.0,
      "logps/rejected": -600.0,
      "loss": 0.4178,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -4.03125,
      "step": 8200
    },
    {
      "epoch": 0.30559639686586887,
      "grad_norm": 10.261721166186682,
      "learning_rate": 4.3834051456466414e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.734375,
      "logps/chosen": -498.0,
      "logps/rejected": -596.0,
      "loss": 0.4286,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -4.0,
      "step": 8210
    },
    {
      "epoch": 0.3059686214661927,
      "grad_norm": 9.903401456310172,
      "learning_rate": 4.381267389079656e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.390625,
      "logps/chosen": -452.0,
      "logps/rejected": -572.0,
      "loss": 0.4203,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.0625,
      "step": 8220
    },
    {
      "epoch": 0.3063408460665165,
      "grad_norm": 9.431580767031763,
      "learning_rate": 4.379126456300796e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.6875,
      "logps/chosen": -466.0,
      "logps/rejected": -604.0,
      "loss": 0.4226,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.125,
      "step": 8230
    },
    {
      "epoch": 0.3067130706668404,
      "grad_norm": 9.023837081235296,
      "learning_rate": 4.3769823509246753e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.484375,
      "logps/chosen": -468.0,
      "logps/rejected": -576.0,
      "loss": 0.4392,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.59375,
      "rewards/margins": 1.328125,
      "rewards/rejected": -3.90625,
      "step": 8240
    },
    {
      "epoch": 0.3070852952671642,
      "grad_norm": 10.93609638015393,
      "learning_rate": 4.374835076571265e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.484375,
      "logps/chosen": -474.0,
      "logps/rejected": -604.0,
      "loss": 0.4306,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.578125,
      "rewards/margins": 1.28125,
      "rewards/rejected": -3.859375,
      "step": 8250
    },
    {
      "epoch": 0.30745751986748804,
      "grad_norm": 11.892539636702104,
      "learning_rate": 4.3726846368658874e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.75,
      "logps/chosen": -492.0,
      "logps/rejected": -604.0,
      "loss": 0.4249,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -4.125,
      "step": 8260
    },
    {
      "epoch": 0.30782974446781186,
      "grad_norm": 12.709713506151513,
      "learning_rate": 4.370531035439206e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.609375,
      "logps/chosen": -456.0,
      "logps/rejected": -576.0,
      "loss": 0.4454,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.359375,
      "rewards/rejected": -3.90625,
      "step": 8270
    },
    {
      "epoch": 0.30820196906813574,
      "grad_norm": 8.748207370578662,
      "learning_rate": 4.3683742759272255e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5625,
      "logps/chosen": -454.0,
      "logps/rejected": -572.0,
      "loss": 0.4152,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.515625,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -3.84375,
      "step": 8280
    },
    {
      "epoch": 0.30857419366845956,
      "grad_norm": 8.22987943774933,
      "learning_rate": 4.36621436197128e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.6875,
      "logps/chosen": -478.0,
      "logps/rejected": -592.0,
      "loss": 0.4179,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.171875,
      "rewards/rejected": -3.984375,
      "step": 8290
    },
    {
      "epoch": 0.3089464182687834,
      "grad_norm": 9.60633182078992,
      "learning_rate": 4.364051297218031e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -490.0,
      "logps/rejected": -648.0,
      "loss": 0.4329,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.640625,
      "rewards/rejected": -4.5,
      "step": 8300
    },
    {
      "epoch": 0.3093186428691072,
      "grad_norm": 10.847058488148798,
      "learning_rate": 4.361885085319459e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -486.0,
      "logps/rejected": -588.0,
      "loss": 0.4471,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -4.0625,
      "step": 8310
    },
    {
      "epoch": 0.30969086746943103,
      "grad_norm": 9.766138272084318,
      "learning_rate": 4.359715729932858e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.609375,
      "logps/chosen": -494.0,
      "logps/rejected": -584.0,
      "loss": 0.4301,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.171875,
      "rewards/rejected": -4.0625,
      "step": 8320
    },
    {
      "epoch": 0.3100630920697549,
      "grad_norm": 13.607386909180434,
      "learning_rate": 4.357543234720829e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -474.0,
      "logps/rejected": -612.0,
      "loss": 0.4411,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.65625,
      "rewards/margins": 1.296875,
      "rewards/rejected": -3.953125,
      "step": 8330
    },
    {
      "epoch": 0.31043531667007873,
      "grad_norm": 9.236696113658688,
      "learning_rate": 4.355367603351275e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -470.0,
      "logps/rejected": -612.0,
      "loss": 0.4183,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.25,
      "step": 8340
    },
    {
      "epoch": 0.31080754127040255,
      "grad_norm": 8.735369662589203,
      "learning_rate": 4.353188839497393e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.65625,
      "logps/chosen": -484.0,
      "logps/rejected": -600.0,
      "loss": 0.4103,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.25,
      "rewards/rejected": -4.0,
      "step": 8350
    },
    {
      "epoch": 0.3111797658707264,
      "grad_norm": 12.332914018418103,
      "learning_rate": 4.3510069468376687e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.75,
      "logps/chosen": -492.0,
      "logps/rejected": -620.0,
      "loss": 0.4245,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.3125,
      "step": 8360
    },
    {
      "epoch": 0.31155199047105026,
      "grad_norm": 12.612183136126026,
      "learning_rate": 4.34882192905587e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.671875,
      "logps/chosen": -486.0,
      "logps/rejected": -592.0,
      "loss": 0.4129,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.28125,
      "rewards/rejected": -4.15625,
      "step": 8370
    },
    {
      "epoch": 0.3119242150713741,
      "grad_norm": 10.456520951305427,
      "learning_rate": 4.3466337898410435e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.703125,
      "logps/chosen": -510.0,
      "logps/rejected": -612.0,
      "loss": 0.4215,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -4.1875,
      "step": 8380
    },
    {
      "epoch": 0.3122964396716979,
      "grad_norm": 10.304233950987694,
      "learning_rate": 4.3444425328875013e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.6875,
      "logps/chosen": -498.0,
      "logps/rejected": -640.0,
      "loss": 0.4256,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -4.53125,
      "step": 8390
    },
    {
      "epoch": 0.3126686642720217,
      "grad_norm": 10.267510096352085,
      "learning_rate": 4.3422481618948236e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.875,
      "logps/chosen": -486.0,
      "logps/rejected": -600.0,
      "loss": 0.4264,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.265625,
      "rewards/rejected": -4.15625,
      "step": 8400
    },
    {
      "epoch": 0.31304088887234555,
      "grad_norm": 8.403692242400545,
      "learning_rate": 4.340050680567846e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.8125,
      "logps/chosen": -496.0,
      "logps/rejected": -616.0,
      "loss": 0.4301,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.34375,
      "step": 8410
    },
    {
      "epoch": 0.3134131134726694,
      "grad_norm": 10.95703015221167,
      "learning_rate": 4.337850092616656e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5,
      "logps/chosen": -470.0,
      "logps/rejected": -584.0,
      "loss": 0.4085,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.09375,
      "step": 8420
    },
    {
      "epoch": 0.31378533807299325,
      "grad_norm": 9.308785530354855,
      "learning_rate": 4.3356464017565856e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.8125,
      "logps/chosen": -498.0,
      "logps/rejected": -596.0,
      "loss": 0.4395,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -4.0625,
      "step": 8430
    },
    {
      "epoch": 0.31415756267331707,
      "grad_norm": 8.67135159177148,
      "learning_rate": 4.333439611708206e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.578125,
      "logps/chosen": -472.0,
      "logps/rejected": -644.0,
      "loss": 0.4096,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -4.5,
      "step": 8440
    },
    {
      "epoch": 0.3145297872736409,
      "grad_norm": 11.064469728532885,
      "learning_rate": 4.3312297261973206e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.8125,
      "logps/chosen": -524.0,
      "logps/rejected": -648.0,
      "loss": 0.409,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.359375,
      "rewards/rejected": -4.40625,
      "step": 8450
    },
    {
      "epoch": 0.31490201187396477,
      "grad_norm": 9.55077719870887,
      "learning_rate": 4.32901674895496e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.71875,
      "logps/chosen": -466.0,
      "logps/rejected": -596.0,
      "loss": 0.4291,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.546875,
      "rewards/rejected": -4.1875,
      "step": 8460
    },
    {
      "epoch": 0.3152742364742886,
      "grad_norm": 10.251781192728178,
      "learning_rate": 4.326800683717373e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -486.0,
      "logps/rejected": -640.0,
      "loss": 0.4334,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.28125,
      "step": 8470
    },
    {
      "epoch": 0.3156464610746124,
      "grad_norm": 7.423660923092191,
      "learning_rate": 4.324581534226023e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.671875,
      "logps/chosen": -520.0,
      "logps/rejected": -616.0,
      "loss": 0.4328,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.015625,
      "rewards/margins": 1.203125,
      "rewards/rejected": -4.21875,
      "step": 8480
    },
    {
      "epoch": 0.31601868567493624,
      "grad_norm": 9.398267279681146,
      "learning_rate": 4.32235930422758e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.8125,
      "logps/chosen": -492.0,
      "logps/rejected": -600.0,
      "loss": 0.4035,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.1875,
      "rewards/rejected": -3.9375,
      "step": 8490
    },
    {
      "epoch": 0.3163909102752601,
      "grad_norm": 8.654274276064998,
      "learning_rate": 4.3201339974739165e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.703125,
      "logps/chosen": -492.0,
      "logps/rejected": -588.0,
      "loss": 0.4383,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.1875,
      "rewards/rejected": -4.0,
      "step": 8500
    },
    {
      "epoch": 0.31676313487558394,
      "grad_norm": 11.375780937890697,
      "learning_rate": 4.3179056177220976e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.84375,
      "logps/chosen": -490.0,
      "logps/rejected": -616.0,
      "loss": 0.4386,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.3125,
      "step": 8510
    },
    {
      "epoch": 0.31713535947590776,
      "grad_norm": 10.879311463238736,
      "learning_rate": 4.3156741687343776e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.75,
      "logps/chosen": -480.0,
      "logps/rejected": -600.0,
      "loss": 0.4543,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.203125,
      "rewards/rejected": -3.984375,
      "step": 8520
    },
    {
      "epoch": 0.3175075840762316,
      "grad_norm": 8.992917625937125,
      "learning_rate": 4.313439654278194e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.71875,
      "logps/chosen": -494.0,
      "logps/rejected": -584.0,
      "loss": 0.4251,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.15625,
      "rewards/rejected": -3.84375,
      "step": 8530
    },
    {
      "epoch": 0.3178798086765554,
      "grad_norm": 8.8744130832388,
      "learning_rate": 4.311202078126156e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.609375,
      "logps/chosen": -512.0,
      "logps/rejected": -608.0,
      "loss": 0.4419,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.28125,
      "rewards/rejected": -4.03125,
      "step": 8540
    },
    {
      "epoch": 0.3182520332768793,
      "grad_norm": 8.580306870258779,
      "learning_rate": 4.308961444056046e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.671875,
      "logps/chosen": -482.0,
      "logps/rejected": -612.0,
      "loss": 0.4041,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -4.09375,
      "step": 8550
    },
    {
      "epoch": 0.3186242578772031,
      "grad_norm": 9.778521158485606,
      "learning_rate": 4.306717755850808e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.65625,
      "logps/chosen": -500.0,
      "logps/rejected": -640.0,
      "loss": 0.4363,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.4375,
      "step": 8560
    },
    {
      "epoch": 0.31899648247752693,
      "grad_norm": 8.345892689429391,
      "learning_rate": 4.304471017298542e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -510.0,
      "logps/rejected": -604.0,
      "loss": 0.4178,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.203125,
      "rewards/rejected": -4.09375,
      "step": 8570
    },
    {
      "epoch": 0.31936870707785076,
      "grad_norm": 9.323516309983708,
      "learning_rate": 4.302221232192497e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.796875,
      "logps/chosen": -494.0,
      "logps/rejected": -620.0,
      "loss": 0.4401,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.25,
      "step": 8580
    },
    {
      "epoch": 0.31974093167817463,
      "grad_norm": 8.800592120454882,
      "learning_rate": 4.2999684043310667e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.734375,
      "logps/chosen": -486.0,
      "logps/rejected": -592.0,
      "loss": 0.4015,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.609375,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -3.96875,
      "step": 8590
    },
    {
      "epoch": 0.32011315627849846,
      "grad_norm": 9.561115191553332,
      "learning_rate": 4.297712537517784e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.71875,
      "logps/chosen": -474.0,
      "logps/rejected": -616.0,
      "loss": 0.3875,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.65625,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.03125,
      "step": 8600
    },
    {
      "epoch": 0.3204853808788223,
      "grad_norm": 11.788894485934472,
      "learning_rate": 4.295453635561308e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.734375,
      "logps/chosen": -520.0,
      "logps/rejected": -640.0,
      "loss": 0.4417,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -4.5,
      "step": 8610
    },
    {
      "epoch": 0.3208576054791461,
      "grad_norm": 9.564191779991367,
      "learning_rate": 4.293191702275426e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.640625,
      "logps/chosen": -508.0,
      "logps/rejected": -616.0,
      "loss": 0.4124,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -4.25,
      "step": 8620
    },
    {
      "epoch": 0.3212298300794699,
      "grad_norm": 9.46590730353525,
      "learning_rate": 4.290926741479043e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.71875,
      "logps/chosen": -508.0,
      "logps/rejected": -604.0,
      "loss": 0.4339,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.328125,
      "rewards/rejected": -4.09375,
      "step": 8630
    },
    {
      "epoch": 0.3216020546797938,
      "grad_norm": 11.190278023651699,
      "learning_rate": 4.288658756996172e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.75,
      "logps/chosen": -496.0,
      "logps/rejected": -624.0,
      "loss": 0.4234,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -4.21875,
      "step": 8640
    },
    {
      "epoch": 0.3219742792801176,
      "grad_norm": 10.190781465513247,
      "learning_rate": 4.2863877526559344e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.71875,
      "logps/chosen": -528.0,
      "logps/rejected": -636.0,
      "loss": 0.4153,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.40625,
      "step": 8650
    },
    {
      "epoch": 0.32234650388044145,
      "grad_norm": 9.818176515917079,
      "learning_rate": 4.2841137322925493e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.703125,
      "logps/chosen": -480.0,
      "logps/rejected": -616.0,
      "loss": 0.4031,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.34375,
      "step": 8660
    },
    {
      "epoch": 0.32271872848076527,
      "grad_norm": 9.913190617990658,
      "learning_rate": 4.281836699745327e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.703125,
      "logps/chosen": -510.0,
      "logps/rejected": -640.0,
      "loss": 0.4248,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.5625,
      "step": 8670
    },
    {
      "epoch": 0.32309095308108915,
      "grad_norm": 9.393758122926913,
      "learning_rate": 4.279556658858665e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.734375,
      "logps/chosen": -496.0,
      "logps/rejected": -612.0,
      "loss": 0.4366,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.3125,
      "rewards/rejected": -4.125,
      "step": 8680
    },
    {
      "epoch": 0.32346317768141297,
      "grad_norm": 9.937420547158876,
      "learning_rate": 4.2772736134820385e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.796875,
      "logps/chosen": -524.0,
      "logps/rejected": -644.0,
      "loss": 0.4207,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.4375,
      "step": 8690
    },
    {
      "epoch": 0.3238354022817368,
      "grad_norm": 11.71519952342745,
      "learning_rate": 4.2749875674699954e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.75,
      "logps/chosen": -516.0,
      "logps/rejected": -656.0,
      "loss": 0.4317,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.46875,
      "step": 8700
    },
    {
      "epoch": 0.3242076268820606,
      "grad_norm": 10.876904218100155,
      "learning_rate": 4.2726985246821507e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.734375,
      "logps/chosen": -494.0,
      "logps/rejected": -640.0,
      "loss": 0.4092,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.28125,
      "step": 8710
    },
    {
      "epoch": 0.3245798514823845,
      "grad_norm": 9.181298901831003,
      "learning_rate": 4.270406488983177e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.65625,
      "logps/chosen": -478.0,
      "logps/rejected": -604.0,
      "loss": 0.4335,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.1875,
      "step": 8720
    },
    {
      "epoch": 0.3249520760827083,
      "grad_norm": 10.789278632579261,
      "learning_rate": 4.268111464242803e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.734375,
      "logps/chosen": -484.0,
      "logps/rejected": -608.0,
      "loss": 0.4197,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.671875,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.0625,
      "step": 8730
    },
    {
      "epoch": 0.32532430068303214,
      "grad_norm": 10.187560721532044,
      "learning_rate": 4.2658134543358e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.703125,
      "logps/chosen": -496.0,
      "logps/rejected": -632.0,
      "loss": 0.412,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -4.25,
      "step": 8740
    },
    {
      "epoch": 0.32569652528335596,
      "grad_norm": 9.622870249632156,
      "learning_rate": 4.2635124631419827e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.765625,
      "logps/chosen": -484.0,
      "logps/rejected": -584.0,
      "loss": 0.4063,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -4.0625,
      "step": 8750
    },
    {
      "epoch": 0.3260687498836798,
      "grad_norm": 9.941397235293854,
      "learning_rate": 4.261208494546198e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.890625,
      "logps/chosen": -478.0,
      "logps/rejected": -616.0,
      "loss": 0.4372,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.4375,
      "step": 8760
    },
    {
      "epoch": 0.32644097448400367,
      "grad_norm": 9.60632002469737,
      "learning_rate": 4.2589015524383187e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.796875,
      "logps/chosen": -486.0,
      "logps/rejected": -592.0,
      "loss": 0.4367,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.0625,
      "step": 8770
    },
    {
      "epoch": 0.3268131990843275,
      "grad_norm": 11.111942045716804,
      "learning_rate": 4.2565916407132393e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.921875,
      "logps/chosen": -508.0,
      "logps/rejected": -604.0,
      "loss": 0.4178,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -4.09375,
      "step": 8780
    },
    {
      "epoch": 0.3271854236846513,
      "grad_norm": 10.2952702928224,
      "learning_rate": 4.254278763270867e-07,
      "logits/chosen": -3.125,
      "logits/rejected": -2.859375,
      "logps/chosen": -512.0,
      "logps/rejected": -644.0,
      "loss": 0.426,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.4375,
      "step": 8790
    },
    {
      "epoch": 0.32755764828497513,
      "grad_norm": 9.393641967241427,
      "learning_rate": 4.251962924016117e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.828125,
      "logps/chosen": -516.0,
      "logps/rejected": -652.0,
      "loss": 0.4251,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.53125,
      "step": 8800
    },
    {
      "epoch": 0.327929872885299,
      "grad_norm": 11.925111633391955,
      "learning_rate": 4.2496441268589047e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.9375,
      "logps/chosen": -532.0,
      "logps/rejected": -648.0,
      "loss": 0.4175,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.4375,
      "step": 8810
    },
    {
      "epoch": 0.32830209748562283,
      "grad_norm": 10.456797530601381,
      "learning_rate": 4.2473223757141386e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.734375,
      "logps/chosen": -524.0,
      "logps/rejected": -672.0,
      "loss": 0.4045,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -4.625,
      "step": 8820
    },
    {
      "epoch": 0.32867432208594666,
      "grad_norm": 11.186238946236713,
      "learning_rate": 4.2449976745017157e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.890625,
      "logps/chosen": -506.0,
      "logps/rejected": -632.0,
      "loss": 0.4158,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.28125,
      "step": 8830
    },
    {
      "epoch": 0.3290465466862705,
      "grad_norm": 11.137530464040744,
      "learning_rate": 4.2426700271465134e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.859375,
      "logps/chosen": -520.0,
      "logps/rejected": -644.0,
      "loss": 0.4071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.46875,
      "step": 8840
    },
    {
      "epoch": 0.32941877128659436,
      "grad_norm": 10.851988902307895,
      "learning_rate": 4.240339437578383e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.734375,
      "logps/chosen": -492.0,
      "logps/rejected": -604.0,
      "loss": 0.4214,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.25,
      "step": 8850
    },
    {
      "epoch": 0.3297909958869182,
      "grad_norm": 9.906177018532707,
      "learning_rate": 4.238005909732144e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -490.0,
      "logps/rejected": -624.0,
      "loss": 0.4136,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.640625,
      "rewards/rejected": -4.375,
      "step": 8860
    },
    {
      "epoch": 0.330163220487242,
      "grad_norm": 9.46639187253771,
      "learning_rate": 4.235669447547574e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.703125,
      "logps/chosen": -484.0,
      "logps/rejected": -632.0,
      "loss": 0.4234,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.4375,
      "step": 8870
    },
    {
      "epoch": 0.3305354450875658,
      "grad_norm": 7.952031719444195,
      "learning_rate": 4.2333300549694085e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.609375,
      "logps/chosen": -484.0,
      "logps/rejected": -592.0,
      "loss": 0.4275,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -4.3125,
      "step": 8880
    },
    {
      "epoch": 0.33090766968788965,
      "grad_norm": 7.739138456320119,
      "learning_rate": 4.2309877359473277e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.640625,
      "logps/chosen": -458.0,
      "logps/rejected": -556.0,
      "loss": 0.4413,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.15625,
      "rewards/rejected": -3.921875,
      "step": 8890
    },
    {
      "epoch": 0.33127989428821353,
      "grad_norm": 10.620065507498312,
      "learning_rate": 4.2286424944359547e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.75,
      "logps/chosen": -486.0,
      "logps/rejected": -584.0,
      "loss": 0.4237,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.0625,
      "step": 8900
    },
    {
      "epoch": 0.33165211888853735,
      "grad_norm": 10.42429763504476,
      "learning_rate": 4.2262943343948445e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.703125,
      "logps/chosen": -486.0,
      "logps/rejected": -616.0,
      "loss": 0.4168,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -4.0625,
      "step": 8910
    },
    {
      "epoch": 0.3320243434888612,
      "grad_norm": 10.027261386541722,
      "learning_rate": 4.223943259788481e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -502.0,
      "logps/rejected": -640.0,
      "loss": 0.4162,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.28125,
      "step": 8920
    },
    {
      "epoch": 0.332396568089185,
      "grad_norm": 9.268786690947321,
      "learning_rate": 4.22158927458627e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.515625,
      "logps/chosen": -480.0,
      "logps/rejected": -612.0,
      "loss": 0.433,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.125,
      "step": 8930
    },
    {
      "epoch": 0.3327687926895089,
      "grad_norm": 9.329010386669555,
      "learning_rate": 4.219232382762528e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.609375,
      "logps/chosen": -472.0,
      "logps/rejected": -600.0,
      "loss": 0.4232,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.359375,
      "rewards/rejected": -4.0,
      "step": 8940
    },
    {
      "epoch": 0.3331410172898327,
      "grad_norm": 9.657807936704513,
      "learning_rate": 4.2168725882964825e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.6875,
      "logps/chosen": -458.0,
      "logps/rejected": -584.0,
      "loss": 0.422,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.59375,
      "rewards/margins": 1.328125,
      "rewards/rejected": -3.90625,
      "step": 8950
    },
    {
      "epoch": 0.3335132418901565,
      "grad_norm": 10.566732091134313,
      "learning_rate": 4.2145098951722584e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.65625,
      "logps/chosen": -496.0,
      "logps/rejected": -644.0,
      "loss": 0.4296,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.46875,
      "step": 8960
    },
    {
      "epoch": 0.33388546649048034,
      "grad_norm": 9.667981193885012,
      "learning_rate": 4.2121443073788775e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.6875,
      "logps/chosen": -482.0,
      "logps/rejected": -624.0,
      "loss": 0.4139,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.25,
      "step": 8970
    },
    {
      "epoch": 0.33425769109080417,
      "grad_norm": 10.643665461966387,
      "learning_rate": 4.209775828910247e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.53125,
      "logps/chosen": -494.0,
      "logps/rejected": -628.0,
      "loss": 0.4216,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.1875,
      "step": 8980
    },
    {
      "epoch": 0.33462991569112804,
      "grad_norm": 9.139035694387253,
      "learning_rate": 4.207404463765154e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.75,
      "logps/chosen": -474.0,
      "logps/rejected": -608.0,
      "loss": 0.4162,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.28125,
      "step": 8990
    },
    {
      "epoch": 0.33500214029145187,
      "grad_norm": 8.433481790453827,
      "learning_rate": 4.205030215947261e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.53125,
      "logps/chosen": -496.0,
      "logps/rejected": -608.0,
      "loss": 0.4329,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.3125,
      "rewards/rejected": -4.15625,
      "step": 9000
    },
    {
      "epoch": 0.3353743648917757,
      "grad_norm": 8.937895696575707,
      "learning_rate": 4.202653089465097e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.59375,
      "logps/chosen": -464.0,
      "logps/rejected": -608.0,
      "loss": 0.4193,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.03125,
      "step": 9010
    },
    {
      "epoch": 0.3357465894920995,
      "grad_norm": 9.94230385747298,
      "learning_rate": 4.2002730883320493e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.78125,
      "logps/chosen": -478.0,
      "logps/rejected": -620.0,
      "loss": 0.4216,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.625,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.1875,
      "step": 9020
    },
    {
      "epoch": 0.3361188140924234,
      "grad_norm": 13.358663082187785,
      "learning_rate": 4.1978902165663616e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.765625,
      "logps/chosen": -484.0,
      "logps/rejected": -604.0,
      "loss": 0.4315,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.15625,
      "step": 9030
    },
    {
      "epoch": 0.3364910386927472,
      "grad_norm": 9.447000230298327,
      "learning_rate": 4.1955044781911215e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.8125,
      "logps/chosen": -490.0,
      "logps/rejected": -624.0,
      "loss": 0.4533,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -4.15625,
      "step": 9040
    },
    {
      "epoch": 0.33686326329307104,
      "grad_norm": 10.87663364658343,
      "learning_rate": 4.193115877234258e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.65625,
      "logps/chosen": -452.0,
      "logps/rejected": -552.0,
      "loss": 0.4233,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.53125,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -3.640625,
      "step": 9050
    },
    {
      "epoch": 0.33723548789339486,
      "grad_norm": 10.170260289333275,
      "learning_rate": 4.1907244177285326e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.8125,
      "logps/chosen": -482.0,
      "logps/rejected": -584.0,
      "loss": 0.4327,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.203125,
      "rewards/rejected": -4.09375,
      "step": 9060
    },
    {
      "epoch": 0.33760771249371874,
      "grad_norm": 9.454348045379046,
      "learning_rate": 4.188330103711533e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.828125,
      "logps/chosen": -482.0,
      "logps/rejected": -612.0,
      "loss": 0.4225,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.21875,
      "step": 9070
    },
    {
      "epoch": 0.33797993709404256,
      "grad_norm": 11.274926082258306,
      "learning_rate": 4.185932939225666e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.71875,
      "logps/chosen": -512.0,
      "logps/rejected": -628.0,
      "loss": 0.4298,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.015625,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -4.21875,
      "step": 9080
    },
    {
      "epoch": 0.3383521616943664,
      "grad_norm": 10.00938351224194,
      "learning_rate": 4.1835329283181506e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -504.0,
      "logps/rejected": -624.0,
      "loss": 0.4076,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -4.15625,
      "step": 9090
    },
    {
      "epoch": 0.3387243862946902,
      "grad_norm": 13.652601287808766,
      "learning_rate": 4.1811300750410137e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.71875,
      "logps/chosen": -494.0,
      "logps/rejected": -604.0,
      "loss": 0.429,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.359375,
      "rewards/rejected": -4.3125,
      "step": 9100
    },
    {
      "epoch": 0.33909661089501403,
      "grad_norm": 9.21923985937886,
      "learning_rate": 4.1787243834510793e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.671875,
      "logps/chosen": -504.0,
      "logps/rejected": -616.0,
      "loss": 0.4224,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.21875,
      "step": 9110
    },
    {
      "epoch": 0.3394688354953379,
      "grad_norm": 12.382296643888058,
      "learning_rate": 4.176315857609963e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.921875,
      "logps/chosen": -556.0,
      "logps/rejected": -624.0,
      "loss": 0.4277,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.109375,
      "rewards/rejected": -4.15625,
      "step": 9120
    },
    {
      "epoch": 0.33984106009566173,
      "grad_norm": 10.731655871923943,
      "learning_rate": 4.173904501584066e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.84375,
      "logps/chosen": -524.0,
      "logps/rejected": -644.0,
      "loss": 0.434,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.65625,
      "step": 9130
    },
    {
      "epoch": 0.34021328469598555,
      "grad_norm": 12.856096233429325,
      "learning_rate": 4.1714903194445686e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.625,
      "logps/chosen": -532.0,
      "logps/rejected": -660.0,
      "loss": 0.4256,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -4.46875,
      "step": 9140
    },
    {
      "epoch": 0.3405855092963094,
      "grad_norm": 10.109646999998054,
      "learning_rate": 4.169073315267421e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.921875,
      "logps/chosen": -516.0,
      "logps/rejected": -592.0,
      "loss": 0.425,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.1796875,
      "rewards/rejected": -4.15625,
      "step": 9150
    },
    {
      "epoch": 0.34095773389663325,
      "grad_norm": 11.01269735009484,
      "learning_rate": 4.1666534931333406e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.765625,
      "logps/chosen": -502.0,
      "logps/rejected": -596.0,
      "loss": 0.4417,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -4.0,
      "step": 9160
    },
    {
      "epoch": 0.3413299584969571,
      "grad_norm": 11.339184119526918,
      "learning_rate": 4.1642308571277996e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.71875,
      "logps/chosen": -478.0,
      "logps/rejected": -600.0,
      "loss": 0.4211,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.125,
      "step": 9170
    },
    {
      "epoch": 0.3417021830972809,
      "grad_norm": 9.755662209893702,
      "learning_rate": 4.1618054113410217e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.6875,
      "logps/chosen": -492.0,
      "logps/rejected": -616.0,
      "loss": 0.4049,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -4.125,
      "step": 9180
    },
    {
      "epoch": 0.3420744076976047,
      "grad_norm": 11.41742746921289,
      "learning_rate": 4.159377159867976e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.71875,
      "logps/chosen": -474.0,
      "logps/rejected": -620.0,
      "loss": 0.4341,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.40625,
      "step": 9190
    },
    {
      "epoch": 0.34244663229792854,
      "grad_norm": 10.524215089549886,
      "learning_rate": 4.1569461068083664e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.703125,
      "logps/chosen": -510.0,
      "logps/rejected": -636.0,
      "loss": 0.414,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.359375,
      "rewards/rejected": -4.3125,
      "step": 9200
    },
    {
      "epoch": 0.3428188568982524,
      "grad_norm": 9.25896102637816,
      "learning_rate": 4.154512256266629e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.90625,
      "logps/chosen": -478.0,
      "logps/rejected": -592.0,
      "loss": 0.4235,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -3.9375,
      "step": 9210
    },
    {
      "epoch": 0.34319108149857624,
      "grad_norm": 10.084130724841001,
      "learning_rate": 4.152075612351921e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.828125,
      "logps/chosen": -488.0,
      "logps/rejected": -604.0,
      "loss": 0.4011,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.421875,
      "rewards/rejected": -4.15625,
      "step": 9220
    },
    {
      "epoch": 0.34356330609890007,
      "grad_norm": 9.220418241999557,
      "learning_rate": 4.149636179178117e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.78125,
      "logps/chosen": -516.0,
      "logps/rejected": -616.0,
      "loss": 0.4274,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -4.28125,
      "step": 9230
    },
    {
      "epoch": 0.3439355306992239,
      "grad_norm": 12.082960818947464,
      "learning_rate": 4.1471939608637997e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.84375,
      "logps/chosen": -516.0,
      "logps/rejected": -664.0,
      "loss": 0.4466,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.59375,
      "step": 9240
    },
    {
      "epoch": 0.34430775529954777,
      "grad_norm": 9.401034919769838,
      "learning_rate": 4.144748961532255e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.8125,
      "logps/chosen": -484.0,
      "logps/rejected": -612.0,
      "loss": 0.4095,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.15625,
      "step": 9250
    },
    {
      "epoch": 0.3446799798998716,
      "grad_norm": 9.389155029668613,
      "learning_rate": 4.1423011853114644e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.890625,
      "logps/chosen": -488.0,
      "logps/rejected": -596.0,
      "loss": 0.4234,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.359375,
      "rewards/rejected": -4.0,
      "step": 9260
    },
    {
      "epoch": 0.3450522045001954,
      "grad_norm": 9.32994978945283,
      "learning_rate": 4.1398506363340965e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.796875,
      "logps/chosen": -492.0,
      "logps/rejected": -620.0,
      "loss": 0.4209,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.25,
      "step": 9270
    },
    {
      "epoch": 0.34542442910051924,
      "grad_norm": 11.03494491753943,
      "learning_rate": 4.137397318737502e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.59375,
      "logps/chosen": -496.0,
      "logps/rejected": -624.0,
      "loss": 0.4349,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.3125,
      "step": 9280
    },
    {
      "epoch": 0.3457966537008431,
      "grad_norm": 10.26059275942047,
      "learning_rate": 4.134941236663706e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.625,
      "logps/chosen": -472.0,
      "logps/rejected": -636.0,
      "loss": 0.422,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -4.4375,
      "step": 9290
    },
    {
      "epoch": 0.34616887830116694,
      "grad_norm": 10.400805450031838,
      "learning_rate": 4.132482394259401e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.6875,
      "logps/chosen": -468.0,
      "logps/rejected": -596.0,
      "loss": 0.4176,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.09375,
      "step": 9300
    },
    {
      "epoch": 0.34654110290149076,
      "grad_norm": 10.61515306409864,
      "learning_rate": 4.1300207956759395e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.796875,
      "logps/chosen": -502.0,
      "logps/rejected": -620.0,
      "loss": 0.4283,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.015625,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -4.3125,
      "step": 9310
    },
    {
      "epoch": 0.3469133275018146,
      "grad_norm": 10.139115451930804,
      "learning_rate": 4.127556445069328e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.625,
      "logps/chosen": -458.0,
      "logps/rejected": -592.0,
      "loss": 0.4047,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.03125,
      "step": 9320
    },
    {
      "epoch": 0.3472855521021384,
      "grad_norm": 10.622762782273034,
      "learning_rate": 4.125089346600218e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.71875,
      "logps/chosen": -512.0,
      "logps/rejected": -648.0,
      "loss": 0.4123,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.53125,
      "step": 9330
    },
    {
      "epoch": 0.3476577767024623,
      "grad_norm": 11.909036980482506,
      "learning_rate": 4.122619504433902e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.734375,
      "logps/chosen": -496.0,
      "logps/rejected": -612.0,
      "loss": 0.4317,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -3.984375,
      "step": 9340
    },
    {
      "epoch": 0.3480300013027861,
      "grad_norm": 10.216990928379289,
      "learning_rate": 4.120146922740303e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.625,
      "logps/chosen": -492.0,
      "logps/rejected": -624.0,
      "loss": 0.4397,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -4.21875,
      "step": 9350
    },
    {
      "epoch": 0.34840222590310993,
      "grad_norm": 8.432050222728119,
      "learning_rate": 4.1176716056939715e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.84375,
      "logps/chosen": -536.0,
      "logps/rejected": -648.0,
      "loss": 0.4028,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -4.40625,
      "step": 9360
    },
    {
      "epoch": 0.34877445050343375,
      "grad_norm": 8.854026586558382,
      "learning_rate": 4.115193557474074e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.6875,
      "logps/chosen": -506.0,
      "logps/rejected": -636.0,
      "loss": 0.425,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.296875,
      "rewards/rejected": -4.46875,
      "step": 9370
    },
    {
      "epoch": 0.34914667510375763,
      "grad_norm": 9.414767033448554,
      "learning_rate": 4.1127127822643894e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.796875,
      "logps/chosen": -498.0,
      "logps/rejected": -636.0,
      "loss": 0.4083,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.40625,
      "step": 9380
    },
    {
      "epoch": 0.34951889970408145,
      "grad_norm": 12.592591150875066,
      "learning_rate": 4.1102292842533004e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.8125,
      "logps/chosen": -502.0,
      "logps/rejected": -648.0,
      "loss": 0.4223,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.625,
      "rewards/rejected": -4.53125,
      "step": 9390
    },
    {
      "epoch": 0.3498911243044053,
      "grad_norm": 7.787234071173059,
      "learning_rate": 4.1077430676337867e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.640625,
      "logps/chosen": -468.0,
      "logps/rejected": -616.0,
      "loss": 0.4189,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.25,
      "step": 9400
    },
    {
      "epoch": 0.3502633489047291,
      "grad_norm": 9.404626079945661,
      "learning_rate": 4.1052541366034174e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -446.0,
      "logps/rejected": -572.0,
      "loss": 0.417,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -3.984375,
      "step": 9410
    },
    {
      "epoch": 0.3506355735050529,
      "grad_norm": 11.93906985225258,
      "learning_rate": 4.102762495364346e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.859375,
      "logps/chosen": -482.0,
      "logps/rejected": -616.0,
      "loss": 0.4312,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.421875,
      "rewards/rejected": -4.3125,
      "step": 9420
    },
    {
      "epoch": 0.3510077981053768,
      "grad_norm": 9.180327531151416,
      "learning_rate": 4.100268148123299e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.734375,
      "logps/chosen": -512.0,
      "logps/rejected": -616.0,
      "loss": 0.4136,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.234375,
      "rewards/rejected": -4.125,
      "step": 9430
    },
    {
      "epoch": 0.3513800227057006,
      "grad_norm": 8.456108480687135,
      "learning_rate": 4.0977710990915747e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.71875,
      "logps/chosen": -498.0,
      "logps/rejected": -628.0,
      "loss": 0.4062,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.5625,
      "step": 9440
    },
    {
      "epoch": 0.35175224730602445,
      "grad_norm": 9.191160400061396,
      "learning_rate": 4.0952713524850313e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.703125,
      "logps/chosen": -540.0,
      "logps/rejected": -668.0,
      "loss": 0.4087,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.625,
      "step": 9450
    },
    {
      "epoch": 0.35212447190634827,
      "grad_norm": 8.902725920195198,
      "learning_rate": 4.0927689125240806e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.734375,
      "logps/chosen": -548.0,
      "logps/rejected": -672.0,
      "loss": 0.4409,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.59375,
      "step": 9460
    },
    {
      "epoch": 0.35249669650667215,
      "grad_norm": 7.408899624839923,
      "learning_rate": 4.090263783433683e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.625,
      "logps/chosen": -496.0,
      "logps/rejected": -628.0,
      "loss": 0.4451,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -4.34375,
      "step": 9470
    },
    {
      "epoch": 0.35286892110699597,
      "grad_norm": 9.216810869637532,
      "learning_rate": 4.0877559694433384e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.65625,
      "logps/chosen": -502.0,
      "logps/rejected": -616.0,
      "loss": 0.4164,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -4.28125,
      "step": 9480
    },
    {
      "epoch": 0.3532411457073198,
      "grad_norm": 10.856396925241992,
      "learning_rate": 4.0852454747870807e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.796875,
      "logps/chosen": -496.0,
      "logps/rejected": -604.0,
      "loss": 0.4485,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.1875,
      "rewards/rejected": -4.15625,
      "step": 9490
    },
    {
      "epoch": 0.3536133703076436,
      "grad_norm": 9.570705153555636,
      "learning_rate": 4.082732303703469e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.765625,
      "logps/chosen": -484.0,
      "logps/rejected": -580.0,
      "loss": 0.4232,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.09375,
      "rewards/rejected": -3.796875,
      "step": 9500
    },
    {
      "epoch": 0.3539855949079675,
      "grad_norm": 9.919881162080282,
      "learning_rate": 4.0802164604355805e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.8125,
      "logps/chosen": -508.0,
      "logps/rejected": -608.0,
      "loss": 0.4218,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.015625,
      "rewards/margins": 1.21875,
      "rewards/rejected": -4.21875,
      "step": 9510
    },
    {
      "epoch": 0.3543578195082913,
      "grad_norm": 9.628474763663824,
      "learning_rate": 4.077697949231005e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.65625,
      "logps/chosen": -482.0,
      "logps/rejected": -604.0,
      "loss": 0.4177,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.234375,
      "rewards/rejected": -4.09375,
      "step": 9520
    },
    {
      "epoch": 0.35473004410861514,
      "grad_norm": 9.814436533447635,
      "learning_rate": 4.075176774341835e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -490.0,
      "logps/rejected": -608.0,
      "loss": 0.4151,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.203125,
      "rewards/rejected": -4.09375,
      "step": 9530
    },
    {
      "epoch": 0.35510226870893896,
      "grad_norm": 10.131474055230331,
      "learning_rate": 4.0726529400246634e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.484375,
      "logps/chosen": -464.0,
      "logps/rejected": -612.0,
      "loss": 0.4245,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.3125,
      "step": 9540
    },
    {
      "epoch": 0.3554744933092628,
      "grad_norm": 10.65808121433148,
      "learning_rate": 4.070126450540569e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.625,
      "logps/chosen": -544.0,
      "logps/rejected": -660.0,
      "loss": 0.4109,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -4.5,
      "step": 9550
    },
    {
      "epoch": 0.35584671790958666,
      "grad_norm": 8.795969285846954,
      "learning_rate": 4.0675973101551177e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.515625,
      "logps/chosen": -492.0,
      "logps/rejected": -652.0,
      "loss": 0.412,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.53125,
      "step": 9560
    },
    {
      "epoch": 0.3562189425099105,
      "grad_norm": 10.103796377585969,
      "learning_rate": 4.065065523138347e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.46875,
      "logps/chosen": -484.0,
      "logps/rejected": -616.0,
      "loss": 0.4298,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.28125,
      "step": 9570
    },
    {
      "epoch": 0.3565911671102343,
      "grad_norm": 9.476361344491268,
      "learning_rate": 4.062531093764764e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.65625,
      "logps/chosen": -502.0,
      "logps/rejected": -628.0,
      "loss": 0.4298,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.25,
      "step": 9580
    },
    {
      "epoch": 0.35696339171055813,
      "grad_norm": 9.520093818350574,
      "learning_rate": 4.05999402631334e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.5,
      "logps/chosen": -528.0,
      "logps/rejected": -640.0,
      "loss": 0.4246,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -4.3125,
      "step": 9590
    },
    {
      "epoch": 0.357335616310882,
      "grad_norm": 9.342698003039136,
      "learning_rate": 4.0574543250674973e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.71875,
      "logps/chosen": -506.0,
      "logps/rejected": -628.0,
      "loss": 0.4375,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.015625,
      "rewards/margins": 1.3125,
      "rewards/rejected": -4.3125,
      "step": 9600
    },
    {
      "epoch": 0.35770784091120583,
      "grad_norm": 9.947643042733679,
      "learning_rate": 4.054911994315104e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.6875,
      "logps/chosen": -564.0,
      "logps/rejected": -676.0,
      "loss": 0.4007,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.53125,
      "step": 9610
    },
    {
      "epoch": 0.35808006551152965,
      "grad_norm": 11.38648411912873,
      "learning_rate": 4.052367038348471e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.4375,
      "logps/chosen": -532.0,
      "logps/rejected": -652.0,
      "loss": 0.3988,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.6875,
      "step": 9620
    },
    {
      "epoch": 0.3584522901118535,
      "grad_norm": 10.108805546501396,
      "learning_rate": 4.049819461464338e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.734375,
      "logps/chosen": -512.0,
      "logps/rejected": -628.0,
      "loss": 0.4161,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.59375,
      "step": 9630
    },
    {
      "epoch": 0.3588245147121773,
      "grad_norm": 9.966077284317917,
      "learning_rate": 4.0472692679638733e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.578125,
      "logps/chosen": -498.0,
      "logps/rejected": -648.0,
      "loss": 0.4037,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.59375,
      "step": 9640
    },
    {
      "epoch": 0.3591967393125012,
      "grad_norm": 9.630928059407173,
      "learning_rate": 4.0447164621526586e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -524.0,
      "logps/rejected": -672.0,
      "loss": 0.414,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.65625,
      "step": 9650
    },
    {
      "epoch": 0.359568963912825,
      "grad_norm": 10.792712488833171,
      "learning_rate": 4.04216104834069e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.65625,
      "logps/chosen": -552.0,
      "logps/rejected": -680.0,
      "loss": 0.4233,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.71875,
      "step": 9660
    },
    {
      "epoch": 0.3599411885131488,
      "grad_norm": 10.064575692907253,
      "learning_rate": 4.0396030308423643e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.671875,
      "logps/chosen": -510.0,
      "logps/rejected": -672.0,
      "loss": 0.434,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -4.65625,
      "step": 9670
    },
    {
      "epoch": 0.36031341311347265,
      "grad_norm": 10.701621742481832,
      "learning_rate": 4.037042413976476e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.6875,
      "logps/chosen": -512.0,
      "logps/rejected": -644.0,
      "loss": 0.4052,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.59375,
      "step": 9680
    },
    {
      "epoch": 0.3606856377137965,
      "grad_norm": 9.697049557863071,
      "learning_rate": 4.0344792020662067e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.75,
      "logps/chosen": -486.0,
      "logps/rejected": -608.0,
      "loss": 0.4034,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.375,
      "step": 9690
    },
    {
      "epoch": 0.36105786231412035,
      "grad_norm": 11.520104843604134,
      "learning_rate": 4.0319133994391206e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.734375,
      "logps/chosen": -552.0,
      "logps/rejected": -684.0,
      "loss": 0.4214,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.65625,
      "step": 9700
    },
    {
      "epoch": 0.36143008691444417,
      "grad_norm": 11.18415677659603,
      "learning_rate": 4.0293450104271544e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -516.0,
      "logps/rejected": -632.0,
      "loss": 0.4353,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -4.53125,
      "step": 9710
    },
    {
      "epoch": 0.361802311514768,
      "grad_norm": 10.253581111666751,
      "learning_rate": 4.026774039366612e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.640625,
      "logps/chosen": -502.0,
      "logps/rejected": -636.0,
      "loss": 0.4097,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.375,
      "step": 9720
    },
    {
      "epoch": 0.36217453611509187,
      "grad_norm": 8.935055524474384,
      "learning_rate": 4.0242004905981587e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.6875,
      "logps/chosen": -504.0,
      "logps/rejected": -600.0,
      "loss": 0.4345,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -4.3125,
      "step": 9730
    },
    {
      "epoch": 0.3625467607154157,
      "grad_norm": 8.41589084943201,
      "learning_rate": 4.0216243684668073e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -480.0,
      "logps/rejected": -592.0,
      "loss": 0.4097,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.234375,
      "rewards/rejected": -4.15625,
      "step": 9740
    },
    {
      "epoch": 0.3629189853157395,
      "grad_norm": 10.900030846013813,
      "learning_rate": 4.019045677321921e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.609375,
      "logps/chosen": -516.0,
      "logps/rejected": -620.0,
      "loss": 0.4293,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.421875,
      "rewards/rejected": -4.375,
      "step": 9750
    },
    {
      "epoch": 0.36329120991606334,
      "grad_norm": 10.037131540330535,
      "learning_rate": 4.016464421517196e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.640625,
      "logps/chosen": -480.0,
      "logps/rejected": -604.0,
      "loss": 0.4202,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.15625,
      "step": 9760
    },
    {
      "epoch": 0.36366343451638716,
      "grad_norm": 8.85982054210182,
      "learning_rate": 4.0138806054106604e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.71875,
      "logps/chosen": -528.0,
      "logps/rejected": -632.0,
      "loss": 0.4105,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -4.3125,
      "step": 9770
    },
    {
      "epoch": 0.36403565911671104,
      "grad_norm": 10.40042645936895,
      "learning_rate": 4.011294233364664e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.640625,
      "logps/chosen": -508.0,
      "logps/rejected": -648.0,
      "loss": 0.4021,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.53125,
      "step": 9780
    },
    {
      "epoch": 0.36440788371703486,
      "grad_norm": 9.712987528039212,
      "learning_rate": 4.0087053097458735e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.90625,
      "logps/chosen": -516.0,
      "logps/rejected": -660.0,
      "loss": 0.3869,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.75,
      "step": 9790
    },
    {
      "epoch": 0.3647801083173587,
      "grad_norm": 11.480600720870623,
      "learning_rate": 4.00611383892526e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.796875,
      "logps/chosen": -476.0,
      "logps/rejected": -612.0,
      "loss": 0.4546,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.40625,
      "step": 9800
    },
    {
      "epoch": 0.3651523329176825,
      "grad_norm": 9.706454650290478,
      "learning_rate": 4.003519825278101e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.640625,
      "logps/chosen": -504.0,
      "logps/rejected": -648.0,
      "loss": 0.4232,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.40625,
      "step": 9810
    },
    {
      "epoch": 0.3655245575180064,
      "grad_norm": 9.486306933250026,
      "learning_rate": 4.000923273183961e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.75,
      "logps/chosen": -456.0,
      "logps/rejected": -596.0,
      "loss": 0.4036,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.671875,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.25,
      "step": 9820
    },
    {
      "epoch": 0.3658967821183302,
      "grad_norm": 9.58067090772537,
      "learning_rate": 3.9983241870266935e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.734375,
      "logps/chosen": -486.0,
      "logps/rejected": -620.0,
      "loss": 0.3991,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.28125,
      "step": 9830
    },
    {
      "epoch": 0.36626900671865403,
      "grad_norm": 9.95629273820144,
      "learning_rate": 3.995722571194431e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.78125,
      "logps/chosen": -482.0,
      "logps/rejected": -612.0,
      "loss": 0.4074,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -4.125,
      "step": 9840
    },
    {
      "epoch": 0.36664123131897786,
      "grad_norm": 12.673776401415893,
      "learning_rate": 3.9931184300795746e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.8125,
      "logps/chosen": -516.0,
      "logps/rejected": -636.0,
      "loss": 0.4238,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.46875,
      "step": 9850
    },
    {
      "epoch": 0.36701345591930173,
      "grad_norm": 9.883571893330275,
      "learning_rate": 3.9905117680787906e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.78125,
      "logps/chosen": -528.0,
      "logps/rejected": -664.0,
      "loss": 0.4308,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.65625,
      "step": 9860
    },
    {
      "epoch": 0.36738568051962556,
      "grad_norm": 10.083896059014323,
      "learning_rate": 3.987902589593e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -544.0,
      "logps/rejected": -660.0,
      "loss": 0.4237,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.5625,
      "step": 9870
    },
    {
      "epoch": 0.3677579051199494,
      "grad_norm": 8.887525657075784,
      "learning_rate": 3.9852908990273737e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.71875,
      "logps/chosen": -486.0,
      "logps/rejected": -616.0,
      "loss": 0.4152,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.3125,
      "step": 9880
    },
    {
      "epoch": 0.3681301297202732,
      "grad_norm": 9.658722206532655,
      "learning_rate": 3.9826767007913246e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.421875,
      "logps/chosen": -484.0,
      "logps/rejected": -640.0,
      "loss": 0.4183,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -4.46875,
      "step": 9890
    },
    {
      "epoch": 0.368502354320597,
      "grad_norm": 9.736984023875639,
      "learning_rate": 3.9800599992984973e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.671875,
      "logps/chosen": -506.0,
      "logps/rejected": -632.0,
      "loss": 0.4315,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.4375,
      "step": 9900
    },
    {
      "epoch": 0.3688745789209209,
      "grad_norm": 10.65089311787536,
      "learning_rate": 3.9774407989667637e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.734375,
      "logps/chosen": -508.0,
      "logps/rejected": -612.0,
      "loss": 0.4198,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -4.34375,
      "step": 9910
    },
    {
      "epoch": 0.3692468035212447,
      "grad_norm": 10.16598534123045,
      "learning_rate": 3.9748191042182143e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.6875,
      "logps/chosen": -502.0,
      "logps/rejected": -628.0,
      "loss": 0.4031,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.40625,
      "step": 9920
    },
    {
      "epoch": 0.36961902812156855,
      "grad_norm": 11.864593071983812,
      "learning_rate": 3.9721949194791527e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -544.0,
      "logps/rejected": -648.0,
      "loss": 0.3972,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.59375,
      "step": 9930
    },
    {
      "epoch": 0.36999125272189237,
      "grad_norm": 9.302649226600774,
      "learning_rate": 3.969568249180083e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.71875,
      "logps/chosen": -528.0,
      "logps/rejected": -628.0,
      "loss": 0.437,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.109375,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -4.34375,
      "step": 9940
    },
    {
      "epoch": 0.37036347732221625,
      "grad_norm": 9.746787970007667,
      "learning_rate": 3.96693909775571e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.640625,
      "logps/chosen": -486.0,
      "logps/rejected": -608.0,
      "loss": 0.4165,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.21875,
      "step": 9950
    },
    {
      "epoch": 0.3707357019225401,
      "grad_norm": 10.965343621439846,
      "learning_rate": 3.9643074696449235e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.640625,
      "logps/chosen": -490.0,
      "logps/rejected": -636.0,
      "loss": 0.4207,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.34375,
      "step": 9960
    },
    {
      "epoch": 0.3711079265228639,
      "grad_norm": 9.797040993093646,
      "learning_rate": 3.961673369290798e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.703125,
      "logps/chosen": -524.0,
      "logps/rejected": -632.0,
      "loss": 0.417,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -4.1875,
      "step": 9970
    },
    {
      "epoch": 0.3714801511231877,
      "grad_norm": 9.982262730561114,
      "learning_rate": 3.9590368011405786e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -484.0,
      "logps/rejected": -624.0,
      "loss": 0.4063,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.21875,
      "step": 9980
    },
    {
      "epoch": 0.37185237572351154,
      "grad_norm": 12.262469951466043,
      "learning_rate": 3.956397769645681e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.65625,
      "logps/chosen": -498.0,
      "logps/rejected": -620.0,
      "loss": 0.4211,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.015625,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.375,
      "step": 9990
    },
    {
      "epoch": 0.3722246003238354,
      "grad_norm": 10.480775737416053,
      "learning_rate": 3.9537562792616753e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5625,
      "logps/chosen": -516.0,
      "logps/rejected": -640.0,
      "loss": 0.4228,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.5625,
      "step": 10000
    },
    {
      "epoch": 0.3722246003238354,
      "eval_logits/chosen": -2.6875,
      "eval_logits/rejected": -2.640625,
      "eval_logps/chosen": -568.0,
      "eval_logps/rejected": -660.0,
      "eval_loss": 0.4865521490573883,
      "eval_rewards/accuracies": 0.7406647801399231,
      "eval_rewards/chosen": -3.40625,
      "eval_rewards/margins": 1.21875,
      "eval_rewards/rejected": -4.625,
      "eval_runtime": 5143.5059,
      "eval_samples_per_second": 37.145,
      "eval_steps_per_second": 0.581,
      "step": 10000
    },
    {
      "epoch": 0.37259682492415924,
      "grad_norm": 11.94008065872819,
      "learning_rate": 3.951112334448288e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5625,
      "logps/chosen": -510.0,
      "logps/rejected": -632.0,
      "loss": 0.4113,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.5,
      "step": 10010
    },
    {
      "epoch": 0.37296904952448307,
      "grad_norm": 11.395384402251135,
      "learning_rate": 3.948465939669385e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.609375,
      "logps/chosen": -496.0,
      "logps/rejected": -608.0,
      "loss": 0.4277,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -4.1875,
      "step": 10020
    },
    {
      "epoch": 0.3733412741248069,
      "grad_norm": 9.387796713458002,
      "learning_rate": 3.9458170993929705e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.4375,
      "logps/chosen": -486.0,
      "logps/rejected": -616.0,
      "loss": 0.4361,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.15625,
      "step": 10030
    },
    {
      "epoch": 0.37371349872513077,
      "grad_norm": 8.088344181673163,
      "learning_rate": 3.94316581809118e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.734375,
      "logps/chosen": -496.0,
      "logps/rejected": -600.0,
      "loss": 0.4187,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.328125,
      "rewards/rejected": -4.15625,
      "step": 10040
    },
    {
      "epoch": 0.3740857233254546,
      "grad_norm": 10.198518001088473,
      "learning_rate": 3.940512100240264e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.796875,
      "logps/chosen": -532.0,
      "logps/rejected": -608.0,
      "loss": 0.4155,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.234375,
      "rewards/rejected": -4.15625,
      "step": 10050
    },
    {
      "epoch": 0.3744579479257784,
      "grad_norm": 9.392886847378989,
      "learning_rate": 3.9378559503205934e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.4375,
      "logps/chosen": -470.0,
      "logps/rejected": -608.0,
      "loss": 0.4108,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.3125,
      "step": 10060
    },
    {
      "epoch": 0.37483017252610223,
      "grad_norm": 11.60048604022286,
      "learning_rate": 3.9351973728166407e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.484375,
      "logps/chosen": -476.0,
      "logps/rejected": -640.0,
      "loss": 0.4241,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.375,
      "step": 10070
    },
    {
      "epoch": 0.3752023971264261,
      "grad_norm": 9.73301748963722,
      "learning_rate": 3.9325363722169787e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.546875,
      "logps/chosen": -528.0,
      "logps/rejected": -676.0,
      "loss": 0.413,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -4.78125,
      "step": 10080
    },
    {
      "epoch": 0.37557462172674994,
      "grad_norm": 8.574361510840918,
      "learning_rate": 3.9298729530142716e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.578125,
      "logps/chosen": -498.0,
      "logps/rejected": -632.0,
      "loss": 0.3884,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.015625,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.375,
      "step": 10090
    },
    {
      "epoch": 0.37594684632707376,
      "grad_norm": 10.20901971117539,
      "learning_rate": 3.927207119705267e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.75,
      "logps/chosen": -536.0,
      "logps/rejected": -668.0,
      "loss": 0.4144,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.71875,
      "step": 10100
    },
    {
      "epoch": 0.3763190709273976,
      "grad_norm": 9.665838470373643,
      "learning_rate": 3.924538876790787e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.484375,
      "logps/chosen": -536.0,
      "logps/rejected": -700.0,
      "loss": 0.3998,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.9375,
      "step": 10110
    },
    {
      "epoch": 0.3766912955277214,
      "grad_norm": 11.621403039213453,
      "learning_rate": 3.9218682287757233e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.625,
      "logps/chosen": -532.0,
      "logps/rejected": -644.0,
      "loss": 0.4108,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.265625,
      "rewards/rejected": -4.5625,
      "step": 10120
    },
    {
      "epoch": 0.3770635201280453,
      "grad_norm": 10.15393275343064,
      "learning_rate": 3.9191951801690273e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -556.0,
      "logps/rejected": -672.0,
      "loss": 0.4164,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.25,
      "rewards/rejected": -4.40625,
      "step": 10130
    },
    {
      "epoch": 0.3774357447283691,
      "grad_norm": 10.46505977499063,
      "learning_rate": 3.916519735483703e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.578125,
      "logps/chosen": -478.0,
      "logps/rejected": -632.0,
      "loss": 0.4067,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.5,
      "step": 10140
    },
    {
      "epoch": 0.3778079693286929,
      "grad_norm": 10.387852958177254,
      "learning_rate": 3.913841899236803e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.71875,
      "logps/chosen": -494.0,
      "logps/rejected": -624.0,
      "loss": 0.4235,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.25,
      "step": 10150
    },
    {
      "epoch": 0.37818019392901675,
      "grad_norm": 9.164778262363997,
      "learning_rate": 3.911161675949412e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.8125,
      "logps/chosen": -490.0,
      "logps/rejected": -608.0,
      "loss": 0.4168,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.25,
      "step": 10160
    },
    {
      "epoch": 0.37855241852934063,
      "grad_norm": 9.409354994292451,
      "learning_rate": 3.90847907014665e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.765625,
      "logps/chosen": -504.0,
      "logps/rejected": -656.0,
      "loss": 0.4137,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.5625,
      "step": 10170
    },
    {
      "epoch": 0.37892464312966445,
      "grad_norm": 11.164565027031653,
      "learning_rate": 3.905794086357658e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.734375,
      "logps/chosen": -560.0,
      "logps/rejected": -680.0,
      "loss": 0.4199,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.78125,
      "step": 10180
    },
    {
      "epoch": 0.3792968677299883,
      "grad_norm": 12.993210697293454,
      "learning_rate": 3.903106729115591e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.796875,
      "logps/chosen": -524.0,
      "logps/rejected": -664.0,
      "loss": 0.4203,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -4.75,
      "step": 10190
    },
    {
      "epoch": 0.3796690923303121,
      "grad_norm": 10.782348398739133,
      "learning_rate": 3.90041700295761e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.859375,
      "logps/chosen": -476.0,
      "logps/rejected": -600.0,
      "loss": 0.4214,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.21875,
      "step": 10200
    },
    {
      "epoch": 0.3800413169306359,
      "grad_norm": 10.114483546419937,
      "learning_rate": 3.897724912424879e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5,
      "logps/chosen": -468.0,
      "logps/rejected": -596.0,
      "loss": 0.4045,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.0625,
      "step": 10210
    },
    {
      "epoch": 0.3804135415309598,
      "grad_norm": 9.140443826007905,
      "learning_rate": 3.8950304620625514e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.625,
      "logps/chosen": -472.0,
      "logps/rejected": -584.0,
      "loss": 0.4386,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -4.125,
      "step": 10220
    },
    {
      "epoch": 0.3807857661312836,
      "grad_norm": 10.6092853864904,
      "learning_rate": 3.892333656419765e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.59375,
      "logps/chosen": -520.0,
      "logps/rejected": -668.0,
      "loss": 0.4344,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.625,
      "step": 10230
    },
    {
      "epoch": 0.38115799073160744,
      "grad_norm": 9.385282576095236,
      "learning_rate": 3.8896345000496343e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.6875,
      "logps/chosen": -512.0,
      "logps/rejected": -592.0,
      "loss": 0.4169,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -4.0625,
      "step": 10240
    },
    {
      "epoch": 0.38153021533193127,
      "grad_norm": 9.962265252441915,
      "learning_rate": 3.886932997509244e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.890625,
      "logps/chosen": -508.0,
      "logps/rejected": -644.0,
      "loss": 0.3856,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.4375,
      "step": 10250
    },
    {
      "epoch": 0.38190243993225514,
      "grad_norm": 13.785572295251605,
      "learning_rate": 3.884229153359637e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.671875,
      "logps/chosen": -492.0,
      "logps/rejected": -612.0,
      "loss": 0.4356,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -4.125,
      "step": 10260
    },
    {
      "epoch": 0.38227466453257897,
      "grad_norm": 10.253873856028465,
      "learning_rate": 3.881522972165812e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.609375,
      "logps/chosen": -488.0,
      "logps/rejected": -628.0,
      "loss": 0.4015,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.28125,
      "step": 10270
    },
    {
      "epoch": 0.3826468891329028,
      "grad_norm": 10.208750338977053,
      "learning_rate": 3.8788144584967135e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -528.0,
      "logps/rejected": -676.0,
      "loss": 0.3985,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.109375,
      "rewards/margins": 1.625,
      "rewards/rejected": -4.75,
      "step": 10280
    },
    {
      "epoch": 0.3830191137332266,
      "grad_norm": 12.299859037985076,
      "learning_rate": 3.876103616925223e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -510.0,
      "logps/rejected": -640.0,
      "loss": 0.4049,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.4375,
      "step": 10290
    },
    {
      "epoch": 0.3833913383335505,
      "grad_norm": 14.691088819849147,
      "learning_rate": 3.873390452028151e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.78125,
      "logps/chosen": -504.0,
      "logps/rejected": -652.0,
      "loss": 0.4124,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.703125,
      "rewards/rejected": -4.6875,
      "step": 10300
    },
    {
      "epoch": 0.3837635629338743,
      "grad_norm": 8.806554390516112,
      "learning_rate": 3.870674968386234e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.71875,
      "logps/chosen": -516.0,
      "logps/rejected": -668.0,
      "loss": 0.4014,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.625,
      "step": 10310
    },
    {
      "epoch": 0.38413578753419814,
      "grad_norm": 10.264423337651365,
      "learning_rate": 3.86795717058412e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.734375,
      "logps/chosen": -494.0,
      "logps/rejected": -620.0,
      "loss": 0.4184,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.25,
      "step": 10320
    },
    {
      "epoch": 0.38450801213452196,
      "grad_norm": 9.60349202670986,
      "learning_rate": 3.8652370632103665e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.796875,
      "logps/chosen": -464.0,
      "logps/rejected": -608.0,
      "loss": 0.4182,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.515625,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -4.1875,
      "step": 10330
    },
    {
      "epoch": 0.3848802367348458,
      "grad_norm": 10.56784620413968,
      "learning_rate": 3.862514650857428e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.703125,
      "logps/chosen": -496.0,
      "logps/rejected": -620.0,
      "loss": 0.4423,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.15625,
      "rewards/rejected": -4.15625,
      "step": 10340
    },
    {
      "epoch": 0.38525246133516966,
      "grad_norm": 8.30409199748182,
      "learning_rate": 3.859789938121654e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.953125,
      "logps/chosen": -524.0,
      "logps/rejected": -648.0,
      "loss": 0.4268,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.40625,
      "step": 10350
    },
    {
      "epoch": 0.3856246859354935,
      "grad_norm": 11.89477795964885,
      "learning_rate": 3.8570629296032734e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.875,
      "logps/chosen": -492.0,
      "logps/rejected": -592.0,
      "loss": 0.4136,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.1796875,
      "rewards/rejected": -3.984375,
      "step": 10360
    },
    {
      "epoch": 0.3859969105358173,
      "grad_norm": 12.155498488930593,
      "learning_rate": 3.854333629906395e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.8125,
      "logps/chosen": -494.0,
      "logps/rejected": -576.0,
      "loss": 0.442,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -3.875,
      "step": 10370
    },
    {
      "epoch": 0.38636913513614113,
      "grad_norm": 9.686003256409352,
      "learning_rate": 3.851602043638994e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.75,
      "logps/chosen": -500.0,
      "logps/rejected": -640.0,
      "loss": 0.4094,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.3125,
      "step": 10380
    },
    {
      "epoch": 0.386741359736465,
      "grad_norm": 8.137956938163095,
      "learning_rate": 3.848868175412906e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.890625,
      "logps/chosen": -516.0,
      "logps/rejected": -616.0,
      "loss": 0.4163,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -4.40625,
      "step": 10390
    },
    {
      "epoch": 0.38711358433678883,
      "grad_norm": 9.5475305710195,
      "learning_rate": 3.8461320298438206e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.84375,
      "logps/chosen": -504.0,
      "logps/rejected": -608.0,
      "loss": 0.4309,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -4.1875,
      "step": 10400
    },
    {
      "epoch": 0.38748580893711265,
      "grad_norm": 11.570817108915902,
      "learning_rate": 3.84339361155127e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.703125,
      "logps/chosen": -494.0,
      "logps/rejected": -620.0,
      "loss": 0.4057,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -4.1875,
      "step": 10410
    },
    {
      "epoch": 0.3878580335374365,
      "grad_norm": 12.764738318932972,
      "learning_rate": 3.8406529251586254e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.671875,
      "logps/chosen": -520.0,
      "logps/rejected": -676.0,
      "loss": 0.4253,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.71875,
      "step": 10420
    },
    {
      "epoch": 0.3882302581377603,
      "grad_norm": 10.758000948274914,
      "learning_rate": 3.837909975293088e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.671875,
      "logps/chosen": -524.0,
      "logps/rejected": -652.0,
      "loss": 0.4258,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.65625,
      "step": 10430
    },
    {
      "epoch": 0.3886024827380842,
      "grad_norm": 11.38890064337307,
      "learning_rate": 3.835164766585679e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.765625,
      "logps/chosen": -498.0,
      "logps/rejected": -636.0,
      "loss": 0.412,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.40625,
      "step": 10440
    },
    {
      "epoch": 0.388974707338408,
      "grad_norm": 10.472818589617436,
      "learning_rate": 3.8324173036712336e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.734375,
      "logps/chosen": -478.0,
      "logps/rejected": -588.0,
      "loss": 0.4172,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -3.953125,
      "step": 10450
    },
    {
      "epoch": 0.3893469319387318,
      "grad_norm": 9.575106342394449,
      "learning_rate": 3.829667591188393e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5625,
      "logps/chosen": -464.0,
      "logps/rejected": -592.0,
      "loss": 0.4065,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.09375,
      "step": 10460
    },
    {
      "epoch": 0.38971915653905564,
      "grad_norm": 8.613764548825879,
      "learning_rate": 3.826915633779596e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.734375,
      "logps/chosen": -528.0,
      "logps/rejected": -640.0,
      "loss": 0.4254,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.296875,
      "rewards/rejected": -4.5625,
      "step": 10470
    },
    {
      "epoch": 0.3900913811393795,
      "grad_norm": 10.302163114251366,
      "learning_rate": 3.8241614360910726e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.71875,
      "logps/chosen": -500.0,
      "logps/rejected": -628.0,
      "loss": 0.4145,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.375,
      "step": 10480
    },
    {
      "epoch": 0.39046360573970335,
      "grad_norm": 12.358984821623201,
      "learning_rate": 3.8214050027728335e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.765625,
      "logps/chosen": -512.0,
      "logps/rejected": -632.0,
      "loss": 0.4114,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.3125,
      "step": 10490
    },
    {
      "epoch": 0.39083583034002717,
      "grad_norm": 8.954530896768585,
      "learning_rate": 3.818646338478665e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.796875,
      "logps/chosen": -516.0,
      "logps/rejected": -632.0,
      "loss": 0.4151,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -4.09375,
      "step": 10500
    },
    {
      "epoch": 0.391208054940351,
      "grad_norm": 10.00686117193749,
      "learning_rate": 3.815885447866121e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.65625,
      "logps/chosen": -532.0,
      "logps/rejected": -628.0,
      "loss": 0.4587,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.296875,
      "rewards/rejected": -4.34375,
      "step": 10510
    },
    {
      "epoch": 0.39158027954067487,
      "grad_norm": 9.855419612029971,
      "learning_rate": 3.813122335596513e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.796875,
      "logps/chosen": -498.0,
      "logps/rejected": -608.0,
      "loss": 0.4211,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -4.15625,
      "step": 10520
    },
    {
      "epoch": 0.3919525041409987,
      "grad_norm": 10.791674898000176,
      "learning_rate": 3.8103570063349027e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.8125,
      "logps/chosen": -510.0,
      "logps/rejected": -616.0,
      "loss": 0.4207,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.125,
      "step": 10530
    },
    {
      "epoch": 0.3923247287413225,
      "grad_norm": 8.711550520885858,
      "learning_rate": 3.807589464750097e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.734375,
      "logps/chosen": -528.0,
      "logps/rejected": -652.0,
      "loss": 0.406,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -4.59375,
      "step": 10540
    },
    {
      "epoch": 0.39269695334164634,
      "grad_norm": 10.224131362315227,
      "learning_rate": 3.8048197155146343e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.765625,
      "logps/chosen": -516.0,
      "logps/rejected": -676.0,
      "loss": 0.399,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -4.625,
      "step": 10550
    },
    {
      "epoch": 0.39306917794197016,
      "grad_norm": 9.524586790933224,
      "learning_rate": 3.8020477633047847e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.96875,
      "logps/chosen": -524.0,
      "logps/rejected": -620.0,
      "loss": 0.4411,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -4.25,
      "step": 10560
    },
    {
      "epoch": 0.39344140254229404,
      "grad_norm": 9.738796598673675,
      "learning_rate": 3.7992736128005344e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.875,
      "logps/chosen": -504.0,
      "logps/rejected": -644.0,
      "loss": 0.4319,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.546875,
      "rewards/rejected": -4.4375,
      "step": 10570
    },
    {
      "epoch": 0.39381362714261786,
      "grad_norm": 9.47596869571761,
      "learning_rate": 3.7964972686855833e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.875,
      "logps/chosen": -520.0,
      "logps/rejected": -624.0,
      "loss": 0.4225,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -4.28125,
      "step": 10580
    },
    {
      "epoch": 0.3941858517429417,
      "grad_norm": 9.578873348009736,
      "learning_rate": 3.7937187356473323e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.8125,
      "logps/chosen": -528.0,
      "logps/rejected": -648.0,
      "loss": 0.3987,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.46875,
      "step": 10590
    },
    {
      "epoch": 0.3945580763432655,
      "grad_norm": 9.27872645306823,
      "learning_rate": 3.7909380183768796e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.8125,
      "logps/chosen": -490.0,
      "logps/rejected": -660.0,
      "loss": 0.3999,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.859375,
      "rewards/rejected": -4.65625,
      "step": 10600
    },
    {
      "epoch": 0.3949303009435894,
      "grad_norm": 11.388409114859941,
      "learning_rate": 3.7881551215690123e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.8125,
      "logps/chosen": -548.0,
      "logps/rejected": -668.0,
      "loss": 0.4319,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.328125,
      "rewards/rejected": -4.625,
      "step": 10610
    },
    {
      "epoch": 0.3953025255439132,
      "grad_norm": 12.70353727173444,
      "learning_rate": 3.785370049922194e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.84375,
      "logps/chosen": -520.0,
      "logps/rejected": -664.0,
      "loss": 0.405,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.65625,
      "step": 10620
    },
    {
      "epoch": 0.39567475014423703,
      "grad_norm": 9.502681719706718,
      "learning_rate": 3.782582808138564e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.78125,
      "logps/chosen": -510.0,
      "logps/rejected": -628.0,
      "loss": 0.4168,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.3125,
      "step": 10630
    },
    {
      "epoch": 0.39604697474456085,
      "grad_norm": 7.207535755228262,
      "learning_rate": 3.7797934009239217e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.03125,
      "logps/chosen": -496.0,
      "logps/rejected": -596.0,
      "loss": 0.3938,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.21875,
      "step": 10640
    },
    {
      "epoch": 0.3964191993448847,
      "grad_norm": 11.650091729668777,
      "learning_rate": 3.777001832987726e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -516.0,
      "logps/rejected": -640.0,
      "loss": 0.438,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.1328125,
      "rewards/rejected": -4.125,
      "step": 10650
    },
    {
      "epoch": 0.39679142394520855,
      "grad_norm": 9.236701589338367,
      "learning_rate": 3.77420810904308e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.703125,
      "logps/chosen": -496.0,
      "logps/rejected": -612.0,
      "loss": 0.4321,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.25,
      "step": 10660
    },
    {
      "epoch": 0.3971636485455324,
      "grad_norm": 10.84194371239337,
      "learning_rate": 3.771412233806731e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.765625,
      "logps/chosen": -516.0,
      "logps/rejected": -636.0,
      "loss": 0.403,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.328125,
      "rewards/rejected": -4.25,
      "step": 10670
    },
    {
      "epoch": 0.3975358731458562,
      "grad_norm": 9.720210618064646,
      "learning_rate": 3.768614211999056e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.625,
      "logps/chosen": -480.0,
      "logps/rejected": -628.0,
      "loss": 0.4014,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.59375,
      "rewards/rejected": -4.34375,
      "step": 10680
    },
    {
      "epoch": 0.39790809774618,
      "grad_norm": 8.74392410605326,
      "learning_rate": 3.765814048344056e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5625,
      "logps/chosen": -484.0,
      "logps/rejected": -612.0,
      "loss": 0.4149,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.34375,
      "step": 10690
    },
    {
      "epoch": 0.3982803223465039,
      "grad_norm": 10.63100610008647,
      "learning_rate": 3.76301174756935e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.75,
      "logps/chosen": -516.0,
      "logps/rejected": -652.0,
      "loss": 0.4178,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -4.53125,
      "step": 10700
    },
    {
      "epoch": 0.3986525469468277,
      "grad_norm": 9.223517021996186,
      "learning_rate": 3.7602073144061617e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.625,
      "logps/chosen": -480.0,
      "logps/rejected": -620.0,
      "loss": 0.4012,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.640625,
      "rewards/rejected": -4.34375,
      "step": 10710
    },
    {
      "epoch": 0.39902477154715155,
      "grad_norm": 10.62601530570878,
      "learning_rate": 3.757400753589319e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.578125,
      "logps/chosen": -488.0,
      "logps/rejected": -628.0,
      "loss": 0.408,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.640625,
      "rewards/rejected": -4.34375,
      "step": 10720
    },
    {
      "epoch": 0.39939699614747537,
      "grad_norm": 12.222252901749451,
      "learning_rate": 3.7545920698572393e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -504.0,
      "logps/rejected": -624.0,
      "loss": 0.3992,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.28125,
      "step": 10730
    },
    {
      "epoch": 0.39976922074779925,
      "grad_norm": 10.211684591049455,
      "learning_rate": 3.7517812679519255e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.546875,
      "logps/chosen": -492.0,
      "logps/rejected": -624.0,
      "loss": 0.4185,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.3125,
      "step": 10740
    },
    {
      "epoch": 0.40014144534812307,
      "grad_norm": 10.19246196533904,
      "learning_rate": 3.748968352618957e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.53125,
      "logps/chosen": -516.0,
      "logps/rejected": -604.0,
      "loss": 0.4054,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -4.1875,
      "step": 10750
    },
    {
      "epoch": 0.4005136699484469,
      "grad_norm": 10.501124026602215,
      "learning_rate": 3.7461533286074785e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.671875,
      "logps/chosen": -494.0,
      "logps/rejected": -644.0,
      "loss": 0.4074,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.46875,
      "step": 10760
    },
    {
      "epoch": 0.4008858945487707,
      "grad_norm": 12.011952415521026,
      "learning_rate": 3.743336200670199e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -532.0,
      "logps/rejected": -680.0,
      "loss": 0.382,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.78125,
      "rewards/rejected": -4.96875,
      "step": 10770
    },
    {
      "epoch": 0.40125811914909454,
      "grad_norm": 9.885441687537561,
      "learning_rate": 3.7405169735633766e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.640625,
      "logps/chosen": -524.0,
      "logps/rejected": -668.0,
      "loss": 0.4216,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.6875,
      "step": 10780
    },
    {
      "epoch": 0.4016303437494184,
      "grad_norm": 11.299462064514877,
      "learning_rate": 3.7376956520468154e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.703125,
      "logps/chosen": -504.0,
      "logps/rejected": -648.0,
      "loss": 0.4098,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.59375,
      "step": 10790
    },
    {
      "epoch": 0.40200256834974224,
      "grad_norm": 11.10606829992472,
      "learning_rate": 3.734872240883854e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.734375,
      "logps/chosen": -484.0,
      "logps/rejected": -580.0,
      "loss": 0.4081,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.09375,
      "step": 10800
    },
    {
      "epoch": 0.40237479295006606,
      "grad_norm": 10.19329943497753,
      "learning_rate": 3.73204674484136e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5,
      "logps/chosen": -492.0,
      "logps/rejected": -612.0,
      "loss": 0.426,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -4.1875,
      "step": 10810
    },
    {
      "epoch": 0.4027470175503899,
      "grad_norm": 11.147779459030296,
      "learning_rate": 3.729219168689721e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.578125,
      "logps/chosen": -492.0,
      "logps/rejected": -604.0,
      "loss": 0.4171,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.265625,
      "rewards/rejected": -4.0,
      "step": 10820
    },
    {
      "epoch": 0.40311924215071376,
      "grad_norm": 12.11609933024967,
      "learning_rate": 3.7263895172028345e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.703125,
      "logps/chosen": -490.0,
      "logps/rejected": -596.0,
      "loss": 0.4233,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -4.125,
      "step": 10830
    },
    {
      "epoch": 0.4034914667510376,
      "grad_norm": 11.002121077939048,
      "learning_rate": 3.7235577951581063e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.625,
      "logps/chosen": -476.0,
      "logps/rejected": -580.0,
      "loss": 0.4244,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.28125,
      "rewards/rejected": -4.0625,
      "step": 10840
    },
    {
      "epoch": 0.4038636913513614,
      "grad_norm": 10.118733941824873,
      "learning_rate": 3.7207240073364334e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.640625,
      "logps/chosen": -478.0,
      "logps/rejected": -596.0,
      "loss": 0.4228,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.28125,
      "step": 10850
    },
    {
      "epoch": 0.40423591595168523,
      "grad_norm": 9.424948865948974,
      "learning_rate": 3.7178881585222035e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -504.0,
      "logps/rejected": -624.0,
      "loss": 0.4089,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.359375,
      "rewards/rejected": -4.21875,
      "step": 10860
    },
    {
      "epoch": 0.40460814055200905,
      "grad_norm": 7.9406245807735365,
      "learning_rate": 3.7150502535032825e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.78125,
      "logps/chosen": -498.0,
      "logps/rejected": -604.0,
      "loss": 0.4108,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -4.15625,
      "step": 10870
    },
    {
      "epoch": 0.40498036515233293,
      "grad_norm": 9.059782339176147,
      "learning_rate": 3.7122102970710065e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -508.0,
      "logps/rejected": -648.0,
      "loss": 0.4161,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.5,
      "step": 10880
    },
    {
      "epoch": 0.40535258975265676,
      "grad_norm": 10.651410061670129,
      "learning_rate": 3.7093682940201803e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.8125,
      "logps/chosen": -504.0,
      "logps/rejected": -628.0,
      "loss": 0.3974,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.34375,
      "step": 10890
    },
    {
      "epoch": 0.4057248143529806,
      "grad_norm": 12.347955414017374,
      "learning_rate": 3.7065242491490576e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.734375,
      "logps/chosen": -496.0,
      "logps/rejected": -636.0,
      "loss": 0.4244,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.4375,
      "step": 10900
    },
    {
      "epoch": 0.4060970389533044,
      "grad_norm": 11.734912878195932,
      "learning_rate": 3.703678167259344e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.671875,
      "logps/chosen": -516.0,
      "logps/rejected": -656.0,
      "loss": 0.4262,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.546875,
      "rewards/rejected": -4.5,
      "step": 10910
    },
    {
      "epoch": 0.4064692635536283,
      "grad_norm": 10.240470172162235,
      "learning_rate": 3.700830053156182e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.609375,
      "logps/chosen": -504.0,
      "logps/rejected": -632.0,
      "loss": 0.4317,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.4375,
      "step": 10920
    },
    {
      "epoch": 0.4068414881539521,
      "grad_norm": 10.763267528240021,
      "learning_rate": 3.6979799116481457e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.640625,
      "logps/chosen": -492.0,
      "logps/rejected": -644.0,
      "loss": 0.3922,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -4.625,
      "step": 10930
    },
    {
      "epoch": 0.4072137127542759,
      "grad_norm": 10.033262206399305,
      "learning_rate": 3.6951277475472333e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5625,
      "logps/chosen": -508.0,
      "logps/rejected": -644.0,
      "loss": 0.4016,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.46875,
      "step": 10940
    },
    {
      "epoch": 0.40758593735459975,
      "grad_norm": 11.21148247072398,
      "learning_rate": 3.6922735656688545e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.71875,
      "logps/chosen": -486.0,
      "logps/rejected": -624.0,
      "loss": 0.4009,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.40625,
      "step": 10950
    },
    {
      "epoch": 0.4079581619549236,
      "grad_norm": 10.07054131286777,
      "learning_rate": 3.6894173708318305e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.609375,
      "logps/chosen": -454.0,
      "logps/rejected": -588.0,
      "loss": 0.4134,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.515625,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -3.96875,
      "step": 10960
    },
    {
      "epoch": 0.40833038655524745,
      "grad_norm": 10.13578660741685,
      "learning_rate": 3.686559167858377e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.65625,
      "logps/chosen": -488.0,
      "logps/rejected": -616.0,
      "loss": 0.4082,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.15625,
      "step": 10970
    },
    {
      "epoch": 0.40870261115557127,
      "grad_norm": 10.7795000253726,
      "learning_rate": 3.683698961574102e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.609375,
      "logps/chosen": -520.0,
      "logps/rejected": -648.0,
      "loss": 0.4097,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.5,
      "step": 10980
    },
    {
      "epoch": 0.4090748357558951,
      "grad_norm": 12.401275265858745,
      "learning_rate": 3.680836756807995e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.59375,
      "logps/chosen": -506.0,
      "logps/rejected": -648.0,
      "loss": 0.4287,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.46875,
      "step": 10990
    },
    {
      "epoch": 0.4094470603562189,
      "grad_norm": 10.55074361201981,
      "learning_rate": 3.677972558392421e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.75,
      "logps/chosen": -520.0,
      "logps/rejected": -672.0,
      "loss": 0.3933,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -4.59375,
      "step": 11000
    },
    {
      "epoch": 0.4098192849565428,
      "grad_norm": 9.8345281410221,
      "learning_rate": 3.675106371163109e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.734375,
      "logps/chosen": -476.0,
      "logps/rejected": -616.0,
      "loss": 0.4461,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.375,
      "step": 11010
    },
    {
      "epoch": 0.4101915095568666,
      "grad_norm": 11.316018711212488,
      "learning_rate": 3.672238199959146e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -458.0,
      "logps/rejected": -592.0,
      "loss": 0.4087,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.1875,
      "step": 11020
    },
    {
      "epoch": 0.41056373415719044,
      "grad_norm": 9.434847886021549,
      "learning_rate": 3.6693680496229717e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.765625,
      "logps/chosen": -500.0,
      "logps/rejected": -604.0,
      "loss": 0.422,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.21875,
      "step": 11030
    },
    {
      "epoch": 0.41093595875751426,
      "grad_norm": 9.79404194407134,
      "learning_rate": 3.6664959250003623e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.484375,
      "logps/chosen": -480.0,
      "logps/rejected": -584.0,
      "loss": 0.4094,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.0625,
      "step": 11040
    },
    {
      "epoch": 0.41130818335783814,
      "grad_norm": 9.847885861324352,
      "learning_rate": 3.663621830940432e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.546875,
      "logps/chosen": -560.0,
      "logps/rejected": -660.0,
      "loss": 0.4403,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -4.53125,
      "step": 11050
    },
    {
      "epoch": 0.41168040795816196,
      "grad_norm": 10.03927527962245,
      "learning_rate": 3.6607457722956163e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.859375,
      "logps/chosen": -510.0,
      "logps/rejected": -612.0,
      "loss": 0.4095,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.015625,
      "rewards/margins": 1.234375,
      "rewards/rejected": -4.25,
      "step": 11060
    },
    {
      "epoch": 0.4120526325584858,
      "grad_norm": 11.811530387350572,
      "learning_rate": 3.6578677539216704e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.828125,
      "logps/chosen": -520.0,
      "logps/rejected": -652.0,
      "loss": 0.4235,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.4375,
      "step": 11070
    },
    {
      "epoch": 0.4124248571588096,
      "grad_norm": 12.476821646552663,
      "learning_rate": 3.6549877806776555e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.71875,
      "logps/chosen": -488.0,
      "logps/rejected": -636.0,
      "loss": 0.4144,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -4.53125,
      "step": 11080
    },
    {
      "epoch": 0.4127970817591335,
      "grad_norm": 11.253124454153857,
      "learning_rate": 3.6521058574259355e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.796875,
      "logps/chosen": -524.0,
      "logps/rejected": -664.0,
      "loss": 0.4372,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.5625,
      "step": 11090
    },
    {
      "epoch": 0.4131693063594573,
      "grad_norm": 11.088435677597,
      "learning_rate": 3.6492219890321674e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.765625,
      "logps/chosen": -504.0,
      "logps/rejected": -636.0,
      "loss": 0.433,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.4375,
      "step": 11100
    },
    {
      "epoch": 0.41354153095978113,
      "grad_norm": 10.134218559619017,
      "learning_rate": 3.6463361803652877e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.734375,
      "logps/chosen": -504.0,
      "logps/rejected": -616.0,
      "loss": 0.3848,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.234375,
      "rewards/rejected": -4.21875,
      "step": 11110
    },
    {
      "epoch": 0.41391375556010496,
      "grad_norm": 12.125916248490608,
      "learning_rate": 3.6434484362975135e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.71875,
      "logps/chosen": -548.0,
      "logps/rejected": -636.0,
      "loss": 0.4145,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.109375,
      "rewards/rejected": -4.4375,
      "step": 11120
    },
    {
      "epoch": 0.4142859801604288,
      "grad_norm": 12.192185242932485,
      "learning_rate": 3.640558761704328e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.734375,
      "logps/chosen": -506.0,
      "logps/rejected": -624.0,
      "loss": 0.4254,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -4.46875,
      "step": 11130
    },
    {
      "epoch": 0.41465820476075266,
      "grad_norm": 13.013585873786397,
      "learning_rate": 3.6376671614644726e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.59375,
      "logps/chosen": -474.0,
      "logps/rejected": -616.0,
      "loss": 0.4208,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.25,
      "step": 11140
    },
    {
      "epoch": 0.4150304293610765,
      "grad_norm": 11.242371287729116,
      "learning_rate": 3.6347736404599404e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.71875,
      "logps/chosen": -482.0,
      "logps/rejected": -628.0,
      "loss": 0.4262,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.28125,
      "step": 11150
    },
    {
      "epoch": 0.4154026539614003,
      "grad_norm": 9.877396880893045,
      "learning_rate": 3.631878203575969e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.40625,
      "logps/chosen": -472.0,
      "logps/rejected": -628.0,
      "loss": 0.3858,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.1875,
      "step": 11160
    },
    {
      "epoch": 0.4157748785617241,
      "grad_norm": 9.899960749856568,
      "learning_rate": 3.628980855701028e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.765625,
      "logps/chosen": -524.0,
      "logps/rejected": -644.0,
      "loss": 0.3981,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.25,
      "step": 11170
    },
    {
      "epoch": 0.416147103162048,
      "grad_norm": 8.963193081113872,
      "learning_rate": 3.626081601726816e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.703125,
      "logps/chosen": -488.0,
      "logps/rejected": -644.0,
      "loss": 0.4113,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.4375,
      "step": 11180
    },
    {
      "epoch": 0.4165193277623718,
      "grad_norm": 9.544342132519867,
      "learning_rate": 3.623180446548248e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.65625,
      "logps/chosen": -528.0,
      "logps/rejected": -640.0,
      "loss": 0.4075,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.109375,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -4.3125,
      "step": 11190
    },
    {
      "epoch": 0.41689155236269565,
      "grad_norm": 11.210583129932887,
      "learning_rate": 3.620277395063449e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.546875,
      "logps/chosen": -488.0,
      "logps/rejected": -632.0,
      "loss": 0.3904,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.4375,
      "step": 11200
    },
    {
      "epoch": 0.4172637769630195,
      "grad_norm": 10.652722202275728,
      "learning_rate": 3.6173724521737467e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.625,
      "logps/chosen": -512.0,
      "logps/rejected": -660.0,
      "loss": 0.399,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.59375,
      "step": 11210
    },
    {
      "epoch": 0.4176360015633433,
      "grad_norm": 10.088382158974337,
      "learning_rate": 3.61446562278366e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.734375,
      "logps/chosen": -528.0,
      "logps/rejected": -636.0,
      "loss": 0.4181,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.5,
      "step": 11220
    },
    {
      "epoch": 0.4180082261636672,
      "grad_norm": 10.130640517286619,
      "learning_rate": 3.6115569118008966e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.75,
      "logps/chosen": -472.0,
      "logps/rejected": -608.0,
      "loss": 0.4041,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.3125,
      "step": 11230
    },
    {
      "epoch": 0.418380450763991,
      "grad_norm": 11.042532126079774,
      "learning_rate": 3.6086463241363365e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.6875,
      "logps/chosen": -494.0,
      "logps/rejected": -624.0,
      "loss": 0.4137,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -4.375,
      "step": 11240
    },
    {
      "epoch": 0.4187526753643148,
      "grad_norm": 11.43699860134575,
      "learning_rate": 3.605733864704031e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -524.0,
      "logps/rejected": -664.0,
      "loss": 0.4259,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.8125,
      "step": 11250
    },
    {
      "epoch": 0.41912489996463864,
      "grad_norm": 10.094985145500816,
      "learning_rate": 3.602819538421191e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.5,
      "logps/chosen": -536.0,
      "logps/rejected": -640.0,
      "loss": 0.4065,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.234375,
      "rewards/rejected": -4.3125,
      "step": 11260
    },
    {
      "epoch": 0.4194971245649625,
      "grad_norm": 11.129915620458062,
      "learning_rate": 3.5999033502081783e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -486.0,
      "logps/rejected": -592.0,
      "loss": 0.4258,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -4.09375,
      "step": 11270
    },
    {
      "epoch": 0.41986934916528634,
      "grad_norm": 9.081794308877965,
      "learning_rate": 3.596985304988501e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.703125,
      "logps/chosen": -528.0,
      "logps/rejected": -648.0,
      "loss": 0.4057,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.46875,
      "step": 11280
    },
    {
      "epoch": 0.42024157376561017,
      "grad_norm": 12.682576551246736,
      "learning_rate": 3.5940654076887977e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.75,
      "logps/chosen": -506.0,
      "logps/rejected": -644.0,
      "loss": 0.428,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.5,
      "step": 11290
    },
    {
      "epoch": 0.420613798365934,
      "grad_norm": 10.26895518786948,
      "learning_rate": 3.5911436632388396e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.734375,
      "logps/chosen": -494.0,
      "logps/rejected": -616.0,
      "loss": 0.396,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -4.28125,
      "step": 11300
    },
    {
      "epoch": 0.42098602296625787,
      "grad_norm": 11.408603552450545,
      "learning_rate": 3.588220076571513e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -488.0,
      "logps/rejected": -648.0,
      "loss": 0.4149,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -4.59375,
      "step": 11310
    },
    {
      "epoch": 0.4213582475665817,
      "grad_norm": 10.506601063322611,
      "learning_rate": 3.5852946526228136e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.859375,
      "logps/chosen": -506.0,
      "logps/rejected": -636.0,
      "loss": 0.3937,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.59375,
      "rewards/rejected": -4.5,
      "step": 11320
    },
    {
      "epoch": 0.4217304721669055,
      "grad_norm": 10.487889535969714,
      "learning_rate": 3.582367396331842e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.71875,
      "logps/chosen": -520.0,
      "logps/rejected": -648.0,
      "loss": 0.4085,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.59375,
      "step": 11330
    },
    {
      "epoch": 0.42210269676722934,
      "grad_norm": 12.237765208950542,
      "learning_rate": 3.5794383126407905e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.84375,
      "logps/chosen": -532.0,
      "logps/rejected": -660.0,
      "loss": 0.4099,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.546875,
      "rewards/rejected": -4.71875,
      "step": 11340
    },
    {
      "epoch": 0.42247492136755316,
      "grad_norm": 9.00078779022958,
      "learning_rate": 3.5765074064949376e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -484.0,
      "logps/rejected": -652.0,
      "loss": 0.3946,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -4.65625,
      "step": 11350
    },
    {
      "epoch": 0.42284714596787704,
      "grad_norm": 9.585583691635222,
      "learning_rate": 3.573574682842637e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.875,
      "logps/chosen": -508.0,
      "logps/rejected": -640.0,
      "loss": 0.4089,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -4.53125,
      "step": 11360
    },
    {
      "epoch": 0.42321937056820086,
      "grad_norm": 9.881159614353601,
      "learning_rate": 3.5706401466353146e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.65625,
      "logps/chosen": -474.0,
      "logps/rejected": -596.0,
      "loss": 0.4169,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.65625,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -3.9375,
      "step": 11370
    },
    {
      "epoch": 0.4235915951685247,
      "grad_norm": 8.794274080134635,
      "learning_rate": 3.567703802827453e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -486.0,
      "logps/rejected": -620.0,
      "loss": 0.4066,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.21875,
      "step": 11380
    },
    {
      "epoch": 0.4239638197688485,
      "grad_norm": 11.493999762293472,
      "learning_rate": 3.564765656376587e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.71875,
      "logps/chosen": -524.0,
      "logps/rejected": -640.0,
      "loss": 0.4113,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.4375,
      "step": 11390
    },
    {
      "epoch": 0.4243360443691724,
      "grad_norm": 12.333078388370637,
      "learning_rate": 3.561825712243297e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.578125,
      "logps/chosen": -496.0,
      "logps/rejected": -640.0,
      "loss": 0.404,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -4.5625,
      "step": 11400
    },
    {
      "epoch": 0.4247082689694962,
      "grad_norm": 12.474456968973234,
      "learning_rate": 3.5588839753911964e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.734375,
      "logps/chosen": -536.0,
      "logps/rejected": -676.0,
      "loss": 0.4049,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.015625,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.46875,
      "step": 11410
    },
    {
      "epoch": 0.42508049356982003,
      "grad_norm": 9.766601136358892,
      "learning_rate": 3.555940450786926e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.6875,
      "logps/chosen": -470.0,
      "logps/rejected": -600.0,
      "loss": 0.4256,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -4.25,
      "step": 11420
    },
    {
      "epoch": 0.42545271817014385,
      "grad_norm": 9.20608886425079,
      "learning_rate": 3.5529951434001453e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.484375,
      "logps/chosen": -478.0,
      "logps/rejected": -636.0,
      "loss": 0.4173,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.21875,
      "step": 11430
    },
    {
      "epoch": 0.4258249427704677,
      "grad_norm": 9.748051823605172,
      "learning_rate": 3.5500480582035236e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.6875,
      "logps/chosen": -486.0,
      "logps/rejected": -604.0,
      "loss": 0.418,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.296875,
      "rewards/rejected": -4.125,
      "step": 11440
    },
    {
      "epoch": 0.42619716737079155,
      "grad_norm": 10.352080826617827,
      "learning_rate": 3.547099200172731e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.734375,
      "logps/chosen": -502.0,
      "logps/rejected": -612.0,
      "loss": 0.4157,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.328125,
      "rewards/rejected": -4.25,
      "step": 11450
    },
    {
      "epoch": 0.4265693919711154,
      "grad_norm": 8.861891778649293,
      "learning_rate": 3.544148574286432e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.796875,
      "logps/chosen": -480.0,
      "logps/rejected": -600.0,
      "loss": 0.4179,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.15625,
      "step": 11460
    },
    {
      "epoch": 0.4269416165714392,
      "grad_norm": 8.954623350046194,
      "learning_rate": 3.541196185526275e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.765625,
      "logps/chosen": -488.0,
      "logps/rejected": -596.0,
      "loss": 0.3994,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.09375,
      "step": 11470
    },
    {
      "epoch": 0.427313841171763,
      "grad_norm": 9.934015164908645,
      "learning_rate": 3.538242038876885e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.640625,
      "logps/chosen": -512.0,
      "logps/rejected": -644.0,
      "loss": 0.4209,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -4.3125,
      "step": 11480
    },
    {
      "epoch": 0.4276860657720869,
      "grad_norm": 8.947247206170509,
      "learning_rate": 3.5352861393258535e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.6875,
      "logps/chosen": -472.0,
      "logps/rejected": -608.0,
      "loss": 0.4237,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.25,
      "step": 11490
    },
    {
      "epoch": 0.4280582903724107,
      "grad_norm": 9.176060075562368,
      "learning_rate": 3.5323284918637346e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.59375,
      "logps/chosen": -490.0,
      "logps/rejected": -624.0,
      "loss": 0.4042,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.25,
      "step": 11500
    },
    {
      "epoch": 0.42843051497273454,
      "grad_norm": 9.570768158157147,
      "learning_rate": 3.529369101484031e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -458.0,
      "logps/rejected": -628.0,
      "loss": 0.429,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.671875,
      "rewards/margins": 1.65625,
      "rewards/rejected": -4.34375,
      "step": 11510
    },
    {
      "epoch": 0.42880273957305837,
      "grad_norm": 9.850431673098951,
      "learning_rate": 3.526407973183188e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.71875,
      "logps/chosen": -496.0,
      "logps/rejected": -664.0,
      "loss": 0.3948,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -4.5625,
      "step": 11520
    },
    {
      "epoch": 0.42917496417338225,
      "grad_norm": 11.584102013170881,
      "learning_rate": 3.5234451119605864e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.765625,
      "logps/chosen": -544.0,
      "logps/rejected": -672.0,
      "loss": 0.3947,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -4.78125,
      "step": 11530
    },
    {
      "epoch": 0.42954718877370607,
      "grad_norm": 10.530217731147992,
      "learning_rate": 3.520480522818532e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.78125,
      "logps/chosen": -504.0,
      "logps/rejected": -668.0,
      "loss": 0.4014,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -4.75,
      "step": 11540
    },
    {
      "epoch": 0.4299194133740299,
      "grad_norm": 10.101201505087621,
      "learning_rate": 3.517514210762248e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.84375,
      "logps/chosen": -516.0,
      "logps/rejected": -644.0,
      "loss": 0.4453,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.46875,
      "step": 11550
    },
    {
      "epoch": 0.4302916379743537,
      "grad_norm": 9.615342651329323,
      "learning_rate": 3.514546180799867e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.875,
      "logps/chosen": -494.0,
      "logps/rejected": -632.0,
      "loss": 0.4264,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -4.4375,
      "step": 11560
    },
    {
      "epoch": 0.43066386257467754,
      "grad_norm": 10.50348054830761,
      "learning_rate": 3.5115764379424204e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.640625,
      "logps/chosen": -478.0,
      "logps/rejected": -612.0,
      "loss": 0.4058,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.15625,
      "step": 11570
    },
    {
      "epoch": 0.4310360871750014,
      "grad_norm": 10.640751731237353,
      "learning_rate": 3.508604987203834e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.6875,
      "logps/chosen": -504.0,
      "logps/rejected": -648.0,
      "loss": 0.4219,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.5,
      "step": 11580
    },
    {
      "epoch": 0.43140831177532524,
      "grad_norm": 10.771217132878187,
      "learning_rate": 3.505631833600914e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.71875,
      "logps/chosen": -552.0,
      "logps/rejected": -640.0,
      "loss": 0.4004,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.28125,
      "rewards/rejected": -4.46875,
      "step": 11590
    },
    {
      "epoch": 0.43178053637564906,
      "grad_norm": 9.731858607684682,
      "learning_rate": 3.5026569821533445e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.5625,
      "logps/chosen": -528.0,
      "logps/rejected": -704.0,
      "loss": 0.3822,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -4.9375,
      "step": 11600
    },
    {
      "epoch": 0.4321527609759729,
      "grad_norm": 8.891847546775562,
      "learning_rate": 3.4996804378836755e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.765625,
      "logps/chosen": -486.0,
      "logps/rejected": -652.0,
      "loss": 0.4211,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.84375,
      "rewards/rejected": -4.78125,
      "step": 11610
    },
    {
      "epoch": 0.43252498557629676,
      "grad_norm": 10.530498233296516,
      "learning_rate": 3.496702205817313e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.6875,
      "logps/chosen": -516.0,
      "logps/rejected": -644.0,
      "loss": 0.3844,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.296875,
      "rewards/rejected": -4.3125,
      "step": 11620
    },
    {
      "epoch": 0.4328972101766206,
      "grad_norm": 10.262608325322335,
      "learning_rate": 3.4937222909825153e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.640625,
      "logps/chosen": -500.0,
      "logps/rejected": -652.0,
      "loss": 0.4168,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -4.53125,
      "step": 11630
    },
    {
      "epoch": 0.4332694347769444,
      "grad_norm": 9.86311213879416,
      "learning_rate": 3.4907406984103794e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.828125,
      "logps/chosen": -490.0,
      "logps/rejected": -624.0,
      "loss": 0.4007,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.375,
      "step": 11640
    },
    {
      "epoch": 0.43364165937726823,
      "grad_norm": 11.914482573463843,
      "learning_rate": 3.4877574331348376e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.703125,
      "logps/chosen": -504.0,
      "logps/rejected": -636.0,
      "loss": 0.4084,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.53125,
      "step": 11650
    },
    {
      "epoch": 0.43401388397759205,
      "grad_norm": 8.556590876035028,
      "learning_rate": 3.484772500192643e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -490.0,
      "logps/rejected": -644.0,
      "loss": 0.4158,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.4375,
      "step": 11660
    },
    {
      "epoch": 0.43438610857791593,
      "grad_norm": 10.016427354004175,
      "learning_rate": 3.481785904623368e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.6875,
      "logps/chosen": -494.0,
      "logps/rejected": -600.0,
      "loss": 0.418,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.3125,
      "step": 11670
    },
    {
      "epoch": 0.43475833317823975,
      "grad_norm": 9.815491388909157,
      "learning_rate": 3.4787976514693873e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5,
      "logps/chosen": -496.0,
      "logps/rejected": -632.0,
      "loss": 0.4053,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.375,
      "step": 11680
    },
    {
      "epoch": 0.4351305577785636,
      "grad_norm": 11.332650379221162,
      "learning_rate": 3.475807745775879e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.65625,
      "logps/chosen": -520.0,
      "logps/rejected": -656.0,
      "loss": 0.4226,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.4375,
      "step": 11690
    },
    {
      "epoch": 0.4355027823788874,
      "grad_norm": 9.485381970065637,
      "learning_rate": 3.4728161925908093e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.71875,
      "logps/chosen": -520.0,
      "logps/rejected": -652.0,
      "loss": 0.4147,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.5,
      "step": 11700
    },
    {
      "epoch": 0.4358750069792113,
      "grad_norm": 10.068361020054198,
      "learning_rate": 3.4698229969649246e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5625,
      "logps/chosen": -488.0,
      "logps/rejected": -608.0,
      "loss": 0.4194,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.125,
      "step": 11710
    },
    {
      "epoch": 0.4362472315795351,
      "grad_norm": 9.60773107679875,
      "learning_rate": 3.466828163951747e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.75,
      "logps/chosen": -484.0,
      "logps/rejected": -612.0,
      "loss": 0.4125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -4.1875,
      "step": 11720
    },
    {
      "epoch": 0.4366194561798589,
      "grad_norm": 9.191516968946305,
      "learning_rate": 3.46383169860756e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.71875,
      "logps/chosen": -488.0,
      "logps/rejected": -600.0,
      "loss": 0.3994,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.3125,
      "rewards/rejected": -4.125,
      "step": 11730
    },
    {
      "epoch": 0.43699168078018275,
      "grad_norm": 10.294887246032577,
      "learning_rate": 3.460833605991405e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -508.0,
      "logps/rejected": -644.0,
      "loss": 0.4,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.4375,
      "step": 11740
    },
    {
      "epoch": 0.4373639053805066,
      "grad_norm": 11.4119505725153,
      "learning_rate": 3.4578338911650713e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.421875,
      "logps/chosen": -520.0,
      "logps/rejected": -676.0,
      "loss": 0.445,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.75,
      "step": 11750
    },
    {
      "epoch": 0.43773612998083045,
      "grad_norm": 13.208931424894557,
      "learning_rate": 3.454832559193085e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.328125,
      "logps/chosen": -520.0,
      "logps/rejected": -672.0,
      "loss": 0.3942,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -4.8125,
      "step": 11760
    },
    {
      "epoch": 0.43810835458115427,
      "grad_norm": 8.531470110101804,
      "learning_rate": 3.4518296151427036e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.578125,
      "logps/chosen": -512.0,
      "logps/rejected": -668.0,
      "loss": 0.3852,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.6875,
      "rewards/rejected": -4.625,
      "step": 11770
    },
    {
      "epoch": 0.4384805791814781,
      "grad_norm": 10.618890584999773,
      "learning_rate": 3.448825064083907e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -524.0,
      "logps/rejected": -648.0,
      "loss": 0.4036,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.5,
      "step": 11780
    },
    {
      "epoch": 0.4388528037818019,
      "grad_norm": 11.289949431544436,
      "learning_rate": 3.445818911089387e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.390625,
      "logps/chosen": -476.0,
      "logps/rejected": -624.0,
      "loss": 0.3945,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -4.40625,
      "step": 11790
    },
    {
      "epoch": 0.4392250283821258,
      "grad_norm": 9.298476297422878,
      "learning_rate": 3.4428111612345413e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -498.0,
      "logps/rejected": -652.0,
      "loss": 0.4115,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.625,
      "rewards/rejected": -4.34375,
      "step": 11800
    },
    {
      "epoch": 0.4395972529824496,
      "grad_norm": 10.083557041928156,
      "learning_rate": 3.439801819597462e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.671875,
      "logps/chosen": -508.0,
      "logps/rejected": -628.0,
      "loss": 0.4273,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.328125,
      "rewards/rejected": -4.28125,
      "step": 11810
    },
    {
      "epoch": 0.43996947758277344,
      "grad_norm": 10.018249130783463,
      "learning_rate": 3.4367908912589307e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.75,
      "logps/chosen": -464.0,
      "logps/rejected": -580.0,
      "loss": 0.4075,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.28125,
      "rewards/rejected": -4.03125,
      "step": 11820
    },
    {
      "epoch": 0.44034170218309726,
      "grad_norm": 9.880858415714359,
      "learning_rate": 3.4337783813024057e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -486.0,
      "logps/rejected": -628.0,
      "loss": 0.4163,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.21875,
      "step": 11830
    },
    {
      "epoch": 0.44071392678342114,
      "grad_norm": 9.608139154299762,
      "learning_rate": 3.430764294814017e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.625,
      "logps/chosen": -520.0,
      "logps/rejected": -644.0,
      "loss": 0.3952,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.59375,
      "step": 11840
    },
    {
      "epoch": 0.44108615138374496,
      "grad_norm": 12.29283250727194,
      "learning_rate": 3.4277486368825557e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.6875,
      "logps/chosen": -510.0,
      "logps/rejected": -652.0,
      "loss": 0.4192,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.5625,
      "step": 11850
    },
    {
      "epoch": 0.4414583759840688,
      "grad_norm": 12.405663513264933,
      "learning_rate": 3.4247314125994677e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.421875,
      "logps/chosen": -504.0,
      "logps/rejected": -668.0,
      "loss": 0.3847,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.5625,
      "step": 11860
    },
    {
      "epoch": 0.4418306005843926,
      "grad_norm": 9.971569529580664,
      "learning_rate": 3.421712627058841e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.59375,
      "logps/chosen": -492.0,
      "logps/rejected": -604.0,
      "loss": 0.4209,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.3125,
      "step": 11870
    },
    {
      "epoch": 0.44220282518471643,
      "grad_norm": 10.27055069441987,
      "learning_rate": 3.4186922853573996e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.328125,
      "logps/chosen": -486.0,
      "logps/rejected": -592.0,
      "loss": 0.3934,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.25,
      "step": 11880
    },
    {
      "epoch": 0.4425750497850403,
      "grad_norm": 11.444544006343502,
      "learning_rate": 3.415670392594498e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -506.0,
      "logps/rejected": -648.0,
      "loss": 0.4194,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.671875,
      "rewards/rejected": -4.625,
      "step": 11890
    },
    {
      "epoch": 0.44294727438536413,
      "grad_norm": 10.621136445884924,
      "learning_rate": 3.4126469538721053e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -478.0,
      "logps/rejected": -624.0,
      "loss": 0.4247,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.34375,
      "step": 11900
    },
    {
      "epoch": 0.44331949898568795,
      "grad_norm": 10.644088791502156,
      "learning_rate": 3.409621974294804e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.640625,
      "logps/chosen": -516.0,
      "logps/rejected": -664.0,
      "loss": 0.3822,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.78125,
      "rewards/rejected": -4.78125,
      "step": 11910
    },
    {
      "epoch": 0.4436917235860118,
      "grad_norm": 9.402100119686166,
      "learning_rate": 3.4065954589697753e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.59375,
      "logps/chosen": -478.0,
      "logps/rejected": -604.0,
      "loss": 0.4373,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.421875,
      "rewards/rejected": -4.21875,
      "step": 11920
    },
    {
      "epoch": 0.44406394818633566,
      "grad_norm": 9.266345157373188,
      "learning_rate": 3.403567413006796e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.53125,
      "logps/chosen": -490.0,
      "logps/rejected": -628.0,
      "loss": 0.4118,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.8125,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.34375,
      "step": 11930
    },
    {
      "epoch": 0.4444361727866595,
      "grad_norm": 9.53993114955968,
      "learning_rate": 3.400537841518224e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.6875,
      "logps/chosen": -484.0,
      "logps/rejected": -620.0,
      "loss": 0.4006,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.1875,
      "step": 11940
    },
    {
      "epoch": 0.4448083973869833,
      "grad_norm": 9.723362616126831,
      "learning_rate": 3.3975067496189963e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.78125,
      "logps/chosen": -500.0,
      "logps/rejected": -652.0,
      "loss": 0.3985,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -4.5625,
      "step": 11950
    },
    {
      "epoch": 0.4451806219873071,
      "grad_norm": 9.596968509834058,
      "learning_rate": 3.394474142426615e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.84375,
      "logps/chosen": -528.0,
      "logps/rejected": -680.0,
      "loss": 0.3813,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -4.625,
      "step": 11960
    },
    {
      "epoch": 0.445552846587631,
      "grad_norm": 12.252415190434464,
      "learning_rate": 3.39144002506114e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.640625,
      "logps/chosen": -536.0,
      "logps/rejected": -676.0,
      "loss": 0.4275,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.625,
      "step": 11970
    },
    {
      "epoch": 0.4459250711879548,
      "grad_norm": 10.382091443647473,
      "learning_rate": 3.388404402645182e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.84375,
      "logps/chosen": -504.0,
      "logps/rejected": -632.0,
      "loss": 0.3988,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.796875,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.4375,
      "step": 11980
    },
    {
      "epoch": 0.44629729578827865,
      "grad_norm": 11.264385118897998,
      "learning_rate": 3.3853672803038916e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.734375,
      "logps/chosen": -516.0,
      "logps/rejected": -640.0,
      "loss": 0.4176,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -4.4375,
      "step": 11990
    },
    {
      "epoch": 0.44666952038860247,
      "grad_norm": 10.273859551302094,
      "learning_rate": 3.382328663164954e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.8125,
      "logps/chosen": -524.0,
      "logps/rejected": -660.0,
      "loss": 0.4148,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.5625,
      "step": 12000
    },
    {
      "epoch": 0.4470417449889263,
      "grad_norm": 11.191022968925884,
      "learning_rate": 3.379288556358574e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.6875,
      "logps/chosen": -490.0,
      "logps/rejected": -624.0,
      "loss": 0.4249,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.375,
      "step": 12010
    },
    {
      "epoch": 0.44741396958925017,
      "grad_norm": 9.287152638809891,
      "learning_rate": 3.376246965017476e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.921875,
      "logps/chosen": -502.0,
      "logps/rejected": -636.0,
      "loss": 0.4019,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.625,
      "rewards/rejected": -4.46875,
      "step": 12020
    },
    {
      "epoch": 0.447786194189574,
      "grad_norm": 10.165945333753767,
      "learning_rate": 3.3732038942768895e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.546875,
      "logps/chosen": -512.0,
      "logps/rejected": -636.0,
      "loss": 0.4048,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.40625,
      "step": 12030
    },
    {
      "epoch": 0.4481584187898978,
      "grad_norm": 11.740840710151364,
      "learning_rate": 3.3701593492745386e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.515625,
      "logps/chosen": -480.0,
      "logps/rejected": -608.0,
      "loss": 0.422,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.375,
      "step": 12040
    },
    {
      "epoch": 0.44853064339022164,
      "grad_norm": 12.65597376815739,
      "learning_rate": 3.36711333515064e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.53125,
      "logps/chosen": -544.0,
      "logps/rejected": -672.0,
      "loss": 0.4083,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -4.75,
      "step": 12050
    },
    {
      "epoch": 0.4489028679905455,
      "grad_norm": 10.613131285592296,
      "learning_rate": 3.364065857047891e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.546875,
      "logps/chosen": -516.0,
      "logps/rejected": -644.0,
      "loss": 0.4198,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.40625,
      "step": 12060
    },
    {
      "epoch": 0.44927509259086934,
      "grad_norm": 10.076508273810601,
      "learning_rate": 3.361016920111458e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.75,
      "logps/chosen": -468.0,
      "logps/rejected": -604.0,
      "loss": 0.3926,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.21875,
      "step": 12070
    },
    {
      "epoch": 0.44964731719119316,
      "grad_norm": 11.603943366147567,
      "learning_rate": 3.3579665294889714e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.640625,
      "logps/chosen": -500.0,
      "logps/rejected": -648.0,
      "loss": 0.4084,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -4.5625,
      "step": 12080
    },
    {
      "epoch": 0.450019541791517,
      "grad_norm": 9.852645690212018,
      "learning_rate": 3.354914690330517e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.578125,
      "logps/chosen": -516.0,
      "logps/rejected": -672.0,
      "loss": 0.4082,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.640625,
      "rewards/rejected": -4.78125,
      "step": 12090
    },
    {
      "epoch": 0.45039176639184086,
      "grad_norm": 8.646923111344188,
      "learning_rate": 3.3518614077886246e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.578125,
      "logps/chosen": -462.0,
      "logps/rejected": -596.0,
      "loss": 0.4056,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.640625,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.21875,
      "step": 12100
    },
    {
      "epoch": 0.4507639909921647,
      "grad_norm": 12.977805299497534,
      "learning_rate": 3.348806687018262e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.671875,
      "logps/chosen": -512.0,
      "logps/rejected": -632.0,
      "loss": 0.4076,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.21875,
      "step": 12110
    },
    {
      "epoch": 0.4511362155924885,
      "grad_norm": 13.816501795384543,
      "learning_rate": 3.3457505331768254e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.4375,
      "logps/chosen": -496.0,
      "logps/rejected": -648.0,
      "loss": 0.4338,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.546875,
      "rewards/rejected": -4.46875,
      "step": 12120
    },
    {
      "epoch": 0.45150844019281233,
      "grad_norm": 9.923995652342334,
      "learning_rate": 3.3426929514241294e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.734375,
      "logps/chosen": -548.0,
      "logps/rejected": -660.0,
      "loss": 0.389,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.5625,
      "step": 12130
    },
    {
      "epoch": 0.45188066479313616,
      "grad_norm": 11.21447984884347,
      "learning_rate": 3.3396339469224e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.46875,
      "logps/chosen": -476.0,
      "logps/rejected": -620.0,
      "loss": 0.4232,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.21875,
      "step": 12140
    },
    {
      "epoch": 0.45225288939346003,
      "grad_norm": 9.74973259756532,
      "learning_rate": 3.336573524836266e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.609375,
      "logps/chosen": -520.0,
      "logps/rejected": -652.0,
      "loss": 0.4072,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.6875,
      "step": 12150
    },
    {
      "epoch": 0.45262511399378386,
      "grad_norm": 10.252427269797703,
      "learning_rate": 3.3335116903327485e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.578125,
      "logps/chosen": -516.0,
      "logps/rejected": -668.0,
      "loss": 0.4003,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.625,
      "rewards/rejected": -4.59375,
      "step": 12160
    },
    {
      "epoch": 0.4529973385941077,
      "grad_norm": 10.6451235409799,
      "learning_rate": 3.330448448581254e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.453125,
      "logps/chosen": -490.0,
      "logps/rejected": -632.0,
      "loss": 0.4235,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.34375,
      "step": 12170
    },
    {
      "epoch": 0.4533695631944315,
      "grad_norm": 13.81507400326777,
      "learning_rate": 3.327383804753563e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.46875,
      "logps/chosen": -488.0,
      "logps/rejected": -652.0,
      "loss": 0.4125,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -4.28125,
      "step": 12180
    },
    {
      "epoch": 0.4537417877947554,
      "grad_norm": 10.206486190690597,
      "learning_rate": 3.3243177640238263e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.4375,
      "logps/chosen": -500.0,
      "logps/rejected": -652.0,
      "loss": 0.4017,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.828125,
      "rewards/rejected": -4.71875,
      "step": 12190
    },
    {
      "epoch": 0.4541140123950792,
      "grad_norm": 9.962865323517494,
      "learning_rate": 3.32125033156855e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.484375,
      "logps/chosen": -532.0,
      "logps/rejected": -656.0,
      "loss": 0.4047,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.5625,
      "step": 12200
    },
    {
      "epoch": 0.454486236995403,
      "grad_norm": 11.497897284294858,
      "learning_rate": 3.3181815125665943e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.484375,
      "logps/chosen": -532.0,
      "logps/rejected": -676.0,
      "loss": 0.3921,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -4.875,
      "step": 12210
    },
    {
      "epoch": 0.45485846159572685,
      "grad_norm": 9.573492753866224,
      "learning_rate": 3.315111312199154e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.453125,
      "logps/chosen": -544.0,
      "logps/rejected": -680.0,
      "loss": 0.404,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.75,
      "step": 12220
    },
    {
      "epoch": 0.45523068619605067,
      "grad_norm": 9.459883549187747,
      "learning_rate": 3.312039735649761e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.671875,
      "logps/chosen": -488.0,
      "logps/rejected": -636.0,
      "loss": 0.4091,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.75,
      "rewards/rejected": -4.53125,
      "step": 12230
    },
    {
      "epoch": 0.45560291079637455,
      "grad_norm": 10.23121856183525,
      "learning_rate": 3.308966788104271e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5625,
      "logps/chosen": -516.0,
      "logps/rejected": -628.0,
      "loss": 0.4284,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.3125,
      "rewards/rejected": -4.375,
      "step": 12240
    },
    {
      "epoch": 0.45597513539669837,
      "grad_norm": 11.122912952226395,
      "learning_rate": 3.305892474750849e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.53125,
      "logps/chosen": -478.0,
      "logps/rejected": -604.0,
      "loss": 0.409,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.59375,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.03125,
      "step": 12250
    },
    {
      "epoch": 0.4563473599970222,
      "grad_norm": 10.1499167152992,
      "learning_rate": 3.3028168007799736e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.765625,
      "logps/chosen": -476.0,
      "logps/rejected": -600.0,
      "loss": 0.4277,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -4.125,
      "step": 12260
    },
    {
      "epoch": 0.456719584597346,
      "grad_norm": 10.136218228331419,
      "learning_rate": 3.299739771384413e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -488.0,
      "logps/rejected": -636.0,
      "loss": 0.4218,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -4.3125,
      "step": 12270
    },
    {
      "epoch": 0.4570918091976699,
      "grad_norm": 10.005230514749659,
      "learning_rate": 3.296661391759229e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.3125,
      "logps/chosen": -478.0,
      "logps/rejected": -628.0,
      "loss": 0.4086,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.1875,
      "step": 12280
    },
    {
      "epoch": 0.4574640337979937,
      "grad_norm": 10.306577902099768,
      "learning_rate": 3.2935816671017624e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.578125,
      "logps/chosen": -482.0,
      "logps/rejected": -624.0,
      "loss": 0.3841,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.375,
      "step": 12290
    },
    {
      "epoch": 0.45783625839831754,
      "grad_norm": 42.21886797718441,
      "learning_rate": 3.290500602611622e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.40625,
      "logps/chosen": -524.0,
      "logps/rejected": -672.0,
      "loss": 0.4124,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.53125,
      "step": 12300
    },
    {
      "epoch": 0.45820848299864136,
      "grad_norm": 11.46522145275124,
      "learning_rate": 3.2874182034906816e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.6875,
      "logps/chosen": -516.0,
      "logps/rejected": -640.0,
      "loss": 0.4187,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.46875,
      "step": 12310
    },
    {
      "epoch": 0.45858070759896524,
      "grad_norm": 10.351333814168004,
      "learning_rate": 3.2843344749430666e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.53125,
      "logps/chosen": -508.0,
      "logps/rejected": -620.0,
      "loss": 0.4157,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.3125,
      "rewards/rejected": -4.21875,
      "step": 12320
    },
    {
      "epoch": 0.45895293219928907,
      "grad_norm": 10.209673050125849,
      "learning_rate": 3.28124942217515e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.484375,
      "logps/chosen": -478.0,
      "logps/rejected": -624.0,
      "loss": 0.4133,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.734375,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -4.375,
      "step": 12330
    },
    {
      "epoch": 0.4593251567996129,
      "grad_norm": 10.101398018581268,
      "learning_rate": 3.2781630503955344e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.609375,
      "logps/chosen": -520.0,
      "logps/rejected": -636.0,
      "loss": 0.398,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.109375,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.5,
      "step": 12340
    },
    {
      "epoch": 0.4596973813999367,
      "grad_norm": 10.525621875903928,
      "learning_rate": 3.275075364815056e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.75,
      "logps/chosen": -536.0,
      "logps/rejected": -664.0,
      "loss": 0.3961,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.546875,
      "rewards/rejected": -4.71875,
      "step": 12350
    },
    {
      "epoch": 0.46006960600026053,
      "grad_norm": 11.13491051307432,
      "learning_rate": 3.271986370646764e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.640625,
      "logps/chosen": -536.0,
      "logps/rejected": -680.0,
      "loss": 0.4004,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.71875,
      "step": 12360
    },
    {
      "epoch": 0.4604418306005844,
      "grad_norm": 9.5023854347198,
      "learning_rate": 3.2688960731059204e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.84375,
      "logps/chosen": -508.0,
      "logps/rejected": -644.0,
      "loss": 0.3795,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.5625,
      "step": 12370
    },
    {
      "epoch": 0.46081405520090823,
      "grad_norm": 12.094519066210644,
      "learning_rate": 3.2658044774099875e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5625,
      "logps/chosen": -548.0,
      "logps/rejected": -700.0,
      "loss": 0.409,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.0,
      "step": 12380
    },
    {
      "epoch": 0.46118627980123206,
      "grad_norm": 14.204867754457572,
      "learning_rate": 3.262711588778616e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.71875,
      "logps/chosen": -536.0,
      "logps/rejected": -668.0,
      "loss": 0.401,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.421875,
      "rewards/rejected": -4.6875,
      "step": 12390
    },
    {
      "epoch": 0.4615585044015559,
      "grad_norm": 12.168081871316623,
      "learning_rate": 3.259617412433644e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -532.0,
      "logps/rejected": -656.0,
      "loss": 0.4195,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.6875,
      "step": 12400
    },
    {
      "epoch": 0.46193072900187976,
      "grad_norm": 10.415914875912112,
      "learning_rate": 3.25652195359908e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.546875,
      "logps/chosen": -516.0,
      "logps/rejected": -656.0,
      "loss": 0.396,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.59375,
      "rewards/rejected": -4.5625,
      "step": 12410
    },
    {
      "epoch": 0.4623029536022036,
      "grad_norm": 11.664126923603071,
      "learning_rate": 3.2534252175010994e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.484375,
      "logps/chosen": -504.0,
      "logps/rejected": -616.0,
      "loss": 0.424,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.28125,
      "step": 12420
    },
    {
      "epoch": 0.4626751782025274,
      "grad_norm": 9.85334740434483,
      "learning_rate": 3.2503272093680347e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.515625,
      "logps/chosen": -476.0,
      "logps/rejected": -596.0,
      "loss": 0.4131,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.75,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.125,
      "step": 12430
    },
    {
      "epoch": 0.4630474028028512,
      "grad_norm": 10.067575049017021,
      "learning_rate": 3.247227934430364e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -504.0,
      "logps/rejected": -648.0,
      "loss": 0.3826,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.53125,
      "step": 12440
    },
    {
      "epoch": 0.46341962740317505,
      "grad_norm": 11.56884295174365,
      "learning_rate": 3.2441273979207074e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.421875,
      "logps/chosen": -486.0,
      "logps/rejected": -636.0,
      "loss": 0.403,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.4375,
      "step": 12450
    },
    {
      "epoch": 0.46379185200349893,
      "grad_norm": 11.194270549533254,
      "learning_rate": 3.2410256050738107e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.46875,
      "logps/chosen": -516.0,
      "logps/rejected": -680.0,
      "loss": 0.4033,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.6875,
      "rewards/rejected": -4.8125,
      "step": 12460
    },
    {
      "epoch": 0.46416407660382275,
      "grad_norm": 10.293188998175731,
      "learning_rate": 3.237922561126545e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.671875,
      "logps/chosen": -516.0,
      "logps/rejected": -676.0,
      "loss": 0.3972,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -4.71875,
      "step": 12470
    },
    {
      "epoch": 0.4645363012041466,
      "grad_norm": 12.51584367017726,
      "learning_rate": 3.23481827131789e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -506.0,
      "logps/rejected": -664.0,
      "loss": 0.425,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.59375,
      "step": 12480
    },
    {
      "epoch": 0.4649085258044704,
      "grad_norm": 9.525962525607687,
      "learning_rate": 3.2317127408889327e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.46875,
      "logps/chosen": -512.0,
      "logps/rejected": -672.0,
      "loss": 0.4102,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.6875,
      "rewards/rejected": -4.5,
      "step": 12490
    },
    {
      "epoch": 0.4652807504047943,
      "grad_norm": 9.887296345226147,
      "learning_rate": 3.228605975082851e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.703125,
      "logps/chosen": -504.0,
      "logps/rejected": -612.0,
      "loss": 0.3965,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.3125,
      "rewards/rejected": -4.25,
      "step": 12500
    },
    {
      "epoch": 0.4656529750051181,
      "grad_norm": 10.945235410031048,
      "learning_rate": 3.225497979144911e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.609375,
      "logps/chosen": -516.0,
      "logps/rejected": -636.0,
      "loss": 0.4134,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.421875,
      "rewards/rejected": -4.5,
      "step": 12510
    },
    {
      "epoch": 0.4660251996054419,
      "grad_norm": 10.365413716299447,
      "learning_rate": 3.222388758322454e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.796875,
      "logps/chosen": -540.0,
      "logps/rejected": -676.0,
      "loss": 0.428,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.65625,
      "rewards/rejected": -4.78125,
      "step": 12520
    },
    {
      "epoch": 0.46639742420576574,
      "grad_norm": 9.86713440485834,
      "learning_rate": 3.219278317864891e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.546875,
      "logps/chosen": -488.0,
      "logps/rejected": -628.0,
      "loss": 0.4244,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -4.28125,
      "step": 12530
    },
    {
      "epoch": 0.4667696488060896,
      "grad_norm": 12.67364537859243,
      "learning_rate": 3.2161666630236913e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.515625,
      "logps/chosen": -524.0,
      "logps/rejected": -644.0,
      "loss": 0.39,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -4.3125,
      "step": 12540
    },
    {
      "epoch": 0.46714187340641344,
      "grad_norm": 10.329377336041905,
      "learning_rate": 3.213053799052373e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.59375,
      "logps/chosen": -490.0,
      "logps/rejected": -636.0,
      "loss": 0.3807,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -4.4375,
      "step": 12550
    },
    {
      "epoch": 0.46751409800673727,
      "grad_norm": 8.413615359846679,
      "learning_rate": 3.209939731206499e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.578125,
      "logps/chosen": -492.0,
      "logps/rejected": -632.0,
      "loss": 0.4104,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.015625,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.53125,
      "step": 12560
    },
    {
      "epoch": 0.4678863226070611,
      "grad_norm": 11.114551604030282,
      "learning_rate": 3.206824464743662e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -536.0,
      "logps/rejected": -676.0,
      "loss": 0.3905,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.75,
      "step": 12570
    },
    {
      "epoch": 0.4682585472073849,
      "grad_norm": 11.046945686181875,
      "learning_rate": 3.2037080049234795e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -508.0,
      "logps/rejected": -632.0,
      "loss": 0.4034,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.59375,
      "step": 12580
    },
    {
      "epoch": 0.4686307718077088,
      "grad_norm": 12.102078514138622,
      "learning_rate": 3.2005903570075826e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.75,
      "logps/chosen": -548.0,
      "logps/rejected": -692.0,
      "loss": 0.4099,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.03125,
      "step": 12590
    },
    {
      "epoch": 0.4690029964080326,
      "grad_norm": 10.759359341171821,
      "learning_rate": 3.197471526259611e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -474.0,
      "logps/rejected": -644.0,
      "loss": 0.4236,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -4.53125,
      "step": 12600
    },
    {
      "epoch": 0.46937522100835644,
      "grad_norm": 10.066023872713469,
      "learning_rate": 3.194351517945197e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.859375,
      "logps/chosen": -510.0,
      "logps/rejected": -612.0,
      "loss": 0.4088,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -4.0625,
      "step": 12610
    },
    {
      "epoch": 0.46974744560868026,
      "grad_norm": 10.316868136491188,
      "learning_rate": 3.1912303373319654e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.734375,
      "logps/chosen": -488.0,
      "logps/rejected": -632.0,
      "loss": 0.4137,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.34375,
      "step": 12620
    },
    {
      "epoch": 0.47011967020900414,
      "grad_norm": 9.611493722001343,
      "learning_rate": 3.1881079896895187e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.625,
      "logps/chosen": -476.0,
      "logps/rejected": -628.0,
      "loss": 0.3966,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.78125,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.3125,
      "step": 12630
    },
    {
      "epoch": 0.47049189480932796,
      "grad_norm": 10.76087567326318,
      "learning_rate": 3.184984480289429e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.65625,
      "logps/chosen": -504.0,
      "logps/rejected": -648.0,
      "loss": 0.388,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.65625,
      "step": 12640
    },
    {
      "epoch": 0.4708641194096518,
      "grad_norm": 11.527541159364649,
      "learning_rate": 3.18185981440523e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.875,
      "logps/chosen": -528.0,
      "logps/rejected": -664.0,
      "loss": 0.4094,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.65625,
      "step": 12650
    },
    {
      "epoch": 0.4712363440099756,
      "grad_norm": 13.178938362760595,
      "learning_rate": 3.1787339973124087e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.640625,
      "logps/chosen": -516.0,
      "logps/rejected": -648.0,
      "loss": 0.4049,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.59375,
      "step": 12660
    },
    {
      "epoch": 0.47160856861029943,
      "grad_norm": 14.615723931449873,
      "learning_rate": 3.1756070342883955e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.453125,
      "logps/chosen": -528.0,
      "logps/rejected": -660.0,
      "loss": 0.412,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.359375,
      "rewards/rejected": -4.5,
      "step": 12670
    },
    {
      "epoch": 0.4719807932106233,
      "grad_norm": 10.72346112789714,
      "learning_rate": 3.1724789306125556e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.65625,
      "logps/chosen": -508.0,
      "logps/rejected": -652.0,
      "loss": 0.3906,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.65625,
      "rewards/rejected": -4.59375,
      "step": 12680
    },
    {
      "epoch": 0.47235301781094713,
      "grad_norm": 11.478147402886373,
      "learning_rate": 3.1693496915661776e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.59375,
      "logps/chosen": -528.0,
      "logps/rejected": -660.0,
      "loss": 0.4093,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.6875,
      "step": 12690
    },
    {
      "epoch": 0.47272524241127095,
      "grad_norm": 11.297685638664312,
      "learning_rate": 3.1662193224324717e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.734375,
      "logps/chosen": -516.0,
      "logps/rejected": -640.0,
      "loss": 0.4226,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.4375,
      "step": 12700
    },
    {
      "epoch": 0.4730974670115948,
      "grad_norm": 8.708204682751672,
      "learning_rate": 3.163087828496553e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.328125,
      "logps/chosen": -516.0,
      "logps/rejected": -644.0,
      "loss": 0.4046,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.109375,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.5,
      "step": 12710
    },
    {
      "epoch": 0.47346969161191865,
      "grad_norm": 10.047003032766604,
      "learning_rate": 3.1599552150454357e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -544.0,
      "logps/rejected": -672.0,
      "loss": 0.3902,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.53125,
      "step": 12720
    },
    {
      "epoch": 0.4738419162122425,
      "grad_norm": 12.140792806656247,
      "learning_rate": 3.1568214873680245e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.640625,
      "logps/chosen": -524.0,
      "logps/rejected": -652.0,
      "loss": 0.4138,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.59375,
      "rewards/rejected": -4.75,
      "step": 12730
    },
    {
      "epoch": 0.4742141408125663,
      "grad_norm": 11.398770806190376,
      "learning_rate": 3.1536866507551047e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -576.0,
      "logps/rejected": -688.0,
      "loss": 0.3915,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -5.03125,
      "step": 12740
    },
    {
      "epoch": 0.4745863654128901,
      "grad_norm": 10.391636003586257,
      "learning_rate": 3.150550710499336e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.53125,
      "logps/chosen": -544.0,
      "logps/rejected": -688.0,
      "loss": 0.386,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.9375,
      "step": 12750
    },
    {
      "epoch": 0.474958590013214,
      "grad_norm": 10.123338584075846,
      "learning_rate": 3.1474136718952384e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.578125,
      "logps/chosen": -540.0,
      "logps/rejected": -688.0,
      "loss": 0.4032,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.96875,
      "step": 12760
    },
    {
      "epoch": 0.4753308146135378,
      "grad_norm": 9.931793501504734,
      "learning_rate": 3.144275540239189e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.546875,
      "logps/chosen": -540.0,
      "logps/rejected": -652.0,
      "loss": 0.4026,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.21875,
      "rewards/rejected": -4.71875,
      "step": 12770
    },
    {
      "epoch": 0.47570303921386164,
      "grad_norm": 11.509461784685843,
      "learning_rate": 3.1411363208294077e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.53125,
      "logps/chosen": -532.0,
      "logps/rejected": -628.0,
      "loss": 0.3939,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.296875,
      "rewards/rejected": -4.46875,
      "step": 12780
    },
    {
      "epoch": 0.47607526381418547,
      "grad_norm": 10.408428030907503,
      "learning_rate": 3.137996018965955e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -532.0,
      "logps/rejected": -672.0,
      "loss": 0.4297,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.65625,
      "step": 12790
    },
    {
      "epoch": 0.4764474884145093,
      "grad_norm": 11.027084293495653,
      "learning_rate": 3.1348546399507137e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5625,
      "logps/chosen": -540.0,
      "logps/rejected": -664.0,
      "loss": 0.4092,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -4.4375,
      "step": 12800
    },
    {
      "epoch": 0.47681971301483317,
      "grad_norm": 11.304247989283922,
      "learning_rate": 3.131712189087389e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.546875,
      "logps/chosen": -510.0,
      "logps/rejected": -636.0,
      "loss": 0.3994,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.59375,
      "step": 12810
    },
    {
      "epoch": 0.477191937615157,
      "grad_norm": 10.123589866192605,
      "learning_rate": 3.128568671681494e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.484375,
      "logps/chosen": -520.0,
      "logps/rejected": -656.0,
      "loss": 0.4204,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.59375,
      "step": 12820
    },
    {
      "epoch": 0.4775641622154808,
      "grad_norm": 11.124533254234715,
      "learning_rate": 3.1254240930403424e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -508.0,
      "logps/rejected": -660.0,
      "loss": 0.3982,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.75,
      "rewards/rejected": -4.9375,
      "step": 12830
    },
    {
      "epoch": 0.47793638681580464,
      "grad_norm": 11.228005892808543,
      "learning_rate": 3.122278458473042e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.484375,
      "logps/chosen": -520.0,
      "logps/rejected": -664.0,
      "loss": 0.3974,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.671875,
      "rewards/rejected": -4.84375,
      "step": 12840
    },
    {
      "epoch": 0.4783086114161285,
      "grad_norm": 10.049594565178314,
      "learning_rate": 3.1191317732904803e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.578125,
      "logps/chosen": -510.0,
      "logps/rejected": -644.0,
      "loss": 0.3953,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.015625,
      "rewards/margins": 1.421875,
      "rewards/rejected": -4.4375,
      "step": 12850
    },
    {
      "epoch": 0.47868083601645234,
      "grad_norm": 9.87205850484632,
      "learning_rate": 3.11598404280532e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.640625,
      "logps/chosen": -524.0,
      "logps/rejected": -624.0,
      "loss": 0.3939,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.015625,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.375,
      "step": 12860
    },
    {
      "epoch": 0.47905306061677616,
      "grad_norm": 11.18276645000694,
      "learning_rate": 3.112835272331989e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.578125,
      "logps/chosen": -548.0,
      "logps/rejected": -668.0,
      "loss": 0.4086,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.84375,
      "step": 12870
    },
    {
      "epoch": 0.4794252852171,
      "grad_norm": 9.807054906829034,
      "learning_rate": 3.1096854671866696e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.453125,
      "logps/chosen": -524.0,
      "logps/rejected": -648.0,
      "loss": 0.4219,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -4.4375,
      "step": 12880
    },
    {
      "epoch": 0.4797975098174238,
      "grad_norm": 9.822726173304444,
      "learning_rate": 3.106534632687293e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.46875,
      "logps/chosen": -524.0,
      "logps/rejected": -636.0,
      "loss": 0.416,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.203125,
      "rewards/rejected": -4.375,
      "step": 12890
    },
    {
      "epoch": 0.4801697344177477,
      "grad_norm": 8.26577330806337,
      "learning_rate": 3.103382774153526e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -512.0,
      "logps/rejected": -628.0,
      "loss": 0.3937,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -4.25,
      "step": 12900
    },
    {
      "epoch": 0.4805419590180715,
      "grad_norm": 13.81618762929805,
      "learning_rate": 3.1002298969067676e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5,
      "logps/chosen": -510.0,
      "logps/rejected": -632.0,
      "loss": 0.426,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.25,
      "rewards/rejected": -4.375,
      "step": 12910
    },
    {
      "epoch": 0.48091418361839533,
      "grad_norm": 11.779445714148306,
      "learning_rate": 3.097076006270132e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -512.0,
      "logps/rejected": -652.0,
      "loss": 0.4077,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.015625,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -4.59375,
      "step": 12920
    },
    {
      "epoch": 0.48128640821871915,
      "grad_norm": 11.17684689082911,
      "learning_rate": 3.093921107568449e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.484375,
      "logps/chosen": -510.0,
      "logps/rejected": -644.0,
      "loss": 0.4093,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.59375,
      "step": 12930
    },
    {
      "epoch": 0.48165863281904303,
      "grad_norm": 9.745149603325487,
      "learning_rate": 3.0907652061282467e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.4375,
      "logps/chosen": -520.0,
      "logps/rejected": -652.0,
      "loss": 0.408,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.6875,
      "step": 12940
    },
    {
      "epoch": 0.48203085741936685,
      "grad_norm": 10.027021831293618,
      "learning_rate": 3.087608307277749e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.59375,
      "logps/chosen": -496.0,
      "logps/rejected": -624.0,
      "loss": 0.3873,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.40625,
      "step": 12950
    },
    {
      "epoch": 0.4824030820196907,
      "grad_norm": 9.654157616107808,
      "learning_rate": 3.0844504163468634e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.78125,
      "logps/chosen": -524.0,
      "logps/rejected": -656.0,
      "loss": 0.4033,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.71875,
      "step": 12960
    },
    {
      "epoch": 0.4827753066200145,
      "grad_norm": 11.5992359884988,
      "learning_rate": 3.081291538667169e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.578125,
      "logps/chosen": -540.0,
      "logps/rejected": -716.0,
      "loss": 0.3971,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.03125,
      "step": 12970
    },
    {
      "epoch": 0.4831475312203384,
      "grad_norm": 10.679603803182971,
      "learning_rate": 3.0781316795719165e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.671875,
      "logps/chosen": -516.0,
      "logps/rejected": -656.0,
      "loss": 0.4138,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.6875,
      "step": 12980
    },
    {
      "epoch": 0.4835197558206622,
      "grad_norm": 9.446194665440425,
      "learning_rate": 3.0749708443960076e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.828125,
      "logps/chosen": -524.0,
      "logps/rejected": -652.0,
      "loss": 0.4267,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.8125,
      "step": 12990
    },
    {
      "epoch": 0.483891980420986,
      "grad_norm": 11.942809620212248,
      "learning_rate": 3.071809038475997e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -516.0,
      "logps/rejected": -668.0,
      "loss": 0.4076,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.75,
      "rewards/rejected": -4.625,
      "step": 13000
    },
    {
      "epoch": 0.48426420502130985,
      "grad_norm": 9.895484015545525,
      "learning_rate": 3.0686462671500744e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.703125,
      "logps/chosen": -536.0,
      "logps/rejected": -648.0,
      "loss": 0.4363,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -4.5625,
      "step": 13010
    },
    {
      "epoch": 0.48463642962163367,
      "grad_norm": 11.754622620341914,
      "learning_rate": 3.065482535758063e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.546875,
      "logps/chosen": -488.0,
      "logps/rejected": -632.0,
      "loss": 0.4186,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.40625,
      "step": 13020
    },
    {
      "epoch": 0.48500865422195755,
      "grad_norm": 10.891584473510711,
      "learning_rate": 3.0623178496414025e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.546875,
      "logps/chosen": -504.0,
      "logps/rejected": -636.0,
      "loss": 0.4108,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.265625,
      "rewards/rejected": -4.46875,
      "step": 13030
    },
    {
      "epoch": 0.48538087882228137,
      "grad_norm": 12.693840620182552,
      "learning_rate": 3.0591522141431495e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5,
      "logps/chosen": -494.0,
      "logps/rejected": -672.0,
      "loss": 0.4224,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -4.65625,
      "step": 13040
    },
    {
      "epoch": 0.4857531034226052,
      "grad_norm": 10.61625824602236,
      "learning_rate": 3.0559856346079605e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -528.0,
      "logps/rejected": -656.0,
      "loss": 0.4082,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.5625,
      "step": 13050
    },
    {
      "epoch": 0.486125328022929,
      "grad_norm": 10.265789793258051,
      "learning_rate": 3.052818116382086e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.375,
      "logps/chosen": -516.0,
      "logps/rejected": -648.0,
      "loss": 0.3967,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.65625,
      "step": 13060
    },
    {
      "epoch": 0.4864975526232529,
      "grad_norm": 11.877650501268057,
      "learning_rate": 3.0496496648133623e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5,
      "logps/chosen": -556.0,
      "logps/rejected": -704.0,
      "loss": 0.3995,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.515625,
      "rewards/rejected": -5.0,
      "step": 13070
    },
    {
      "epoch": 0.4868697772235767,
      "grad_norm": 9.31556878811011,
      "learning_rate": 3.0464802852512e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.625,
      "logps/chosen": -524.0,
      "logps/rejected": -660.0,
      "loss": 0.399,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.65625,
      "rewards/rejected": -4.84375,
      "step": 13080
    },
    {
      "epoch": 0.48724200182390054,
      "grad_norm": 13.511752252560536,
      "learning_rate": 3.0433099830465784e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -540.0,
      "logps/rejected": -656.0,
      "loss": 0.402,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.6875,
      "step": 13090
    },
    {
      "epoch": 0.48761422642422436,
      "grad_norm": 11.054394853275857,
      "learning_rate": 3.040138763552034e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.46875,
      "logps/chosen": -536.0,
      "logps/rejected": -672.0,
      "loss": 0.3841,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.8125,
      "step": 13100
    },
    {
      "epoch": 0.48798645102454824,
      "grad_norm": 10.58175396211903,
      "learning_rate": 3.0369666321216494e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.421875,
      "logps/chosen": -524.0,
      "logps/rejected": -668.0,
      "loss": 0.3937,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.734375,
      "rewards/rejected": -4.90625,
      "step": 13110
    },
    {
      "epoch": 0.48835867562487206,
      "grad_norm": 10.988399191515072,
      "learning_rate": 3.03379359411105e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -512.0,
      "logps/rejected": -648.0,
      "loss": 0.4058,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.546875,
      "rewards/rejected": -4.625,
      "step": 13120
    },
    {
      "epoch": 0.4887309002251959,
      "grad_norm": 10.648214933510312,
      "learning_rate": 3.0306196548773914e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.453125,
      "logps/chosen": -512.0,
      "logps/rejected": -684.0,
      "loss": 0.4166,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.796875,
      "rewards/rejected": -4.84375,
      "step": 13130
    },
    {
      "epoch": 0.4891031248255197,
      "grad_norm": 8.636152773059381,
      "learning_rate": 3.027444819779351e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.515625,
      "logps/chosen": -540.0,
      "logps/rejected": -668.0,
      "loss": 0.4098,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.625,
      "step": 13140
    },
    {
      "epoch": 0.48947534942584353,
      "grad_norm": 10.477912319770926,
      "learning_rate": 3.024269094177116e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.546875,
      "logps/chosen": -532.0,
      "logps/rejected": -652.0,
      "loss": 0.3936,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.28125,
      "rewards/rejected": -4.4375,
      "step": 13150
    },
    {
      "epoch": 0.4898475740261674,
      "grad_norm": 12.268569663341038,
      "learning_rate": 3.0210924834323794e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.484375,
      "logps/chosen": -524.0,
      "logps/rejected": -660.0,
      "loss": 0.3999,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.71875,
      "step": 13160
    },
    {
      "epoch": 0.49021979862649123,
      "grad_norm": 11.271928163988976,
      "learning_rate": 3.0179149929083293e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -520.0,
      "logps/rejected": -648.0,
      "loss": 0.4212,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.625,
      "step": 13170
    },
    {
      "epoch": 0.49059202322681505,
      "grad_norm": 9.429688490698707,
      "learning_rate": 3.014736627969636e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.546875,
      "logps/chosen": -532.0,
      "logps/rejected": -644.0,
      "loss": 0.4139,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -4.5625,
      "step": 13180
    },
    {
      "epoch": 0.4909642478271389,
      "grad_norm": 10.734526651501632,
      "learning_rate": 3.0115573939824507e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -536.0,
      "logps/rejected": -668.0,
      "loss": 0.4125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.265625,
      "rewards/rejected": -4.5625,
      "step": 13190
    },
    {
      "epoch": 0.49133647242746276,
      "grad_norm": 9.465668516074857,
      "learning_rate": 3.0083772963143866e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -520.0,
      "logps/rejected": -648.0,
      "loss": 0.4149,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.40625,
      "step": 13200
    },
    {
      "epoch": 0.4917086970277866,
      "grad_norm": 10.182805056995917,
      "learning_rate": 3.00519634033452e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.640625,
      "logps/chosen": -520.0,
      "logps/rejected": -660.0,
      "loss": 0.3819,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.546875,
      "rewards/rejected": -4.5,
      "step": 13210
    },
    {
      "epoch": 0.4920809216281104,
      "grad_norm": 9.040467362535427,
      "learning_rate": 3.002014531413374e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.5,
      "logps/chosen": -528.0,
      "logps/rejected": -672.0,
      "loss": 0.414,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.671875,
      "rewards/rejected": -4.8125,
      "step": 13220
    },
    {
      "epoch": 0.4924531462284342,
      "grad_norm": 9.262080797218843,
      "learning_rate": 2.99883187492291e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -500.0,
      "logps/rejected": -652.0,
      "loss": 0.4071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.65625,
      "step": 13230
    },
    {
      "epoch": 0.49282537082875805,
      "grad_norm": 10.023304301813573,
      "learning_rate": 2.995648376236524e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.421875,
      "logps/chosen": -490.0,
      "logps/rejected": -620.0,
      "loss": 0.4127,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.28125,
      "step": 13240
    },
    {
      "epoch": 0.4931975954290819,
      "grad_norm": 9.766891351020508,
      "learning_rate": 2.992464040729031e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.484375,
      "logps/chosen": -506.0,
      "logps/rejected": -632.0,
      "loss": 0.4018,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.34375,
      "step": 13250
    },
    {
      "epoch": 0.49356982002940575,
      "grad_norm": 9.71669931359203,
      "learning_rate": 2.989278873776661e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.53125,
      "logps/chosen": -524.0,
      "logps/rejected": -644.0,
      "loss": 0.4273,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.4375,
      "step": 13260
    },
    {
      "epoch": 0.49394204462972957,
      "grad_norm": 8.94419421296106,
      "learning_rate": 2.986092880757046e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.546875,
      "logps/chosen": -500.0,
      "logps/rejected": -604.0,
      "loss": 0.3909,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.296875,
      "rewards/rejected": -4.21875,
      "step": 13270
    },
    {
      "epoch": 0.4943142692300534,
      "grad_norm": 9.613891613637715,
      "learning_rate": 2.9829060670492134e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.59375,
      "logps/chosen": -532.0,
      "logps/rejected": -660.0,
      "loss": 0.4037,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.4375,
      "step": 13280
    },
    {
      "epoch": 0.49468649383037727,
      "grad_norm": 10.393575351255263,
      "learning_rate": 2.9797184380335763e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.453125,
      "logps/chosen": -510.0,
      "logps/rejected": -668.0,
      "loss": 0.3962,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.734375,
      "rewards/rejected": -4.6875,
      "step": 13290
    },
    {
      "epoch": 0.4950587184307011,
      "grad_norm": 10.714823567345197,
      "learning_rate": 2.976529999091924e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.59375,
      "logps/chosen": -498.0,
      "logps/rejected": -660.0,
      "loss": 0.3935,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.78125,
      "rewards/rejected": -4.75,
      "step": 13300
    },
    {
      "epoch": 0.4954309430310249,
      "grad_norm": 12.554927575175762,
      "learning_rate": 2.9733407556074146e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.578125,
      "logps/chosen": -516.0,
      "logps/rejected": -660.0,
      "loss": 0.3946,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.6875,
      "rewards/rejected": -4.78125,
      "step": 13310
    },
    {
      "epoch": 0.49580316763134874,
      "grad_norm": 11.210295209597918,
      "learning_rate": 2.9701507129645625e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -536.0,
      "logps/rejected": -668.0,
      "loss": 0.4137,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.6875,
      "step": 13320
    },
    {
      "epoch": 0.4961753922316726,
      "grad_norm": 10.778072150309274,
      "learning_rate": 2.966959876549233e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.671875,
      "logps/chosen": -528.0,
      "logps/rejected": -648.0,
      "loss": 0.3853,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -4.53125,
      "step": 13330
    },
    {
      "epoch": 0.49654761683199644,
      "grad_norm": 9.710156361240516,
      "learning_rate": 2.9637682517486284e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -512.0,
      "logps/rejected": -672.0,
      "loss": 0.3909,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.734375,
      "rewards/rejected": -4.78125,
      "step": 13340
    },
    {
      "epoch": 0.49691984143232026,
      "grad_norm": 11.15263178780461,
      "learning_rate": 2.9605758439512874e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.671875,
      "logps/chosen": -528.0,
      "logps/rejected": -672.0,
      "loss": 0.3961,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -4.90625,
      "step": 13350
    },
    {
      "epoch": 0.4972920660326441,
      "grad_norm": 12.904528040819834,
      "learning_rate": 2.957382658547066e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.640625,
      "logps/chosen": -540.0,
      "logps/rejected": -676.0,
      "loss": 0.3949,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.03125,
      "step": 13360
    },
    {
      "epoch": 0.4976642906329679,
      "grad_norm": 11.148464177627579,
      "learning_rate": 2.9541887009271354e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.71875,
      "logps/chosen": -552.0,
      "logps/rejected": -688.0,
      "loss": 0.3935,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -5.09375,
      "step": 13370
    },
    {
      "epoch": 0.4980365152332918,
      "grad_norm": 13.330696095792772,
      "learning_rate": 2.95099397648397e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.6875,
      "logps/chosen": -520.0,
      "logps/rejected": -652.0,
      "loss": 0.437,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.71875,
      "step": 13380
    },
    {
      "epoch": 0.4984087398336156,
      "grad_norm": 12.055970773708623,
      "learning_rate": 2.947798490611339e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -504.0,
      "logps/rejected": -660.0,
      "loss": 0.4148,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.5625,
      "step": 13390
    },
    {
      "epoch": 0.49878096443393943,
      "grad_norm": 13.225155971936921,
      "learning_rate": 2.9446022487042966e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.46875,
      "logps/chosen": -506.0,
      "logps/rejected": -640.0,
      "loss": 0.4092,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -4.46875,
      "step": 13400
    },
    {
      "epoch": 0.49915318903426326,
      "grad_norm": 10.743153603437303,
      "learning_rate": 2.9414052561591727e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.59375,
      "logps/chosen": -520.0,
      "logps/rejected": -644.0,
      "loss": 0.3942,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.59375,
      "step": 13410
    },
    {
      "epoch": 0.49952541363458713,
      "grad_norm": 10.00076555167372,
      "learning_rate": 2.9382075183735675e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.515625,
      "logps/chosen": -552.0,
      "logps/rejected": -712.0,
      "loss": 0.3987,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.0,
      "step": 13420
    },
    {
      "epoch": 0.49989763823491096,
      "grad_norm": 11.307977571286317,
      "learning_rate": 2.9350090407463363e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -540.0,
      "logps/rejected": -684.0,
      "loss": 0.397,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.875,
      "step": 13430
    },
    {
      "epoch": 0.5002698628352348,
      "grad_norm": 11.750040566555116,
      "learning_rate": 2.9318098286775857e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.515625,
      "logps/chosen": -520.0,
      "logps/rejected": -664.0,
      "loss": 0.422,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.625,
      "step": 13440
    },
    {
      "epoch": 0.5006420874355586,
      "grad_norm": 11.502620336500394,
      "learning_rate": 2.92860988756866e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5,
      "logps/chosen": -498.0,
      "logps/rejected": -632.0,
      "loss": 0.4138,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -4.28125,
      "step": 13450
    },
    {
      "epoch": 0.5010143120358824,
      "grad_norm": 10.016074213241748,
      "learning_rate": 2.9254092228221365e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.703125,
      "logps/chosen": -508.0,
      "logps/rejected": -632.0,
      "loss": 0.3802,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.90625,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.46875,
      "step": 13460
    },
    {
      "epoch": 0.5013865366362062,
      "grad_norm": 10.130185317743589,
      "learning_rate": 2.922207839841814e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5625,
      "logps/chosen": -492.0,
      "logps/rejected": -628.0,
      "loss": 0.3774,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.84375,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.4375,
      "step": 13470
    },
    {
      "epoch": 0.5017587612365301,
      "grad_norm": 11.00004965575782,
      "learning_rate": 2.919005744032702e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5625,
      "logps/chosen": -544.0,
      "logps/rejected": -668.0,
      "loss": 0.3815,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.90625,
      "step": 13480
    },
    {
      "epoch": 0.502130985836854,
      "grad_norm": 11.030440967356249,
      "learning_rate": 2.915802940801016e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.53125,
      "logps/chosen": -548.0,
      "logps/rejected": -652.0,
      "loss": 0.4298,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -4.59375,
      "step": 13490
    },
    {
      "epoch": 0.5025032104371778,
      "grad_norm": 9.714543841641737,
      "learning_rate": 2.9125994355541637e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.4375,
      "logps/chosen": -504.0,
      "logps/rejected": -656.0,
      "loss": 0.3861,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.59375,
      "rewards/rejected": -4.625,
      "step": 13500
    },
    {
      "epoch": 0.5028754350375017,
      "grad_norm": 10.761709546754888,
      "learning_rate": 2.9093952337007404e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.375,
      "logps/chosen": -536.0,
      "logps/rejected": -652.0,
      "loss": 0.3908,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.3125,
      "rewards/rejected": -4.5625,
      "step": 13510
    },
    {
      "epoch": 0.5032476596378255,
      "grad_norm": 9.605524325185442,
      "learning_rate": 2.9061903406505153e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.640625,
      "logps/chosen": -536.0,
      "logps/rejected": -632.0,
      "loss": 0.4004,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.65625,
      "step": 13520
    },
    {
      "epoch": 0.5036198842381493,
      "grad_norm": 11.695390795811319,
      "learning_rate": 2.9029847618144243e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.515625,
      "logps/chosen": -510.0,
      "logps/rejected": -644.0,
      "loss": 0.4309,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.46875,
      "step": 13530
    },
    {
      "epoch": 0.5039921088384731,
      "grad_norm": 11.522719137691588,
      "learning_rate": 2.8997785026045643e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.40625,
      "logps/chosen": -516.0,
      "logps/rejected": -668.0,
      "loss": 0.4121,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.65625,
      "step": 13540
    },
    {
      "epoch": 0.5043643334387969,
      "grad_norm": 12.156383768231338,
      "learning_rate": 2.8965715684341767e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.421875,
      "logps/chosen": -512.0,
      "logps/rejected": -684.0,
      "loss": 0.415,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -4.875,
      "step": 13550
    },
    {
      "epoch": 0.5047365580391208,
      "grad_norm": 10.191718250960458,
      "learning_rate": 2.893363964717647e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -548.0,
      "logps/rejected": -704.0,
      "loss": 0.4143,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.8125,
      "step": 13560
    },
    {
      "epoch": 0.5051087826394446,
      "grad_norm": 9.400465337716618,
      "learning_rate": 2.8901556968704875e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.734375,
      "logps/chosen": -548.0,
      "logps/rejected": -648.0,
      "loss": 0.4131,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.109375,
      "rewards/rejected": -4.4375,
      "step": 13570
    },
    {
      "epoch": 0.5054810072397685,
      "grad_norm": 10.709979957020575,
      "learning_rate": 2.886946770309333e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.703125,
      "logps/chosen": -532.0,
      "logps/rejected": -656.0,
      "loss": 0.3781,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.625,
      "step": 13580
    },
    {
      "epoch": 0.5058532318400923,
      "grad_norm": 11.041133964489664,
      "learning_rate": 2.8837371904519315e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.796875,
      "logps/chosen": -524.0,
      "logps/rejected": -664.0,
      "loss": 0.3874,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.84375,
      "step": 13590
    },
    {
      "epoch": 0.5062254564404162,
      "grad_norm": 11.908939732067688,
      "learning_rate": 2.88052696271713e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.734375,
      "logps/chosen": -560.0,
      "logps/rejected": -672.0,
      "loss": 0.4199,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -4.8125,
      "step": 13600
    },
    {
      "epoch": 0.50659768104074,
      "grad_norm": 10.972351639025932,
      "learning_rate": 2.877316092524876e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.609375,
      "logps/chosen": -580.0,
      "logps/rejected": -696.0,
      "loss": 0.4098,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.90625,
      "step": 13610
    },
    {
      "epoch": 0.5069699056410638,
      "grad_norm": 9.800382376551802,
      "learning_rate": 2.8741045852961955e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -520.0,
      "logps/rejected": -648.0,
      "loss": 0.4052,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.5625,
      "step": 13620
    },
    {
      "epoch": 0.5073421302413876,
      "grad_norm": 11.65258984252477,
      "learning_rate": 2.870892446453193e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.546875,
      "logps/chosen": -502.0,
      "logps/rejected": -664.0,
      "loss": 0.3896,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -4.71875,
      "step": 13630
    },
    {
      "epoch": 0.5077143548417115,
      "grad_norm": 9.190150285224146,
      "learning_rate": 2.867679681419039e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -560.0,
      "logps/rejected": -688.0,
      "loss": 0.3699,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.875,
      "step": 13640
    },
    {
      "epoch": 0.5080865794420353,
      "grad_norm": 13.391896136497547,
      "learning_rate": 2.8644662956179617e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.453125,
      "logps/chosen": -544.0,
      "logps/rejected": -664.0,
      "loss": 0.4133,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -4.59375,
      "step": 13650
    },
    {
      "epoch": 0.5084588040423592,
      "grad_norm": 12.090915396209757,
      "learning_rate": 2.861252294475236e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.65625,
      "logps/chosen": -596.0,
      "logps/rejected": -744.0,
      "loss": 0.3891,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.84375,
      "rewards/rejected": -5.40625,
      "step": 13660
    },
    {
      "epoch": 0.508831028642683,
      "grad_norm": 12.580430965316948,
      "learning_rate": 2.858037683417176e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -544.0,
      "logps/rejected": -680.0,
      "loss": 0.418,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.90625,
      "step": 13670
    },
    {
      "epoch": 0.5092032532430069,
      "grad_norm": 12.02983735571647,
      "learning_rate": 2.8548224678711276e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.546875,
      "logps/chosen": -532.0,
      "logps/rejected": -652.0,
      "loss": 0.4105,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.5625,
      "step": 13680
    },
    {
      "epoch": 0.5095754778433307,
      "grad_norm": 13.285091814760671,
      "learning_rate": 2.8516066532654536e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.6875,
      "logps/chosen": -580.0,
      "logps/rejected": -720.0,
      "loss": 0.4077,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.0625,
      "step": 13690
    },
    {
      "epoch": 0.5099477024436545,
      "grad_norm": 9.895365174727138,
      "learning_rate": 2.8483902450295324e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.59375,
      "logps/chosen": -524.0,
      "logps/rejected": -680.0,
      "loss": 0.3851,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.796875,
      "rewards/rejected": -4.9375,
      "step": 13700
    },
    {
      "epoch": 0.5103199270439783,
      "grad_norm": 10.580595340598114,
      "learning_rate": 2.8451732485937405e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.609375,
      "logps/chosen": -532.0,
      "logps/rejected": -632.0,
      "loss": 0.3947,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.46875,
      "step": 13710
    },
    {
      "epoch": 0.5106921516443022,
      "grad_norm": 11.428308530119892,
      "learning_rate": 2.841955669389451e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.609375,
      "logps/chosen": -520.0,
      "logps/rejected": -644.0,
      "loss": 0.4051,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.71875,
      "step": 13720
    },
    {
      "epoch": 0.511064376244626,
      "grad_norm": 9.218226478874573,
      "learning_rate": 2.838737512849019e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.765625,
      "logps/chosen": -548.0,
      "logps/rejected": -680.0,
      "loss": 0.4016,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.8125,
      "step": 13730
    },
    {
      "epoch": 0.5114366008449498,
      "grad_norm": 13.382842045973717,
      "learning_rate": 2.8355187844057736e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -540.0,
      "logps/rejected": -672.0,
      "loss": 0.3973,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.875,
      "step": 13740
    },
    {
      "epoch": 0.5118088254452737,
      "grad_norm": 11.608498129581927,
      "learning_rate": 2.8322994894940127e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.78125,
      "logps/chosen": -544.0,
      "logps/rejected": -644.0,
      "loss": 0.4246,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.25,
      "rewards/rejected": -4.75,
      "step": 13750
    },
    {
      "epoch": 0.5121810500455976,
      "grad_norm": 9.51620644030257,
      "learning_rate": 2.829079633548987e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.421875,
      "logps/chosen": -576.0,
      "logps/rejected": -704.0,
      "loss": 0.4185,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.09375,
      "step": 13760
    },
    {
      "epoch": 0.5125532746459214,
      "grad_norm": 12.947789165764844,
      "learning_rate": 2.8258592220068966e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -516.0,
      "logps/rejected": -648.0,
      "loss": 0.4204,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.59375,
      "step": 13770
    },
    {
      "epoch": 0.5129254992462452,
      "grad_norm": 10.767857924419452,
      "learning_rate": 2.8226382603048784e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.640625,
      "logps/chosen": -532.0,
      "logps/rejected": -652.0,
      "loss": 0.4005,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.328125,
      "rewards/rejected": -4.53125,
      "step": 13780
    },
    {
      "epoch": 0.513297723846569,
      "grad_norm": 11.092320924362014,
      "learning_rate": 2.819416753880999e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.640625,
      "logps/chosen": -520.0,
      "logps/rejected": -640.0,
      "loss": 0.4032,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.375,
      "step": 13790
    },
    {
      "epoch": 0.5136699484468928,
      "grad_norm": 12.404723014954202,
      "learning_rate": 2.8161947081742447e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -528.0,
      "logps/rejected": -664.0,
      "loss": 0.4128,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.59375,
      "step": 13800
    },
    {
      "epoch": 0.5140421730472167,
      "grad_norm": 13.144102203199274,
      "learning_rate": 2.812972128624511e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -536.0,
      "logps/rejected": -668.0,
      "loss": 0.3959,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.65625,
      "step": 13810
    },
    {
      "epoch": 0.5144143976475405,
      "grad_norm": 8.532305406978567,
      "learning_rate": 2.8097490206725976e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.578125,
      "logps/chosen": -508.0,
      "logps/rejected": -640.0,
      "loss": 0.4025,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.59375,
      "step": 13820
    },
    {
      "epoch": 0.5147866222478643,
      "grad_norm": 11.898406112246471,
      "learning_rate": 2.806525389760192e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5625,
      "logps/chosen": -520.0,
      "logps/rejected": -704.0,
      "loss": 0.4195,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.6875,
      "rewards/rejected": -4.90625,
      "step": 13830
    },
    {
      "epoch": 0.5151588468481882,
      "grad_norm": 11.173847322046855,
      "learning_rate": 2.803301241329869e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -528.0,
      "logps/rejected": -652.0,
      "loss": 0.4056,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.46875,
      "step": 13840
    },
    {
      "epoch": 0.5155310714485121,
      "grad_norm": 11.393142704256274,
      "learning_rate": 2.800076580825074e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -500.0,
      "logps/rejected": -640.0,
      "loss": 0.4026,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.5,
      "step": 13850
    },
    {
      "epoch": 0.5159032960488359,
      "grad_norm": 11.057877321115464,
      "learning_rate": 2.796851413690119e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5625,
      "logps/chosen": -528.0,
      "logps/rejected": -676.0,
      "loss": 0.4072,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.78125,
      "step": 13860
    },
    {
      "epoch": 0.5162755206491597,
      "grad_norm": 14.449340412726023,
      "learning_rate": 2.7936257453701693e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -524.0,
      "logps/rejected": -676.0,
      "loss": 0.4006,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -4.9375,
      "step": 13870
    },
    {
      "epoch": 0.5166477452494835,
      "grad_norm": 13.12870003583221,
      "learning_rate": 2.790399581311238e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.53125,
      "logps/chosen": -516.0,
      "logps/rejected": -668.0,
      "loss": 0.409,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -4.96875,
      "step": 13880
    },
    {
      "epoch": 0.5170199698498074,
      "grad_norm": 10.42534857847161,
      "learning_rate": 2.7871729269601763e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.4375,
      "logps/chosen": -528.0,
      "logps/rejected": -692.0,
      "loss": 0.3931,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.671875,
      "rewards/rejected": -4.84375,
      "step": 13890
    },
    {
      "epoch": 0.5173921944501312,
      "grad_norm": 9.56044220915324,
      "learning_rate": 2.7839457877646587e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.59375,
      "logps/chosen": -516.0,
      "logps/rejected": -652.0,
      "loss": 0.3843,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.59375,
      "step": 13900
    },
    {
      "epoch": 0.517764419050455,
      "grad_norm": 11.552293662279222,
      "learning_rate": 2.780718169173184e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.6875,
      "logps/chosen": -512.0,
      "logps/rejected": -660.0,
      "loss": 0.388,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.015625,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -4.78125,
      "step": 13910
    },
    {
      "epoch": 0.5181366436507788,
      "grad_norm": 12.881962831777097,
      "learning_rate": 2.7774900766350564e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.75,
      "logps/chosen": -536.0,
      "logps/rejected": -656.0,
      "loss": 0.4206,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.328125,
      "rewards/rejected": -4.59375,
      "step": 13920
    },
    {
      "epoch": 0.5185088682511028,
      "grad_norm": 10.638108403365319,
      "learning_rate": 2.7742615156003805e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.484375,
      "logps/chosen": -556.0,
      "logps/rejected": -712.0,
      "loss": 0.3906,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.03125,
      "step": 13930
    },
    {
      "epoch": 0.5188810928514266,
      "grad_norm": 10.758841566292167,
      "learning_rate": 2.7710324915200546e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -536.0,
      "logps/rejected": -676.0,
      "loss": 0.409,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.546875,
      "rewards/rejected": -4.75,
      "step": 13940
    },
    {
      "epoch": 0.5192533174517504,
      "grad_norm": 11.757394817756559,
      "learning_rate": 2.767803009845756e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -516.0,
      "logps/rejected": -656.0,
      "loss": 0.4077,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.5,
      "step": 13950
    },
    {
      "epoch": 0.5196255420520742,
      "grad_norm": 12.14578750345256,
      "learning_rate": 2.764573076029935e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.546875,
      "logps/chosen": -564.0,
      "logps/rejected": -676.0,
      "loss": 0.3918,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.6875,
      "step": 13960
    },
    {
      "epoch": 0.519997766652398,
      "grad_norm": 13.093678010859824,
      "learning_rate": 2.7613426955258074e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.578125,
      "logps/chosen": -540.0,
      "logps/rejected": -656.0,
      "loss": 0.3951,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.359375,
      "rewards/rejected": -4.65625,
      "step": 13970
    },
    {
      "epoch": 0.5203699912527219,
      "grad_norm": 10.337294263385294,
      "learning_rate": 2.758111873787341e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.796875,
      "logps/chosen": -552.0,
      "logps/rejected": -652.0,
      "loss": 0.427,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.28125,
      "rewards/rejected": -4.6875,
      "step": 13980
    },
    {
      "epoch": 0.5207422158530457,
      "grad_norm": 11.461635321978326,
      "learning_rate": 2.754880616269248e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -536.0,
      "logps/rejected": -636.0,
      "loss": 0.4012,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -4.4375,
      "step": 13990
    },
    {
      "epoch": 0.5211144404533695,
      "grad_norm": 11.162084038118728,
      "learning_rate": 2.75164892842698e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.359375,
      "logps/chosen": -544.0,
      "logps/rejected": -704.0,
      "loss": 0.3727,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.0,
      "step": 14000
    },
    {
      "epoch": 0.5214866650536933,
      "grad_norm": 13.28139915139561,
      "learning_rate": 2.7484168157167095e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.484375,
      "logps/chosen": -560.0,
      "logps/rejected": -708.0,
      "loss": 0.4069,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -5.125,
      "step": 14010
    },
    {
      "epoch": 0.5218588896540173,
      "grad_norm": 10.905202706275512,
      "learning_rate": 2.7451842835953326e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.390625,
      "logps/chosen": -536.0,
      "logps/rejected": -684.0,
      "loss": 0.4263,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.78125,
      "step": 14020
    },
    {
      "epoch": 0.5222311142543411,
      "grad_norm": 10.960264407119798,
      "learning_rate": 2.7419513375204485e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.515625,
      "logps/chosen": -556.0,
      "logps/rejected": -684.0,
      "loss": 0.3788,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.96875,
      "step": 14030
    },
    {
      "epoch": 0.5226033388546649,
      "grad_norm": 16.031889316516537,
      "learning_rate": 2.738717982950357e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5,
      "logps/chosen": -536.0,
      "logps/rejected": -668.0,
      "loss": 0.4112,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.875,
      "step": 14040
    },
    {
      "epoch": 0.5229755634549887,
      "grad_norm": 11.124696414047365,
      "learning_rate": 2.7354842253440484e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.625,
      "logps/chosen": -544.0,
      "logps/rejected": -692.0,
      "loss": 0.3702,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.0625,
      "step": 14050
    },
    {
      "epoch": 0.5233477880553126,
      "grad_norm": 13.932414387798751,
      "learning_rate": 2.7322500701611926e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.671875,
      "logps/chosen": -560.0,
      "logps/rejected": -712.0,
      "loss": 0.3913,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -5.25,
      "step": 14060
    },
    {
      "epoch": 0.5237200126556364,
      "grad_norm": 10.274700270685361,
      "learning_rate": 2.7290155228621315e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -544.0,
      "logps/rejected": -692.0,
      "loss": 0.3891,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.1875,
      "step": 14070
    },
    {
      "epoch": 0.5240922372559602,
      "grad_norm": 11.93405165909286,
      "learning_rate": 2.7257805889078677e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.375,
      "logps/chosen": -544.0,
      "logps/rejected": -720.0,
      "loss": 0.4072,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -5.125,
      "step": 14080
    },
    {
      "epoch": 0.524464461856284,
      "grad_norm": 11.55262864313798,
      "learning_rate": 2.7225452737600574e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5625,
      "logps/chosen": -494.0,
      "logps/rejected": -644.0,
      "loss": 0.4017,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -4.5625,
      "step": 14090
    },
    {
      "epoch": 0.524836686456608,
      "grad_norm": 10.914266072428223,
      "learning_rate": 2.7193095828810016e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.453125,
      "logps/chosen": -544.0,
      "logps/rejected": -664.0,
      "loss": 0.4042,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.8125,
      "step": 14100
    },
    {
      "epoch": 0.5252089110569318,
      "grad_norm": 17.305554309062174,
      "learning_rate": 2.7160735217336314e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -524.0,
      "logps/rejected": -708.0,
      "loss": 0.3975,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.953125,
      "rewards/rejected": -5.1875,
      "step": 14110
    },
    {
      "epoch": 0.5255811356572556,
      "grad_norm": 13.368492900048402,
      "learning_rate": 2.71283709578151e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.46875,
      "logps/chosen": -544.0,
      "logps/rejected": -708.0,
      "loss": 0.3915,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.1875,
      "step": 14120
    },
    {
      "epoch": 0.5259533602575794,
      "grad_norm": 12.869540202338149,
      "learning_rate": 2.709600310488809e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -552.0,
      "logps/rejected": -696.0,
      "loss": 0.3889,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.09375,
      "step": 14130
    },
    {
      "epoch": 0.5263255848579033,
      "grad_norm": 10.090878327190481,
      "learning_rate": 2.706363171320313e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -572.0,
      "logps/rejected": -680.0,
      "loss": 0.4077,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.9375,
      "step": 14140
    },
    {
      "epoch": 0.5266978094582271,
      "grad_norm": 11.242431063240378,
      "learning_rate": 2.7031256837414e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -548.0,
      "logps/rejected": -700.0,
      "loss": 0.4058,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.03125,
      "step": 14150
    },
    {
      "epoch": 0.5270700340585509,
      "grad_norm": 11.295425402351867,
      "learning_rate": 2.6998878532180374e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -560.0,
      "logps/rejected": -692.0,
      "loss": 0.3884,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.421875,
      "rewards/rejected": -5.0,
      "step": 14160
    },
    {
      "epoch": 0.5274422586588747,
      "grad_norm": 12.094347560521838,
      "learning_rate": 2.6966496852167723e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.46875,
      "logps/chosen": -572.0,
      "logps/rejected": -696.0,
      "loss": 0.4402,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.875,
      "step": 14170
    },
    {
      "epoch": 0.5278144832591986,
      "grad_norm": 10.078336375594612,
      "learning_rate": 2.693411185204721e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -560.0,
      "logps/rejected": -676.0,
      "loss": 0.4111,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.8125,
      "step": 14180
    },
    {
      "epoch": 0.5281867078595225,
      "grad_norm": 10.483565268062478,
      "learning_rate": 2.69017235864956e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -512.0,
      "logps/rejected": -644.0,
      "loss": 0.4046,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.984375,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.46875,
      "step": 14190
    },
    {
      "epoch": 0.5285589324598463,
      "grad_norm": 11.496393040761557,
      "learning_rate": 2.686933211019517e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -490.0,
      "logps/rejected": -644.0,
      "loss": 0.3908,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.875,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.46875,
      "step": 14200
    },
    {
      "epoch": 0.5289311570601701,
      "grad_norm": 9.636358878914821,
      "learning_rate": 2.6836937477833637e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -532.0,
      "logps/rejected": -668.0,
      "loss": 0.4006,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.65625,
      "step": 14210
    },
    {
      "epoch": 0.529303381660494,
      "grad_norm": 13.19825892462097,
      "learning_rate": 2.680453974410402e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.625,
      "logps/chosen": -494.0,
      "logps/rejected": -628.0,
      "loss": 0.4204,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.53125,
      "step": 14220
    },
    {
      "epoch": 0.5296756062608178,
      "grad_norm": 11.638298820879593,
      "learning_rate": 2.677213896370459e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -484.0,
      "logps/rejected": -632.0,
      "loss": 0.4045,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.671875,
      "rewards/rejected": -4.5,
      "step": 14230
    },
    {
      "epoch": 0.5300478308611416,
      "grad_norm": 10.546874104475124,
      "learning_rate": 2.6739735191338765e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.875,
      "logps/chosen": -524.0,
      "logps/rejected": -644.0,
      "loss": 0.3942,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -4.5,
      "step": 14240
    },
    {
      "epoch": 0.5304200554614654,
      "grad_norm": 11.726324897061103,
      "learning_rate": 2.6707328481714997e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -560.0,
      "logps/rejected": -696.0,
      "loss": 0.4148,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -5.0,
      "step": 14250
    },
    {
      "epoch": 0.5307922800617892,
      "grad_norm": 12.560858705420372,
      "learning_rate": 2.6674918889546713e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.625,
      "logps/chosen": -536.0,
      "logps/rejected": -680.0,
      "loss": 0.3879,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -4.96875,
      "step": 14260
    },
    {
      "epoch": 0.5311645046621131,
      "grad_norm": 13.04516395123105,
      "learning_rate": 2.6642506469552197e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5625,
      "logps/chosen": -548.0,
      "logps/rejected": -688.0,
      "loss": 0.3945,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.0,
      "step": 14270
    },
    {
      "epoch": 0.531536729262437,
      "grad_norm": 11.494409337425308,
      "learning_rate": 2.6610091276454523e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.546875,
      "logps/chosen": -520.0,
      "logps/rejected": -664.0,
      "loss": 0.4068,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.6875,
      "step": 14280
    },
    {
      "epoch": 0.5319089538627608,
      "grad_norm": 9.7541175549173,
      "learning_rate": 2.657767336498142e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.734375,
      "logps/chosen": -544.0,
      "logps/rejected": -708.0,
      "loss": 0.3904,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -5.21875,
      "step": 14290
    },
    {
      "epoch": 0.5322811784630846,
      "grad_norm": 9.661131247961436,
      "learning_rate": 2.654525278986523e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.6875,
      "logps/chosen": -528.0,
      "logps/rejected": -640.0,
      "loss": 0.4066,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -4.59375,
      "step": 14300
    },
    {
      "epoch": 0.5326534030634085,
      "grad_norm": 9.296730028549963,
      "learning_rate": 2.651282960584279e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -508.0,
      "logps/rejected": -640.0,
      "loss": 0.4244,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -4.46875,
      "step": 14310
    },
    {
      "epoch": 0.5330256276637323,
      "grad_norm": 10.845087706163364,
      "learning_rate": 2.6480403867655325e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.46875,
      "logps/chosen": -516.0,
      "logps/rejected": -648.0,
      "loss": 0.4072,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -4.65625,
      "step": 14320
    },
    {
      "epoch": 0.5333978522640561,
      "grad_norm": 10.48398915076274,
      "learning_rate": 2.644797563004839e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.375,
      "logps/chosen": -524.0,
      "logps/rejected": -652.0,
      "loss": 0.4082,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.5625,
      "step": 14330
    },
    {
      "epoch": 0.5337700768643799,
      "grad_norm": 10.644691610379436,
      "learning_rate": 2.641554494777175e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -544.0,
      "logps/rejected": -712.0,
      "loss": 0.3864,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.734375,
      "rewards/rejected": -4.96875,
      "step": 14340
    },
    {
      "epoch": 0.5341423014647038,
      "grad_norm": 9.943326643314396,
      "learning_rate": 2.6383111875579313e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.53125,
      "logps/chosen": -540.0,
      "logps/rejected": -676.0,
      "loss": 0.3867,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.8125,
      "step": 14350
    },
    {
      "epoch": 0.5345145260650276,
      "grad_norm": 12.48129870801112,
      "learning_rate": 2.635067646822898e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.359375,
      "logps/chosen": -544.0,
      "logps/rejected": -688.0,
      "loss": 0.3953,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.0,
      "step": 14360
    },
    {
      "epoch": 0.5348867506653515,
      "grad_norm": 11.768608638727255,
      "learning_rate": 2.631823878048266e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -532.0,
      "logps/rejected": -680.0,
      "loss": 0.4145,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -4.9375,
      "step": 14370
    },
    {
      "epoch": 0.5352589752656753,
      "grad_norm": 11.378494713620181,
      "learning_rate": 2.628579886710605e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.390625,
      "logps/chosen": -552.0,
      "logps/rejected": -684.0,
      "loss": 0.4004,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.875,
      "step": 14380
    },
    {
      "epoch": 0.5356311998659992,
      "grad_norm": 12.858935519838798,
      "learning_rate": 2.625335678286864e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.40625,
      "logps/chosen": -544.0,
      "logps/rejected": -680.0,
      "loss": 0.4046,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.875,
      "step": 14390
    },
    {
      "epoch": 0.536003424466323,
      "grad_norm": 9.746735911907042,
      "learning_rate": 2.622091258254358e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.65625,
      "logps/chosen": -532.0,
      "logps/rejected": -692.0,
      "loss": 0.375,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -5.0,
      "step": 14400
    },
    {
      "epoch": 0.5363756490666468,
      "grad_norm": 10.54527932079587,
      "learning_rate": 2.618846632090758e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.515625,
      "logps/chosen": -524.0,
      "logps/rejected": -660.0,
      "loss": 0.3932,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.9375,
      "step": 14410
    },
    {
      "epoch": 0.5367478736669706,
      "grad_norm": 10.894925568110256,
      "learning_rate": 2.6156018052740844e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.4375,
      "logps/chosen": -528.0,
      "logps/rejected": -668.0,
      "loss": 0.4169,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.8125,
      "step": 14420
    },
    {
      "epoch": 0.5371200982672945,
      "grad_norm": 10.792921717930996,
      "learning_rate": 2.6123567832826955e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.4375,
      "logps/chosen": -520.0,
      "logps/rejected": -664.0,
      "loss": 0.4035,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.59375,
      "rewards/rejected": -4.6875,
      "step": 14430
    },
    {
      "epoch": 0.5374923228676183,
      "grad_norm": 11.796602518906829,
      "learning_rate": 2.60911157159528e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.34375,
      "logps/chosen": -496.0,
      "logps/rejected": -624.0,
      "loss": 0.4114,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.53125,
      "step": 14440
    },
    {
      "epoch": 0.5378645474679422,
      "grad_norm": 9.998080560884492,
      "learning_rate": 2.6058661756908463e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -500.0,
      "logps/rejected": -624.0,
      "loss": 0.3799,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.890625,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -4.3125,
      "step": 14450
    },
    {
      "epoch": 0.538236772068266,
      "grad_norm": 9.689784761170017,
      "learning_rate": 2.6026206010487135e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -506.0,
      "logps/rejected": -632.0,
      "loss": 0.3927,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.4375,
      "step": 14460
    },
    {
      "epoch": 0.5386089966685899,
      "grad_norm": 10.95227871362134,
      "learning_rate": 2.599374853148502e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.6875,
      "logps/chosen": -532.0,
      "logps/rejected": -664.0,
      "loss": 0.388,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.78125,
      "step": 14470
    },
    {
      "epoch": 0.5389812212689137,
      "grad_norm": 11.447742808292213,
      "learning_rate": 2.596128937470127e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.578125,
      "logps/chosen": -560.0,
      "logps/rejected": -696.0,
      "loss": 0.4075,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.15625,
      "step": 14480
    },
    {
      "epoch": 0.5393534458692375,
      "grad_norm": 11.139772060588125,
      "learning_rate": 2.592882859493785e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -540.0,
      "logps/rejected": -696.0,
      "loss": 0.3921,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -5.03125,
      "step": 14490
    },
    {
      "epoch": 0.5397256704695613,
      "grad_norm": 10.488311762026338,
      "learning_rate": 2.5896366246999474e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.453125,
      "logps/chosen": -528.0,
      "logps/rejected": -676.0,
      "loss": 0.3938,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.75,
      "step": 14500
    },
    {
      "epoch": 0.5400978950698851,
      "grad_norm": 12.027577583782012,
      "learning_rate": 2.586390238569349e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.40625,
      "logps/chosen": -492.0,
      "logps/rejected": -648.0,
      "loss": 0.406,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.109375,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.625,
      "step": 14510
    },
    {
      "epoch": 0.540470119670209,
      "grad_norm": 11.66960201849853,
      "learning_rate": 2.583143706582983e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.53125,
      "logps/chosen": -488.0,
      "logps/rejected": -628.0,
      "loss": 0.3988,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.59375,
      "step": 14520
    },
    {
      "epoch": 0.5408423442705328,
      "grad_norm": 10.19646154500491,
      "learning_rate": 2.579897034222084e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.625,
      "logps/chosen": -548.0,
      "logps/rejected": -720.0,
      "loss": 0.3957,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.15625,
      "step": 14530
    },
    {
      "epoch": 0.5412145688708567,
      "grad_norm": 11.588038772720942,
      "learning_rate": 2.57665022696813e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.546875,
      "logps/chosen": -552.0,
      "logps/rejected": -692.0,
      "loss": 0.3882,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.0,
      "step": 14540
    },
    {
      "epoch": 0.5415867934711805,
      "grad_norm": 10.370744231049999,
      "learning_rate": 2.573403290302819e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.375,
      "logps/chosen": -524.0,
      "logps/rejected": -688.0,
      "loss": 0.3869,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.625,
      "rewards/rejected": -4.90625,
      "step": 14550
    },
    {
      "epoch": 0.5419590180715044,
      "grad_norm": 9.536086395100577,
      "learning_rate": 2.5701562297080755e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.53125,
      "logps/chosen": -556.0,
      "logps/rejected": -708.0,
      "loss": 0.406,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -4.9375,
      "step": 14560
    },
    {
      "epoch": 0.5423312426718282,
      "grad_norm": 11.651479098612384,
      "learning_rate": 2.5669090506660284e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.625,
      "logps/chosen": -576.0,
      "logps/rejected": -696.0,
      "loss": 0.4028,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.4375,
      "rewards/rejected": -5.0,
      "step": 14570
    },
    {
      "epoch": 0.542703467272152,
      "grad_norm": 10.524122347034334,
      "learning_rate": 2.563661758659007e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.671875,
      "logps/chosen": -568.0,
      "logps/rejected": -708.0,
      "loss": 0.4048,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.09375,
      "step": 14580
    },
    {
      "epoch": 0.5430756918724758,
      "grad_norm": 12.220909933512496,
      "learning_rate": 2.560414359169533e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.765625,
      "logps/chosen": -560.0,
      "logps/rejected": -680.0,
      "loss": 0.3767,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.9375,
      "step": 14590
    },
    {
      "epoch": 0.5434479164727997,
      "grad_norm": 10.930037676384817,
      "learning_rate": 2.5571668576803083e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.6875,
      "logps/chosen": -540.0,
      "logps/rejected": -652.0,
      "loss": 0.395,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.3125,
      "rewards/rejected": -4.75,
      "step": 14600
    },
    {
      "epoch": 0.5438201410731235,
      "grad_norm": 10.947811255347254,
      "learning_rate": 2.553919259674207e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.71875,
      "logps/chosen": -548.0,
      "logps/rejected": -688.0,
      "loss": 0.3909,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.03125,
      "step": 14610
    },
    {
      "epoch": 0.5441923656734473,
      "grad_norm": 10.982714047971053,
      "learning_rate": 2.5506715706342665e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -552.0,
      "logps/rejected": -700.0,
      "loss": 0.4092,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.9375,
      "step": 14620
    },
    {
      "epoch": 0.5445645902737712,
      "grad_norm": 11.3901040057847,
      "learning_rate": 2.547423796043678e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.703125,
      "logps/chosen": -504.0,
      "logps/rejected": -684.0,
      "loss": 0.3905,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.109375,
      "rewards/margins": 1.8046875,
      "rewards/rejected": -4.90625,
      "step": 14630
    },
    {
      "epoch": 0.5449368148740951,
      "grad_norm": 12.369327569756441,
      "learning_rate": 2.5441759413857764e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.71875,
      "logps/chosen": -544.0,
      "logps/rejected": -680.0,
      "loss": 0.4073,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.96875,
      "step": 14640
    },
    {
      "epoch": 0.5453090394744189,
      "grad_norm": 11.218166992864466,
      "learning_rate": 2.540928012144033e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.65625,
      "logps/chosen": -552.0,
      "logps/rejected": -688.0,
      "loss": 0.4017,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.9375,
      "step": 14650
    },
    {
      "epoch": 0.5456812640747427,
      "grad_norm": 11.792581552926515,
      "learning_rate": 2.537680013802045e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.765625,
      "logps/chosen": -540.0,
      "logps/rejected": -684.0,
      "loss": 0.408,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.59375,
      "rewards/rejected": -4.96875,
      "step": 14660
    },
    {
      "epoch": 0.5460534886750665,
      "grad_norm": 10.466573957676689,
      "learning_rate": 2.534431951843524e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.734375,
      "logps/chosen": -532.0,
      "logps/rejected": -644.0,
      "loss": 0.4023,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.71875,
      "step": 14670
    },
    {
      "epoch": 0.5464257132753904,
      "grad_norm": 11.407250493614074,
      "learning_rate": 2.5311838317522924e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.609375,
      "logps/chosen": -520.0,
      "logps/rejected": -656.0,
      "loss": 0.3981,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.359375,
      "rewards/rejected": -4.65625,
      "step": 14680
    },
    {
      "epoch": 0.5467979378757142,
      "grad_norm": 12.504089614386393,
      "learning_rate": 2.5279356590122674e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.734375,
      "logps/chosen": -532.0,
      "logps/rejected": -680.0,
      "loss": 0.388,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.03125,
      "step": 14690
    },
    {
      "epoch": 0.547170162476038,
      "grad_norm": 10.497326918113691,
      "learning_rate": 2.524687439107458e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.828125,
      "logps/chosen": -560.0,
      "logps/rejected": -684.0,
      "loss": 0.4133,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.8125,
      "step": 14700
    },
    {
      "epoch": 0.5475423870763618,
      "grad_norm": 11.202147388685283,
      "learning_rate": 2.5214391775219506e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.640625,
      "logps/chosen": -532.0,
      "logps/rejected": -676.0,
      "loss": 0.4193,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.6875,
      "step": 14710
    },
    {
      "epoch": 0.5479146116766858,
      "grad_norm": 12.31562744740878,
      "learning_rate": 2.5181908797399033e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.671875,
      "logps/chosen": -556.0,
      "logps/rejected": -696.0,
      "loss": 0.3962,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.9375,
      "step": 14720
    },
    {
      "epoch": 0.5482868362770096,
      "grad_norm": 9.859274207061718,
      "learning_rate": 2.5149425512455346e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.625,
      "logps/chosen": -540.0,
      "logps/rejected": -668.0,
      "loss": 0.4043,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.78125,
      "step": 14730
    },
    {
      "epoch": 0.5486590608773334,
      "grad_norm": 11.085545654858212,
      "learning_rate": 2.5116941975231153e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.640625,
      "logps/chosen": -552.0,
      "logps/rejected": -680.0,
      "loss": 0.4089,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.75,
      "step": 14740
    },
    {
      "epoch": 0.5490312854776572,
      "grad_norm": 13.319778465670396,
      "learning_rate": 2.5084458240569595e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -532.0,
      "logps/rejected": -664.0,
      "loss": 0.4053,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.65625,
      "step": 14750
    },
    {
      "epoch": 0.549403510077981,
      "grad_norm": 11.302100311697052,
      "learning_rate": 2.5051974363314126e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.515625,
      "logps/chosen": -568.0,
      "logps/rejected": -696.0,
      "loss": 0.382,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.90625,
      "step": 14760
    },
    {
      "epoch": 0.5497757346783049,
      "grad_norm": 11.879420926463425,
      "learning_rate": 2.501949039830846e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.46875,
      "logps/chosen": -556.0,
      "logps/rejected": -708.0,
      "loss": 0.4089,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.84375,
      "rewards/rejected": -5.1875,
      "step": 14770
    },
    {
      "epoch": 0.5501479592786287,
      "grad_norm": 10.567554499328198,
      "learning_rate": 2.4987006400396445e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.609375,
      "logps/chosen": -516.0,
      "logps/rejected": -664.0,
      "loss": 0.3825,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -4.6875,
      "step": 14780
    },
    {
      "epoch": 0.5505201838789525,
      "grad_norm": 11.861369907335414,
      "learning_rate": 2.4954522424422e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.640625,
      "logps/chosen": -512.0,
      "logps/rejected": -656.0,
      "loss": 0.399,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.625,
      "step": 14790
    },
    {
      "epoch": 0.5508924084792763,
      "grad_norm": 11.823289547160709,
      "learning_rate": 2.492203852522898e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.546875,
      "logps/chosen": -552.0,
      "logps/rejected": -656.0,
      "loss": 0.3967,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.625,
      "step": 14800
    },
    {
      "epoch": 0.5512646330796003,
      "grad_norm": 12.265256103739516,
      "learning_rate": 2.4889554757661154e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.625,
      "logps/chosen": -548.0,
      "logps/rejected": -668.0,
      "loss": 0.4162,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.421875,
      "rewards/rejected": -4.75,
      "step": 14810
    },
    {
      "epoch": 0.5516368576799241,
      "grad_norm": 10.698345424396742,
      "learning_rate": 2.4857071176562026e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.625,
      "logps/chosen": -524.0,
      "logps/rejected": -652.0,
      "loss": 0.3979,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -4.5625,
      "step": 14820
    },
    {
      "epoch": 0.5520090822802479,
      "grad_norm": 10.878096306361364,
      "learning_rate": 2.482458783677481e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.484375,
      "logps/chosen": -528.0,
      "logps/rejected": -688.0,
      "loss": 0.393,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -4.90625,
      "step": 14830
    },
    {
      "epoch": 0.5523813068805717,
      "grad_norm": 10.824006893791395,
      "learning_rate": 2.47921047931423e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.703125,
      "logps/chosen": -532.0,
      "logps/rejected": -656.0,
      "loss": 0.3895,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.6875,
      "step": 14840
    },
    {
      "epoch": 0.5527535314808956,
      "grad_norm": 9.648787289574171,
      "learning_rate": 2.47596221005068e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.59375,
      "logps/chosen": -532.0,
      "logps/rejected": -656.0,
      "loss": 0.4112,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.625,
      "step": 14850
    },
    {
      "epoch": 0.5531257560812194,
      "grad_norm": 11.071222451236748,
      "learning_rate": 2.472713981371002e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5625,
      "logps/chosen": -548.0,
      "logps/rejected": -696.0,
      "loss": 0.3771,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.0,
      "step": 14860
    },
    {
      "epoch": 0.5534979806815432,
      "grad_norm": 11.951110459491893,
      "learning_rate": 2.4694657987592964e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.625,
      "logps/chosen": -600.0,
      "logps/rejected": -752.0,
      "loss": 0.3803,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.828125,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.5,
      "step": 14870
    },
    {
      "epoch": 0.553870205281867,
      "grad_norm": 11.634659177336625,
      "learning_rate": 2.4662176676995895e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -544.0,
      "logps/rejected": -704.0,
      "loss": 0.3902,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.0625,
      "step": 14880
    },
    {
      "epoch": 0.554242429882191,
      "grad_norm": 10.743037936737462,
      "learning_rate": 2.4629695936758176e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.765625,
      "logps/chosen": -552.0,
      "logps/rejected": -708.0,
      "loss": 0.3869,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.0625,
      "step": 14890
    },
    {
      "epoch": 0.5546146544825148,
      "grad_norm": 14.638869610591355,
      "learning_rate": 2.459721582171822e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.6875,
      "logps/chosen": -544.0,
      "logps/rejected": -676.0,
      "loss": 0.432,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.84375,
      "step": 14900
    },
    {
      "epoch": 0.5549868790828386,
      "grad_norm": 11.660376224206919,
      "learning_rate": 2.4564736386713383e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.71875,
      "logps/chosen": -520.0,
      "logps/rejected": -668.0,
      "loss": 0.3913,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.703125,
      "rewards/rejected": -4.84375,
      "step": 14910
    },
    {
      "epoch": 0.5553591036831624,
      "grad_norm": 13.515249941840294,
      "learning_rate": 2.453225768657987e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.46875,
      "logps/chosen": -490.0,
      "logps/rejected": -624.0,
      "loss": 0.4079,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.5,
      "step": 14920
    },
    {
      "epoch": 0.5557313282834863,
      "grad_norm": 10.80433702816929,
      "learning_rate": 2.4499779776152644e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.578125,
      "logps/chosen": -548.0,
      "logps/rejected": -668.0,
      "loss": 0.4404,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.1875,
      "rewards/rejected": -4.59375,
      "step": 14930
    },
    {
      "epoch": 0.5561035528838101,
      "grad_norm": 11.068862565727704,
      "learning_rate": 2.446730271026534e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.75,
      "logps/chosen": -556.0,
      "logps/rejected": -656.0,
      "loss": 0.4037,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -4.59375,
      "step": 14940
    },
    {
      "epoch": 0.5564757774841339,
      "grad_norm": 14.254942129893266,
      "learning_rate": 2.443482654375017e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5625,
      "logps/chosen": -486.0,
      "logps/rejected": -624.0,
      "loss": 0.3952,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.4375,
      "step": 14950
    },
    {
      "epoch": 0.5568480020844577,
      "grad_norm": 10.447384198045437,
      "learning_rate": 2.440235133143781e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.609375,
      "logps/chosen": -516.0,
      "logps/rejected": -668.0,
      "loss": 0.3865,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.78125,
      "step": 14960
    },
    {
      "epoch": 0.5572202266847815,
      "grad_norm": 11.635971576644259,
      "learning_rate": 2.4369877128157353e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.546875,
      "logps/chosen": -572.0,
      "logps/rejected": -684.0,
      "loss": 0.4098,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -4.65625,
      "step": 14970
    },
    {
      "epoch": 0.5575924512851055,
      "grad_norm": 9.628318687410301,
      "learning_rate": 2.433740398873616e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.453125,
      "logps/chosen": -512.0,
      "logps/rejected": -644.0,
      "loss": 0.4014,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.53125,
      "step": 14980
    },
    {
      "epoch": 0.5579646758854293,
      "grad_norm": 11.86057670840013,
      "learning_rate": 2.4304931967999817e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.546875,
      "logps/chosen": -528.0,
      "logps/rejected": -644.0,
      "loss": 0.3875,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.234375,
      "rewards/rejected": -4.5,
      "step": 14990
    },
    {
      "epoch": 0.5583369004857531,
      "grad_norm": 12.046992524879336,
      "learning_rate": 2.4272461120772e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -564.0,
      "logps/rejected": -680.0,
      "loss": 0.3983,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.875,
      "step": 15000
    },
    {
      "epoch": 0.558709125086077,
      "grad_norm": 10.191791713506277,
      "learning_rate": 2.4239991501874437e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.578125,
      "logps/chosen": -556.0,
      "logps/rejected": -684.0,
      "loss": 0.4164,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.9375,
      "step": 15010
    },
    {
      "epoch": 0.5590813496864008,
      "grad_norm": 10.968746056622113,
      "learning_rate": 2.4207523166126737e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.390625,
      "logps/chosen": -536.0,
      "logps/rejected": -696.0,
      "loss": 0.4034,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.9375,
      "step": 15020
    },
    {
      "epoch": 0.5594535742867246,
      "grad_norm": 10.626423684293183,
      "learning_rate": 2.417505616834638e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.46875,
      "logps/chosen": -520.0,
      "logps/rejected": -648.0,
      "loss": 0.3893,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.421875,
      "rewards/rejected": -4.6875,
      "step": 15030
    },
    {
      "epoch": 0.5598257988870484,
      "grad_norm": 13.151727947145728,
      "learning_rate": 2.4142590563348586e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.625,
      "logps/chosen": -520.0,
      "logps/rejected": -628.0,
      "loss": 0.408,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.234375,
      "rewards/rejected": -4.5,
      "step": 15040
    },
    {
      "epoch": 0.5601980234873722,
      "grad_norm": 11.417445728476237,
      "learning_rate": 2.411012640594619e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -560.0,
      "logps/rejected": -708.0,
      "loss": 0.3877,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.0625,
      "step": 15050
    },
    {
      "epoch": 0.5605702480876961,
      "grad_norm": 11.973454834327857,
      "learning_rate": 2.4077663750949624e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.515625,
      "logps/chosen": -524.0,
      "logps/rejected": -660.0,
      "loss": 0.3878,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.59375,
      "step": 15060
    },
    {
      "epoch": 0.56094247268802,
      "grad_norm": 11.210465961829893,
      "learning_rate": 2.404520265316675e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.796875,
      "logps/chosen": -560.0,
      "logps/rejected": -684.0,
      "loss": 0.4121,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.71875,
      "step": 15070
    },
    {
      "epoch": 0.5613146972883438,
      "grad_norm": 11.419540360670357,
      "learning_rate": 2.4012743167402823e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -520.0,
      "logps/rejected": -660.0,
      "loss": 0.4181,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.6875,
      "step": 15080
    },
    {
      "epoch": 0.5616869218886676,
      "grad_norm": 11.489529294521382,
      "learning_rate": 2.3980285348460363e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -516.0,
      "logps/rejected": -656.0,
      "loss": 0.3999,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.65625,
      "step": 15090
    },
    {
      "epoch": 0.5620591464889915,
      "grad_norm": 11.8861077721709,
      "learning_rate": 2.3947829251139076e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.609375,
      "logps/chosen": -524.0,
      "logps/rejected": -640.0,
      "loss": 0.3949,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.53125,
      "step": 15100
    },
    {
      "epoch": 0.5624313710893153,
      "grad_norm": 11.052918557743498,
      "learning_rate": 2.391537493023579e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5625,
      "logps/chosen": -536.0,
      "logps/rejected": -688.0,
      "loss": 0.3892,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.828125,
      "rewards/rejected": -5.15625,
      "step": 15110
    },
    {
      "epoch": 0.5628035956896391,
      "grad_norm": 12.799258844110204,
      "learning_rate": 2.388292244054429e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.53125,
      "logps/chosen": -524.0,
      "logps/rejected": -664.0,
      "loss": 0.4084,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -4.9375,
      "step": 15120
    },
    {
      "epoch": 0.5631758202899629,
      "grad_norm": 11.72842413705576,
      "learning_rate": 2.3850471836855297e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.59375,
      "logps/chosen": -548.0,
      "logps/rejected": -668.0,
      "loss": 0.4084,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.8125,
      "step": 15130
    },
    {
      "epoch": 0.5635480448902868,
      "grad_norm": 10.431945250492477,
      "learning_rate": 2.3818023173956334e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -556.0,
      "logps/rejected": -696.0,
      "loss": 0.3928,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.90625,
      "step": 15140
    },
    {
      "epoch": 0.5639202694906106,
      "grad_norm": 11.382711452137613,
      "learning_rate": 2.3785576506631665e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.515625,
      "logps/chosen": -496.0,
      "logps/rejected": -664.0,
      "loss": 0.3968,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -4.78125,
      "step": 15150
    },
    {
      "epoch": 0.5642924940909345,
      "grad_norm": 11.55959034806221,
      "learning_rate": 2.3753131889662162e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.484375,
      "logps/chosen": -552.0,
      "logps/rejected": -668.0,
      "loss": 0.4203,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -4.71875,
      "step": 15160
    },
    {
      "epoch": 0.5646647186912583,
      "grad_norm": 9.83671389671711,
      "learning_rate": 2.372068937782526e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -540.0,
      "logps/rejected": -672.0,
      "loss": 0.3929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.71875,
      "step": 15170
    },
    {
      "epoch": 0.5650369432915822,
      "grad_norm": 11.1988687718354,
      "learning_rate": 2.368824902589481e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.625,
      "logps/chosen": -528.0,
      "logps/rejected": -648.0,
      "loss": 0.416,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.6875,
      "step": 15180
    },
    {
      "epoch": 0.565409167891906,
      "grad_norm": 10.049726830677425,
      "learning_rate": 2.3655810888641044e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5625,
      "logps/chosen": -528.0,
      "logps/rejected": -664.0,
      "loss": 0.3775,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.65625,
      "step": 15190
    },
    {
      "epoch": 0.5657813924922298,
      "grad_norm": 10.065802975081048,
      "learning_rate": 2.362337502083045e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.71875,
      "logps/chosen": -528.0,
      "logps/rejected": -644.0,
      "loss": 0.4171,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -4.53125,
      "step": 15200
    },
    {
      "epoch": 0.5661536170925536,
      "grad_norm": 10.592166366573403,
      "learning_rate": 2.3590941477225666e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.65625,
      "logps/chosen": -504.0,
      "logps/rejected": -668.0,
      "loss": 0.3996,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.8046875,
      "rewards/rejected": -4.78125,
      "step": 15210
    },
    {
      "epoch": 0.5665258416928775,
      "grad_norm": 11.307551025811334,
      "learning_rate": 2.3558510312585425e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5,
      "logps/chosen": -560.0,
      "logps/rejected": -668.0,
      "loss": 0.4219,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.296875,
      "rewards/rejected": -4.65625,
      "step": 15220
    },
    {
      "epoch": 0.5668980662932013,
      "grad_norm": 14.424631787125016,
      "learning_rate": 2.3526081581664434e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -512.0,
      "logps/rejected": -640.0,
      "loss": 0.4123,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.109375,
      "rewards/margins": 1.328125,
      "rewards/rejected": -4.4375,
      "step": 15230
    },
    {
      "epoch": 0.5672702908935251,
      "grad_norm": 10.628834639240628,
      "learning_rate": 2.3493655339213303e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.578125,
      "logps/chosen": -540.0,
      "logps/rejected": -672.0,
      "loss": 0.4128,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.75,
      "step": 15240
    },
    {
      "epoch": 0.567642515493849,
      "grad_norm": 10.74859616807647,
      "learning_rate": 2.346123163997841e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.71875,
      "logps/chosen": -552.0,
      "logps/rejected": -720.0,
      "loss": 0.3953,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -4.875,
      "step": 15250
    },
    {
      "epoch": 0.5680147400941729,
      "grad_norm": 10.171948890320895,
      "learning_rate": 2.3428810538701893e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.703125,
      "logps/chosen": -502.0,
      "logps/rejected": -668.0,
      "loss": 0.3788,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.734375,
      "rewards/rejected": -4.78125,
      "step": 15260
    },
    {
      "epoch": 0.5683869646944967,
      "grad_norm": 12.535567067334457,
      "learning_rate": 2.3396392090121435e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.796875,
      "logps/chosen": -516.0,
      "logps/rejected": -656.0,
      "loss": 0.414,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.6875,
      "step": 15270
    },
    {
      "epoch": 0.5687591892948205,
      "grad_norm": 11.014713261432707,
      "learning_rate": 2.3363976348970303e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.703125,
      "logps/chosen": -544.0,
      "logps/rejected": -708.0,
      "loss": 0.3999,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -4.875,
      "step": 15280
    },
    {
      "epoch": 0.5691314138951443,
      "grad_norm": 9.212033619284401,
      "learning_rate": 2.3331563369977163e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.828125,
      "logps/chosen": -568.0,
      "logps/rejected": -660.0,
      "loss": 0.4078,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -4.5625,
      "step": 15290
    },
    {
      "epoch": 0.5695036384954681,
      "grad_norm": 10.658605141343397,
      "learning_rate": 2.3299153207866013e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.6875,
      "logps/chosen": -528.0,
      "logps/rejected": -676.0,
      "loss": 0.3762,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.84375,
      "step": 15300
    },
    {
      "epoch": 0.569875863095792,
      "grad_norm": 11.606544429357127,
      "learning_rate": 2.326674591735612e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.546875,
      "logps/chosen": -516.0,
      "logps/rejected": -656.0,
      "loss": 0.3982,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.65625,
      "step": 15310
    },
    {
      "epoch": 0.5702480876961158,
      "grad_norm": 10.588978714010306,
      "learning_rate": 2.3234341553161858e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.546875,
      "logps/chosen": -568.0,
      "logps/rejected": -704.0,
      "loss": 0.3861,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.0,
      "step": 15320
    },
    {
      "epoch": 0.5706203122964397,
      "grad_norm": 10.51187566655422,
      "learning_rate": 2.3201940169992723e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.78125,
      "logps/chosen": -556.0,
      "logps/rejected": -672.0,
      "loss": 0.3973,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.8125,
      "step": 15330
    },
    {
      "epoch": 0.5709925368967635,
      "grad_norm": 10.769424736751306,
      "learning_rate": 2.316954182255311e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -528.0,
      "logps/rejected": -676.0,
      "loss": 0.41,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -4.8125,
      "step": 15340
    },
    {
      "epoch": 0.5713647614970874,
      "grad_norm": 10.267248371842772,
      "learning_rate": 2.3137146565542343e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.515625,
      "logps/chosen": -516.0,
      "logps/rejected": -648.0,
      "loss": 0.4041,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.109375,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.625,
      "step": 15350
    },
    {
      "epoch": 0.5717369860974112,
      "grad_norm": 11.183331585702776,
      "learning_rate": 2.3104754453654514e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.671875,
      "logps/chosen": -528.0,
      "logps/rejected": -640.0,
      "loss": 0.4256,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.6875,
      "step": 15360
    },
    {
      "epoch": 0.572109210697735,
      "grad_norm": 12.99642611301677,
      "learning_rate": 2.307236554157838e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.53125,
      "logps/chosen": -524.0,
      "logps/rejected": -660.0,
      "loss": 0.383,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.65625,
      "step": 15370
    },
    {
      "epoch": 0.5724814352980588,
      "grad_norm": 11.071125147699444,
      "learning_rate": 2.3039979883997333e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.6875,
      "logps/chosen": -540.0,
      "logps/rejected": -676.0,
      "loss": 0.3841,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.875,
      "step": 15380
    },
    {
      "epoch": 0.5728536598983827,
      "grad_norm": 12.294047291124796,
      "learning_rate": 2.3007597535589224e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.84375,
      "logps/chosen": -560.0,
      "logps/rejected": -680.0,
      "loss": 0.4088,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.0,
      "step": 15390
    },
    {
      "epoch": 0.5732258844987065,
      "grad_norm": 10.36069743562513,
      "learning_rate": 2.297521855102638e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.6875,
      "logps/chosen": -524.0,
      "logps/rejected": -688.0,
      "loss": 0.3869,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.03125,
      "step": 15400
    },
    {
      "epoch": 0.5735981090990303,
      "grad_norm": 9.724721234789264,
      "learning_rate": 2.2942842984975383e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -528.0,
      "logps/rejected": -688.0,
      "loss": 0.3926,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -4.96875,
      "step": 15410
    },
    {
      "epoch": 0.5739703336993542,
      "grad_norm": 12.98754567757196,
      "learning_rate": 2.2910470892097102e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.765625,
      "logps/chosen": -536.0,
      "logps/rejected": -696.0,
      "loss": 0.3957,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.890625,
      "rewards/rejected": -5.125,
      "step": 15420
    },
    {
      "epoch": 0.5743425582996781,
      "grad_norm": 10.951612188529957,
      "learning_rate": 2.287810232704649e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.625,
      "logps/chosen": -508.0,
      "logps/rejected": -680.0,
      "loss": 0.3961,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.109375,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.71875,
      "step": 15430
    },
    {
      "epoch": 0.5747147829000019,
      "grad_norm": 10.78086660950833,
      "learning_rate": 2.2845737344472564e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.65625,
      "logps/chosen": -516.0,
      "logps/rejected": -672.0,
      "loss": 0.3932,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -4.84375,
      "step": 15440
    },
    {
      "epoch": 0.5750870075003257,
      "grad_norm": 11.278736979808773,
      "learning_rate": 2.2813375999018334e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -524.0,
      "logps/rejected": -660.0,
      "loss": 0.3835,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.6875,
      "rewards/rejected": -4.78125,
      "step": 15450
    },
    {
      "epoch": 0.5754592321006495,
      "grad_norm": 9.584334223953856,
      "learning_rate": 2.2781018345320585e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.46875,
      "logps/chosen": -516.0,
      "logps/rejected": -672.0,
      "loss": 0.4098,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.734375,
      "rewards/rejected": -4.8125,
      "step": 15460
    },
    {
      "epoch": 0.5758314567009734,
      "grad_norm": 11.313854377399833,
      "learning_rate": 2.2748664438009953e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.546875,
      "logps/chosen": -528.0,
      "logps/rejected": -728.0,
      "loss": 0.3811,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.265625,
      "rewards/margins": 2.03125,
      "rewards/rejected": -5.3125,
      "step": 15470
    },
    {
      "epoch": 0.5762036813012972,
      "grad_norm": 12.627304547147256,
      "learning_rate": 2.2716314331710684e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.609375,
      "logps/chosen": -564.0,
      "logps/rejected": -668.0,
      "loss": 0.3941,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.71875,
      "step": 15480
    },
    {
      "epoch": 0.576575905901621,
      "grad_norm": 12.283023616233105,
      "learning_rate": 2.268396808104066e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.765625,
      "logps/chosen": -504.0,
      "logps/rejected": -640.0,
      "loss": 0.3868,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.0625,
      "rewards/margins": 1.625,
      "rewards/rejected": -4.6875,
      "step": 15490
    },
    {
      "epoch": 0.5769481305019448,
      "grad_norm": 9.438378757930678,
      "learning_rate": 2.2651625740611206e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -564.0,
      "logps/rejected": -700.0,
      "loss": 0.4025,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.0,
      "step": 15500
    },
    {
      "epoch": 0.5773203551022688,
      "grad_norm": 9.67194964890529,
      "learning_rate": 2.2619287365027075e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.765625,
      "logps/chosen": -536.0,
      "logps/rejected": -660.0,
      "loss": 0.4004,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -4.6875,
      "step": 15510
    },
    {
      "epoch": 0.5776925797025926,
      "grad_norm": 11.078075314477548,
      "learning_rate": 2.2586953008886314e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.71875,
      "logps/chosen": -532.0,
      "logps/rejected": -652.0,
      "loss": 0.3897,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.75,
      "step": 15520
    },
    {
      "epoch": 0.5780648043029164,
      "grad_norm": 10.27740647159642,
      "learning_rate": 2.2554622726780186e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.65625,
      "logps/chosen": -540.0,
      "logps/rejected": -704.0,
      "loss": 0.3871,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.03125,
      "step": 15530
    },
    {
      "epoch": 0.5784370289032402,
      "grad_norm": 11.081030659225025,
      "learning_rate": 2.25222965732931e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.625,
      "logps/chosen": -506.0,
      "logps/rejected": -664.0,
      "loss": 0.4019,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.640625,
      "rewards/rejected": -4.8125,
      "step": 15540
    },
    {
      "epoch": 0.578809253503564,
      "grad_norm": 10.442783400870418,
      "learning_rate": 2.2489974603002437e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.6875,
      "logps/chosen": -516.0,
      "logps/rejected": -664.0,
      "loss": 0.4243,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.59375,
      "step": 15550
    },
    {
      "epoch": 0.5791814781038879,
      "grad_norm": 11.192604819436848,
      "learning_rate": 2.2457656870478587e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5625,
      "logps/chosen": -556.0,
      "logps/rejected": -700.0,
      "loss": 0.4006,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.78125,
      "rewards/rejected": -5.0625,
      "step": 15560
    },
    {
      "epoch": 0.5795537027042117,
      "grad_norm": 10.011153263370383,
      "learning_rate": 2.2425343430284716e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -548.0,
      "logps/rejected": -676.0,
      "loss": 0.4009,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.6875,
      "step": 15570
    },
    {
      "epoch": 0.5799259273045355,
      "grad_norm": 12.282473402855977,
      "learning_rate": 2.239303433697679e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.46875,
      "logps/chosen": -532.0,
      "logps/rejected": -664.0,
      "loss": 0.3947,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.8125,
      "step": 15580
    },
    {
      "epoch": 0.5802981519048593,
      "grad_norm": 13.225968331721072,
      "learning_rate": 2.2360729645103418e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.640625,
      "logps/chosen": -524.0,
      "logps/rejected": -660.0,
      "loss": 0.3923,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.65625,
      "step": 15590
    },
    {
      "epoch": 0.5806703765051833,
      "grad_norm": 10.628676209523752,
      "learning_rate": 2.232842940920579e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.578125,
      "logps/chosen": -556.0,
      "logps/rejected": -688.0,
      "loss": 0.3963,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.3125,
      "rewards/rejected": -4.875,
      "step": 15600
    },
    {
      "epoch": 0.5810426011055071,
      "grad_norm": 11.13126855578364,
      "learning_rate": 2.229613368381755e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.484375,
      "logps/chosen": -512.0,
      "logps/rejected": -660.0,
      "loss": 0.3919,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.59375,
      "step": 15610
    },
    {
      "epoch": 0.5814148257058309,
      "grad_norm": 11.261278980183306,
      "learning_rate": 2.2263842523464744e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.40625,
      "logps/chosen": -552.0,
      "logps/rejected": -700.0,
      "loss": 0.3878,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -4.90625,
      "step": 15620
    },
    {
      "epoch": 0.5817870503061547,
      "grad_norm": 14.159907209330534,
      "learning_rate": 2.2231555982665728e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.703125,
      "logps/chosen": -556.0,
      "logps/rejected": -676.0,
      "loss": 0.4003,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -4.90625,
      "step": 15630
    },
    {
      "epoch": 0.5821592749064786,
      "grad_norm": 12.43463438620818,
      "learning_rate": 2.2199274115931012e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5625,
      "logps/chosen": -552.0,
      "logps/rejected": -688.0,
      "loss": 0.3926,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -5.0,
      "step": 15640
    },
    {
      "epoch": 0.5825314995068024,
      "grad_norm": 11.718557503815672,
      "learning_rate": 2.216699697776326e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.46875,
      "logps/chosen": -548.0,
      "logps/rejected": -740.0,
      "loss": 0.37,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.21875,
      "rewards/margins": 2.09375,
      "rewards/rejected": -5.28125,
      "step": 15650
    },
    {
      "epoch": 0.5829037241071262,
      "grad_norm": 11.676750613477928,
      "learning_rate": 2.2134724622657114e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.34375,
      "logps/chosen": -576.0,
      "logps/rejected": -700.0,
      "loss": 0.406,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.125,
      "step": 15660
    },
    {
      "epoch": 0.58327594870745,
      "grad_norm": 11.892864143126674,
      "learning_rate": 2.2102457105099177e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.765625,
      "logps/chosen": -568.0,
      "logps/rejected": -724.0,
      "loss": 0.3855,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.21875,
      "step": 15670
    },
    {
      "epoch": 0.583648173307774,
      "grad_norm": 11.267131748465983,
      "learning_rate": 2.2070194479567858e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.671875,
      "logps/chosen": -584.0,
      "logps/rejected": -728.0,
      "loss": 0.3989,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -5.15625,
      "step": 15680
    },
    {
      "epoch": 0.5840203979080978,
      "grad_norm": 13.192769653647185,
      "learning_rate": 2.2037936800533319e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.515625,
      "logps/chosen": -516.0,
      "logps/rejected": -628.0,
      "loss": 0.3856,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.5625,
      "step": 15690
    },
    {
      "epoch": 0.5843926225084216,
      "grad_norm": 10.558514318316936,
      "learning_rate": 2.2005684122457374e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.671875,
      "logps/chosen": -524.0,
      "logps/rejected": -700.0,
      "loss": 0.3946,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -4.96875,
      "step": 15700
    },
    {
      "epoch": 0.5847648471087454,
      "grad_norm": 9.85391232896492,
      "learning_rate": 2.1973436499793377e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.546875,
      "logps/chosen": -536.0,
      "logps/rejected": -680.0,
      "loss": 0.3887,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.875,
      "step": 15710
    },
    {
      "epoch": 0.5851370717090693,
      "grad_norm": 11.637185526733372,
      "learning_rate": 2.194119398698617e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.609375,
      "logps/chosen": -536.0,
      "logps/rejected": -664.0,
      "loss": 0.4172,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.65625,
      "rewards/rejected": -4.9375,
      "step": 15720
    },
    {
      "epoch": 0.5855092963093931,
      "grad_norm": 11.246372417749836,
      "learning_rate": 2.190895663847194e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -528.0,
      "logps/rejected": -680.0,
      "loss": 0.3876,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -5.0,
      "step": 15730
    },
    {
      "epoch": 0.5858815209097169,
      "grad_norm": 9.835916411069492,
      "learning_rate": 2.1876724508678184e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.484375,
      "logps/chosen": -504.0,
      "logps/rejected": -652.0,
      "loss": 0.3727,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -4.625,
      "step": 15740
    },
    {
      "epoch": 0.5862537455100407,
      "grad_norm": 9.76849142957356,
      "learning_rate": 2.1844497652023563e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.59375,
      "logps/chosen": -540.0,
      "logps/rejected": -680.0,
      "loss": 0.4009,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.90625,
      "step": 15750
    },
    {
      "epoch": 0.5866259701103645,
      "grad_norm": 9.67122121661811,
      "learning_rate": 2.181227612291786e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.65625,
      "logps/chosen": -524.0,
      "logps/rejected": -680.0,
      "loss": 0.3835,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -4.90625,
      "step": 15760
    },
    {
      "epoch": 0.5869981947106885,
      "grad_norm": 11.909729312987448,
      "learning_rate": 2.1780059975761833e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.390625,
      "logps/chosen": -532.0,
      "logps/rejected": -676.0,
      "loss": 0.4077,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.03125,
      "step": 15770
    },
    {
      "epoch": 0.5873704193110123,
      "grad_norm": 10.4816431669514,
      "learning_rate": 2.1747849264947177e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.53125,
      "logps/chosen": -524.0,
      "logps/rejected": -668.0,
      "loss": 0.3781,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 1.765625,
      "rewards/rejected": -4.71875,
      "step": 15780
    },
    {
      "epoch": 0.5877426439113361,
      "grad_norm": 9.744246990248232,
      "learning_rate": 2.1715644044856413e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.65625,
      "logps/chosen": -544.0,
      "logps/rejected": -680.0,
      "loss": 0.4046,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.9375,
      "step": 15790
    },
    {
      "epoch": 0.58811486851166,
      "grad_norm": 13.129768398584128,
      "learning_rate": 2.1683444369862763e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.5,
      "logps/chosen": -536.0,
      "logps/rejected": -664.0,
      "loss": 0.4091,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.75,
      "step": 15800
    },
    {
      "epoch": 0.5884870931119838,
      "grad_norm": 11.767395657586917,
      "learning_rate": 2.165125029433012e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.375,
      "logps/chosen": -500.0,
      "logps/rejected": -652.0,
      "loss": 0.4085,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.59375,
      "step": 15810
    },
    {
      "epoch": 0.5888593177123076,
      "grad_norm": 13.16546828131715,
      "learning_rate": 2.1619061872612892e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.625,
      "logps/chosen": -576.0,
      "logps/rejected": -684.0,
      "loss": 0.4126,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.21875,
      "rewards/rejected": -4.75,
      "step": 15820
    },
    {
      "epoch": 0.5892315423126314,
      "grad_norm": 11.093510026742583,
      "learning_rate": 2.158687915905597e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.703125,
      "logps/chosen": -560.0,
      "logps/rejected": -700.0,
      "loss": 0.4082,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.0,
      "step": 15830
    },
    {
      "epoch": 0.5896037669129552,
      "grad_norm": 10.901836948878616,
      "learning_rate": 2.155470220799459e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.40625,
      "logps/chosen": -532.0,
      "logps/rejected": -700.0,
      "loss": 0.4038,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -5.0625,
      "step": 15840
    },
    {
      "epoch": 0.5899759915132791,
      "grad_norm": 11.21483553114483,
      "learning_rate": 2.152253107375427e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.609375,
      "logps/chosen": -524.0,
      "logps/rejected": -672.0,
      "loss": 0.3909,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -4.96875,
      "step": 15850
    },
    {
      "epoch": 0.590348216113603,
      "grad_norm": 11.706538083153525,
      "learning_rate": 2.1490365810650686e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.515625,
      "logps/chosen": -584.0,
      "logps/rejected": -728.0,
      "loss": 0.4082,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.09375,
      "step": 15860
    },
    {
      "epoch": 0.5907204407139268,
      "grad_norm": 10.8577776390526,
      "learning_rate": 2.1458206472989626e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5625,
      "logps/chosen": -516.0,
      "logps/rejected": -656.0,
      "loss": 0.3865,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.03125,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -4.71875,
      "step": 15870
    },
    {
      "epoch": 0.5910926653142506,
      "grad_norm": 11.498976674597369,
      "learning_rate": 2.1426053115066875e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.3125,
      "logps/chosen": -536.0,
      "logps/rejected": -676.0,
      "loss": 0.3941,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.875,
      "step": 15880
    },
    {
      "epoch": 0.5914648899145745,
      "grad_norm": 12.008352883402377,
      "learning_rate": 2.1393905791168093e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.390625,
      "logps/chosen": -496.0,
      "logps/rejected": -648.0,
      "loss": 0.3843,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.625,
      "rewards/rejected": -4.6875,
      "step": 15890
    },
    {
      "epoch": 0.5918371145148983,
      "grad_norm": 12.285355004111189,
      "learning_rate": 2.136176455556879e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.65625,
      "logps/chosen": -536.0,
      "logps/rejected": -684.0,
      "loss": 0.3814,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.84375,
      "step": 15900
    },
    {
      "epoch": 0.5922093391152221,
      "grad_norm": 11.87751692637389,
      "learning_rate": 2.132962946253416e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.46875,
      "logps/chosen": -552.0,
      "logps/rejected": -684.0,
      "loss": 0.3871,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.8125,
      "step": 15910
    },
    {
      "epoch": 0.5925815637155459,
      "grad_norm": 11.947849986124226,
      "learning_rate": 2.129750056631906e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.65625,
      "logps/chosen": -600.0,
      "logps/rejected": -728.0,
      "loss": 0.4095,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.484375,
      "rewards/rejected": -5.1875,
      "step": 15920
    },
    {
      "epoch": 0.5929537883158698,
      "grad_norm": 12.740515223468906,
      "learning_rate": 2.1265377921167855e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.546875,
      "logps/chosen": -524.0,
      "logps/rejected": -648.0,
      "loss": 0.3923,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.625,
      "step": 15930
    },
    {
      "epoch": 0.5933260129161936,
      "grad_norm": 11.948810056148275,
      "learning_rate": 2.1233261581314385e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.578125,
      "logps/chosen": -568.0,
      "logps/rejected": -712.0,
      "loss": 0.3815,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.03125,
      "step": 15940
    },
    {
      "epoch": 0.5936982375165175,
      "grad_norm": 14.548976367386404,
      "learning_rate": 2.1201151600981814e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.546875,
      "logps/chosen": -516.0,
      "logps/rejected": -704.0,
      "loss": 0.4145,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.8359375,
      "rewards/rejected": -5.03125,
      "step": 15950
    },
    {
      "epoch": 0.5940704621168413,
      "grad_norm": 10.507482334076236,
      "learning_rate": 2.1169048034382598e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.640625,
      "logps/chosen": -540.0,
      "logps/rejected": -696.0,
      "loss": 0.3828,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.78125,
      "step": 15960
    },
    {
      "epoch": 0.5944426867171652,
      "grad_norm": 9.946417694061546,
      "learning_rate": 2.1136950935718348e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.40625,
      "logps/chosen": -548.0,
      "logps/rejected": -676.0,
      "loss": 0.3986,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.71875,
      "step": 15970
    },
    {
      "epoch": 0.594814911317489,
      "grad_norm": 11.871741511623012,
      "learning_rate": 2.1104860359179756e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.390625,
      "logps/chosen": -524.0,
      "logps/rejected": -708.0,
      "loss": 0.3782,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.0,
      "step": 15980
    },
    {
      "epoch": 0.5951871359178128,
      "grad_norm": 12.090268928606394,
      "learning_rate": 2.1072776358946507e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.640625,
      "logps/chosen": -524.0,
      "logps/rejected": -688.0,
      "loss": 0.3579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.78125,
      "rewards/rejected": -4.9375,
      "step": 15990
    },
    {
      "epoch": 0.5955593605181366,
      "grad_norm": 12.180407857407511,
      "learning_rate": 2.1040698989187175e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.671875,
      "logps/chosen": -560.0,
      "logps/rejected": -716.0,
      "loss": 0.355,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.8046875,
      "rewards/rejected": -5.25,
      "step": 16000
    },
    {
      "epoch": 0.5959315851184604,
      "grad_norm": 11.01991660093265,
      "learning_rate": 2.1008628304059155e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5625,
      "logps/chosen": -512.0,
      "logps/rejected": -656.0,
      "loss": 0.3912,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.65625,
      "step": 16010
    },
    {
      "epoch": 0.5963038097187843,
      "grad_norm": 10.692460145439028,
      "learning_rate": 2.0976564357708538e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.703125,
      "logps/chosen": -544.0,
      "logps/rejected": -724.0,
      "loss": 0.3883,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.34375,
      "rewards/margins": 2.015625,
      "rewards/rejected": -5.375,
      "step": 16020
    },
    {
      "epoch": 0.5966760343191081,
      "grad_norm": 9.74284356593455,
      "learning_rate": 2.0944507204270044e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.515625,
      "logps/chosen": -536.0,
      "logps/rejected": -684.0,
      "loss": 0.381,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.9375,
      "step": 16030
    },
    {
      "epoch": 0.597048258919432,
      "grad_norm": 10.349749614164152,
      "learning_rate": 2.0912456897866942e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.578125,
      "logps/chosen": -552.0,
      "logps/rejected": -676.0,
      "loss": 0.4024,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.8125,
      "step": 16040
    },
    {
      "epoch": 0.5974204835197559,
      "grad_norm": 9.168465000336461,
      "learning_rate": 2.0880413492610904e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.546875,
      "logps/chosen": -504.0,
      "logps/rejected": -648.0,
      "loss": 0.3977,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -4.5,
      "step": 16050
    },
    {
      "epoch": 0.5977927081200797,
      "grad_norm": 10.90623539979671,
      "learning_rate": 2.0848377042601992e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -524.0,
      "logps/rejected": -680.0,
      "loss": 0.403,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.109375,
      "rewards/margins": 1.765625,
      "rewards/rejected": -4.875,
      "step": 16060
    },
    {
      "epoch": 0.5981649327204035,
      "grad_norm": 11.675232497009773,
      "learning_rate": 2.081634760192849e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.546875,
      "logps/chosen": -512.0,
      "logps/rejected": -660.0,
      "loss": 0.4236,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.71875,
      "step": 16070
    },
    {
      "epoch": 0.5985371573207273,
      "grad_norm": 11.248628900027406,
      "learning_rate": 2.078432522466687e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -568.0,
      "logps/rejected": -688.0,
      "loss": 0.389,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.84375,
      "step": 16080
    },
    {
      "epoch": 0.5989093819210511,
      "grad_norm": 11.663173192145843,
      "learning_rate": 2.0752309964881664e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.65625,
      "logps/chosen": -528.0,
      "logps/rejected": -676.0,
      "loss": 0.4096,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.671875,
      "rewards/rejected": -4.96875,
      "step": 16090
    },
    {
      "epoch": 0.599281606521375,
      "grad_norm": 12.072950882976857,
      "learning_rate": 2.0720301876625416e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.703125,
      "logps/chosen": -532.0,
      "logps/rejected": -668.0,
      "loss": 0.4079,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -4.96875,
      "step": 16100
    },
    {
      "epoch": 0.5996538311216988,
      "grad_norm": 12.337653576248238,
      "learning_rate": 2.0688301013938504e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.703125,
      "logps/chosen": -552.0,
      "logps/rejected": -672.0,
      "loss": 0.3937,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.875,
      "step": 16110
    },
    {
      "epoch": 0.6000260557220227,
      "grad_norm": 10.785630518801755,
      "learning_rate": 2.065630743084917e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -536.0,
      "logps/rejected": -656.0,
      "loss": 0.415,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.78125,
      "step": 16120
    },
    {
      "epoch": 0.6003982803223465,
      "grad_norm": 10.068220385594872,
      "learning_rate": 2.062432118137334e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.40625,
      "logps/chosen": -520.0,
      "logps/rejected": -656.0,
      "loss": 0.3788,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.5625,
      "step": 16130
    },
    {
      "epoch": 0.6007705049226704,
      "grad_norm": 11.889130695441512,
      "learning_rate": 2.0592342319514547e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -502.0,
      "logps/rejected": -648.0,
      "loss": 0.379,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.671875,
      "rewards/rejected": -4.65625,
      "step": 16140
    },
    {
      "epoch": 0.6011427295229942,
      "grad_norm": 14.146662020770203,
      "learning_rate": 2.0560370899263867e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -544.0,
      "logps/rejected": -692.0,
      "loss": 0.3966,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -4.875,
      "step": 16150
    },
    {
      "epoch": 0.601514954123318,
      "grad_norm": 11.191892358716721,
      "learning_rate": 2.0528406974599808e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.796875,
      "logps/chosen": -556.0,
      "logps/rejected": -672.0,
      "loss": 0.4141,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.71875,
      "step": 16160
    },
    {
      "epoch": 0.6018871787236418,
      "grad_norm": 11.36582174886065,
      "learning_rate": 2.0496450599488224e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.78125,
      "logps/chosen": -544.0,
      "logps/rejected": -672.0,
      "loss": 0.3901,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -5.0,
      "step": 16170
    },
    {
      "epoch": 0.6022594033239657,
      "grad_norm": 11.486082448362078,
      "learning_rate": 2.0464501827882202e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5625,
      "logps/chosen": -512.0,
      "logps/rejected": -668.0,
      "loss": 0.3832,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.75,
      "step": 16180
    },
    {
      "epoch": 0.6026316279242895,
      "grad_norm": 10.134171396136102,
      "learning_rate": 2.0432560713722043e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.578125,
      "logps/chosen": -552.0,
      "logps/rejected": -704.0,
      "loss": 0.3998,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -5.03125,
      "step": 16190
    },
    {
      "epoch": 0.6030038525246133,
      "grad_norm": 11.36496705865593,
      "learning_rate": 2.040062731093505e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.515625,
      "logps/chosen": -572.0,
      "logps/rejected": -700.0,
      "loss": 0.3879,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -5.0,
      "step": 16200
    },
    {
      "epoch": 0.6033760771249372,
      "grad_norm": 11.39344118438762,
      "learning_rate": 2.0368701673435567e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.359375,
      "logps/chosen": -536.0,
      "logps/rejected": -704.0,
      "loss": 0.3642,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.71875,
      "rewards/rejected": -4.96875,
      "step": 16210
    },
    {
      "epoch": 0.6037483017252611,
      "grad_norm": 10.522678291091369,
      "learning_rate": 2.0336783855124806e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.53125,
      "logps/chosen": -532.0,
      "logps/rejected": -672.0,
      "loss": 0.3804,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.75,
      "step": 16220
    },
    {
      "epoch": 0.6041205263255849,
      "grad_norm": 11.473709142581226,
      "learning_rate": 2.0304873909890763e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.625,
      "logps/chosen": -548.0,
      "logps/rejected": -668.0,
      "loss": 0.3941,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.84375,
      "step": 16230
    },
    {
      "epoch": 0.6044927509259087,
      "grad_norm": 11.346556128007297,
      "learning_rate": 2.027297189160817e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.578125,
      "logps/chosen": -532.0,
      "logps/rejected": -704.0,
      "loss": 0.3965,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -4.9375,
      "step": 16240
    },
    {
      "epoch": 0.6048649755262325,
      "grad_norm": 10.18289719032508,
      "learning_rate": 2.0241077854138337e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.296875,
      "logps/chosen": -540.0,
      "logps/rejected": -704.0,
      "loss": 0.3829,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.03125,
      "step": 16250
    },
    {
      "epoch": 0.6052372001265564,
      "grad_norm": 11.53084874276256,
      "learning_rate": 2.0209191851329149e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -500.0,
      "logps/rejected": -668.0,
      "loss": 0.3819,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.828125,
      "rewards/rejected": -4.875,
      "step": 16260
    },
    {
      "epoch": 0.6056094247268802,
      "grad_norm": 9.81006831488332,
      "learning_rate": 2.0177313937014873e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5,
      "logps/chosen": -544.0,
      "logps/rejected": -672.0,
      "loss": 0.3961,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.0,
      "step": 16270
    },
    {
      "epoch": 0.605981649327204,
      "grad_norm": 11.515670226497821,
      "learning_rate": 2.0145444165016164e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.671875,
      "logps/chosen": -572.0,
      "logps/rejected": -724.0,
      "loss": 0.3829,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.28125,
      "step": 16280
    },
    {
      "epoch": 0.6063538739275278,
      "grad_norm": 10.061948419898904,
      "learning_rate": 2.0113582589139917e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.5625,
      "logps/chosen": -560.0,
      "logps/rejected": -704.0,
      "loss": 0.3978,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.75,
      "rewards/rejected": -5.0625,
      "step": 16290
    },
    {
      "epoch": 0.6067260985278518,
      "grad_norm": 11.635022675467932,
      "learning_rate": 2.0081729263179165e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.34375,
      "logps/chosen": -502.0,
      "logps/rejected": -652.0,
      "loss": 0.3767,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -4.6875,
      "step": 16300
    },
    {
      "epoch": 0.6070983231281756,
      "grad_norm": 13.244979887718303,
      "learning_rate": 2.0049884240913056e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.515625,
      "logps/chosen": -540.0,
      "logps/rejected": -648.0,
      "loss": 0.4115,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.65625,
      "step": 16310
    },
    {
      "epoch": 0.6074705477284994,
      "grad_norm": 9.92861138915856,
      "learning_rate": 2.0018047576106666e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.65625,
      "logps/chosen": -524.0,
      "logps/rejected": -652.0,
      "loss": 0.3868,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -4.5625,
      "step": 16320
    },
    {
      "epoch": 0.6078427723288232,
      "grad_norm": 11.70010045716,
      "learning_rate": 1.998621932251102e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -560.0,
      "logps/rejected": -672.0,
      "loss": 0.3762,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.875,
      "step": 16330
    },
    {
      "epoch": 0.608214996929147,
      "grad_norm": 10.70083492168648,
      "learning_rate": 1.9954399533862883e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -506.0,
      "logps/rejected": -648.0,
      "loss": 0.4095,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.109375,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.53125,
      "step": 16340
    },
    {
      "epoch": 0.6085872215294709,
      "grad_norm": 11.077189624222086,
      "learning_rate": 1.9922588263884785e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.546875,
      "logps/chosen": -540.0,
      "logps/rejected": -672.0,
      "loss": 0.3958,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.6875,
      "step": 16350
    },
    {
      "epoch": 0.6089594461297947,
      "grad_norm": 9.487351484232182,
      "learning_rate": 1.9890785566284817e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.484375,
      "logps/chosen": -556.0,
      "logps/rejected": -684.0,
      "loss": 0.3844,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.6875,
      "step": 16360
    },
    {
      "epoch": 0.6093316707301185,
      "grad_norm": 13.308204459687627,
      "learning_rate": 1.9858991494756635e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.546875,
      "logps/chosen": -544.0,
      "logps/rejected": -664.0,
      "loss": 0.3913,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.71875,
      "step": 16370
    },
    {
      "epoch": 0.6097038953304423,
      "grad_norm": 10.99504567662108,
      "learning_rate": 1.9827206102979343e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.59375,
      "logps/chosen": -584.0,
      "logps/rejected": -708.0,
      "loss": 0.4161,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.28125,
      "rewards/rejected": -4.9375,
      "step": 16380
    },
    {
      "epoch": 0.6100761199307663,
      "grad_norm": 11.01878151188055,
      "learning_rate": 1.9795429444617334e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.515625,
      "logps/chosen": -568.0,
      "logps/rejected": -672.0,
      "loss": 0.4247,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -4.78125,
      "step": 16390
    },
    {
      "epoch": 0.6104483445310901,
      "grad_norm": 11.34009913382641,
      "learning_rate": 1.9763661573320322e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.453125,
      "logps/chosen": -532.0,
      "logps/rejected": -660.0,
      "loss": 0.4061,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.84375,
      "step": 16400
    },
    {
      "epoch": 0.6108205691314139,
      "grad_norm": 12.516207649816698,
      "learning_rate": 1.9731902542723128e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -540.0,
      "logps/rejected": -656.0,
      "loss": 0.4168,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.15625,
      "rewards/rejected": -4.5625,
      "step": 16410
    },
    {
      "epoch": 0.6111927937317377,
      "grad_norm": 11.127875294713842,
      "learning_rate": 1.9700152406445704e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.671875,
      "logps/chosen": -544.0,
      "logps/rejected": -684.0,
      "loss": 0.3838,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -4.90625,
      "step": 16420
    },
    {
      "epoch": 0.6115650183320616,
      "grad_norm": 13.711852530031221,
      "learning_rate": 1.9668411218092928e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.671875,
      "logps/chosen": -532.0,
      "logps/rejected": -656.0,
      "loss": 0.386,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.359375,
      "rewards/rejected": -4.65625,
      "step": 16430
    },
    {
      "epoch": 0.6119372429323854,
      "grad_norm": 11.672816288129743,
      "learning_rate": 1.963667903125461e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -524.0,
      "logps/rejected": -676.0,
      "loss": 0.3951,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -4.875,
      "step": 16440
    },
    {
      "epoch": 0.6123094675327092,
      "grad_norm": 11.761685316379138,
      "learning_rate": 1.9604955899505346e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.515625,
      "logps/chosen": -524.0,
      "logps/rejected": -676.0,
      "loss": 0.3963,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.75,
      "step": 16450
    },
    {
      "epoch": 0.612681692133033,
      "grad_norm": 11.241597349399761,
      "learning_rate": 1.9573241876404445e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.4375,
      "logps/chosen": -544.0,
      "logps/rejected": -692.0,
      "loss": 0.3776,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -4.9375,
      "step": 16460
    },
    {
      "epoch": 0.613053916733357,
      "grad_norm": 13.281251133890015,
      "learning_rate": 1.9541537015495864e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.53125,
      "logps/chosen": -536.0,
      "logps/rejected": -668.0,
      "loss": 0.3942,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.84375,
      "step": 16470
    },
    {
      "epoch": 0.6134261413336808,
      "grad_norm": 12.159989660280024,
      "learning_rate": 1.9509841370308034e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.40625,
      "logps/chosen": -544.0,
      "logps/rejected": -700.0,
      "loss": 0.3957,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.78125,
      "rewards/rejected": -5.03125,
      "step": 16480
    },
    {
      "epoch": 0.6137983659340046,
      "grad_norm": 11.669568257244658,
      "learning_rate": 1.9478154994353897e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -536.0,
      "logps/rejected": -684.0,
      "loss": 0.3826,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.8125,
      "step": 16490
    },
    {
      "epoch": 0.6141705905343284,
      "grad_norm": 15.29784302071235,
      "learning_rate": 1.9446477941130677e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.578125,
      "logps/chosen": -536.0,
      "logps/rejected": -676.0,
      "loss": 0.414,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.109375,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.59375,
      "step": 16500
    },
    {
      "epoch": 0.6145428151346523,
      "grad_norm": 14.902702829837702,
      "learning_rate": 1.9414810264119907e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.140625,
      "logps/chosen": -552.0,
      "logps/rejected": -724.0,
      "loss": 0.4032,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.75,
      "rewards/rejected": -5.28125,
      "step": 16510
    },
    {
      "epoch": 0.6149150397349761,
      "grad_norm": 13.268384235881625,
      "learning_rate": 1.9383152016787266e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -532.0,
      "logps/rejected": -680.0,
      "loss": 0.3829,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.875,
      "step": 16520
    },
    {
      "epoch": 0.6152872643352999,
      "grad_norm": 12.223311076078794,
      "learning_rate": 1.935150325258253e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -552.0,
      "logps/rejected": -700.0,
      "loss": 0.3918,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.0,
      "step": 16530
    },
    {
      "epoch": 0.6156594889356237,
      "grad_norm": 12.305463212710697,
      "learning_rate": 1.931986402493944e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.546875,
      "logps/chosen": -560.0,
      "logps/rejected": -704.0,
      "loss": 0.3991,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.09375,
      "step": 16540
    },
    {
      "epoch": 0.6160317135359475,
      "grad_norm": 11.240286729865648,
      "learning_rate": 1.9288234387275656e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -556.0,
      "logps/rejected": -680.0,
      "loss": 0.3899,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.90625,
      "step": 16550
    },
    {
      "epoch": 0.6164039381362715,
      "grad_norm": 13.000922915696004,
      "learning_rate": 1.9256614392992647e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.671875,
      "logps/chosen": -504.0,
      "logps/rejected": -652.0,
      "loss": 0.3856,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -4.8125,
      "step": 16560
    },
    {
      "epoch": 0.6167761627365953,
      "grad_norm": 10.888438630883298,
      "learning_rate": 1.9225004095475587e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.671875,
      "logps/chosen": -548.0,
      "logps/rejected": -672.0,
      "loss": 0.4022,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -4.8125,
      "step": 16570
    },
    {
      "epoch": 0.6171483873369191,
      "grad_norm": 11.515537733316387,
      "learning_rate": 1.9193403548093294e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -552.0,
      "logps/rejected": -656.0,
      "loss": 0.4115,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.65625,
      "step": 16580
    },
    {
      "epoch": 0.617520611937243,
      "grad_norm": 11.533455283457247,
      "learning_rate": 1.9161812804198107e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.546875,
      "logps/chosen": -532.0,
      "logps/rejected": -664.0,
      "loss": 0.3892,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.625,
      "rewards/rejected": -4.78125,
      "step": 16590
    },
    {
      "epoch": 0.6178928365375668,
      "grad_norm": 12.895106816189335,
      "learning_rate": 1.9130231917125833e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.546875,
      "logps/chosen": -524.0,
      "logps/rejected": -664.0,
      "loss": 0.4116,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.046875,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.625,
      "step": 16600
    },
    {
      "epoch": 0.6182650611378906,
      "grad_norm": 12.349752277912232,
      "learning_rate": 1.9098660940195618e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5,
      "logps/chosen": -544.0,
      "logps/rejected": -680.0,
      "loss": 0.3881,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.78125,
      "step": 16610
    },
    {
      "epoch": 0.6186372857382144,
      "grad_norm": 12.347214671449272,
      "learning_rate": 1.906709992670989e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.375,
      "logps/chosen": -520.0,
      "logps/rejected": -672.0,
      "loss": 0.4118,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.84375,
      "step": 16620
    },
    {
      "epoch": 0.6190095103385382,
      "grad_norm": 10.428808175700105,
      "learning_rate": 1.9035548929954257e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -548.0,
      "logps/rejected": -680.0,
      "loss": 0.3957,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.8125,
      "step": 16630
    },
    {
      "epoch": 0.6193817349388621,
      "grad_norm": 12.920293212987472,
      "learning_rate": 1.9004008003197398e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.53125,
      "logps/chosen": -540.0,
      "logps/rejected": -668.0,
      "loss": 0.3879,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.75,
      "step": 16640
    },
    {
      "epoch": 0.619753959539186,
      "grad_norm": 10.072185963348142,
      "learning_rate": 1.8972477199691014e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.5,
      "logps/chosen": -502.0,
      "logps/rejected": -632.0,
      "loss": 0.3886,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.96875,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.40625,
      "step": 16650
    },
    {
      "epoch": 0.6201261841395098,
      "grad_norm": 10.697679313030482,
      "learning_rate": 1.8940956572669692e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.203125,
      "logps/chosen": -524.0,
      "logps/rejected": -648.0,
      "loss": 0.4108,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.75,
      "step": 16660
    },
    {
      "epoch": 0.6204984087398336,
      "grad_norm": 9.533817117773951,
      "learning_rate": 1.8909446175350856e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -552.0,
      "logps/rejected": -664.0,
      "loss": 0.3938,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.421875,
      "rewards/rejected": -4.59375,
      "step": 16670
    },
    {
      "epoch": 0.6208706333401575,
      "grad_norm": 11.038292971096084,
      "learning_rate": 1.887794606093465e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.59375,
      "logps/chosen": -540.0,
      "logps/rejected": -704.0,
      "loss": 0.3808,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.59375,
      "rewards/rejected": -4.9375,
      "step": 16680
    },
    {
      "epoch": 0.6212428579404813,
      "grad_norm": 10.906083150915899,
      "learning_rate": 1.8846456282603856e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.625,
      "logps/chosen": -524.0,
      "logps/rejected": -680.0,
      "loss": 0.3799,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -4.84375,
      "step": 16690
    },
    {
      "epoch": 0.6216150825408051,
      "grad_norm": 12.302360385532486,
      "learning_rate": 1.8814976893523804e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.46875,
      "logps/chosen": -536.0,
      "logps/rejected": -688.0,
      "loss": 0.3798,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.671875,
      "rewards/rejected": -4.96875,
      "step": 16700
    },
    {
      "epoch": 0.6219873071411289,
      "grad_norm": 11.651889290999334,
      "learning_rate": 1.8783507946842291e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -524.0,
      "logps/rejected": -668.0,
      "loss": 0.4051,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.75,
      "step": 16710
    },
    {
      "epoch": 0.6223595317414528,
      "grad_norm": 12.687090652127434,
      "learning_rate": 1.8752049495689483e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.4375,
      "logps/chosen": -532.0,
      "logps/rejected": -684.0,
      "loss": 0.4085,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.703125,
      "rewards/rejected": -4.9375,
      "step": 16720
    },
    {
      "epoch": 0.6227317563417766,
      "grad_norm": 10.697420755764199,
      "learning_rate": 1.872060159317782e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -506.0,
      "logps/rejected": -668.0,
      "loss": 0.3798,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.6875,
      "rewards/rejected": -4.78125,
      "step": 16730
    },
    {
      "epoch": 0.6231039809421005,
      "grad_norm": 11.869174794677454,
      "learning_rate": 1.8689164292401932e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.59375,
      "logps/chosen": -536.0,
      "logps/rejected": -664.0,
      "loss": 0.4031,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.8125,
      "step": 16740
    },
    {
      "epoch": 0.6234762055424243,
      "grad_norm": 13.052471834379485,
      "learning_rate": 1.865773764643855e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.421875,
      "logps/chosen": -572.0,
      "logps/rejected": -688.0,
      "loss": 0.4114,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -4.8125,
      "step": 16750
    },
    {
      "epoch": 0.6238484301427482,
      "grad_norm": 9.834531216266502,
      "learning_rate": 1.8626321708346432e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.359375,
      "logps/chosen": -564.0,
      "logps/rejected": -728.0,
      "loss": 0.3816,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -5.09375,
      "step": 16760
    },
    {
      "epoch": 0.624220654743072,
      "grad_norm": 9.561930608523639,
      "learning_rate": 1.8594916531166226e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.46875,
      "logps/chosen": -568.0,
      "logps/rejected": -688.0,
      "loss": 0.4131,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.90625,
      "step": 16770
    },
    {
      "epoch": 0.6245928793433958,
      "grad_norm": 12.249134374347507,
      "learning_rate": 1.8563522167920445e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.4375,
      "logps/chosen": -506.0,
      "logps/rejected": -668.0,
      "loss": 0.3845,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.8046875,
      "rewards/rejected": -4.875,
      "step": 16780
    },
    {
      "epoch": 0.6249651039437196,
      "grad_norm": 13.514585650818214,
      "learning_rate": 1.853213867161332e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.703125,
      "logps/chosen": -560.0,
      "logps/rejected": -712.0,
      "loss": 0.3843,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.796875,
      "rewards/rejected": -5.3125,
      "step": 16790
    },
    {
      "epoch": 0.6253373285440434,
      "grad_norm": 13.099569141464192,
      "learning_rate": 1.8500766095230746e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.578125,
      "logps/chosen": -516.0,
      "logps/rejected": -676.0,
      "loss": 0.398,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -4.78125,
      "step": 16800
    },
    {
      "epoch": 0.6257095531443673,
      "grad_norm": 12.524591107985774,
      "learning_rate": 1.8469404491740193e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.625,
      "logps/chosen": -532.0,
      "logps/rejected": -648.0,
      "loss": 0.4162,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.6875,
      "step": 16810
    },
    {
      "epoch": 0.6260817777446911,
      "grad_norm": 9.718046026339756,
      "learning_rate": 1.8438053914090574e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -536.0,
      "logps/rejected": -680.0,
      "loss": 0.3941,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -4.6875,
      "step": 16820
    },
    {
      "epoch": 0.626454002345015,
      "grad_norm": 11.025619678547592,
      "learning_rate": 1.8406714415212216e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -524.0,
      "logps/rejected": -652.0,
      "loss": 0.3931,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.59375,
      "step": 16830
    },
    {
      "epoch": 0.6268262269453388,
      "grad_norm": 11.44606248994624,
      "learning_rate": 1.8375386048016722e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.546875,
      "logps/chosen": -524.0,
      "logps/rejected": -644.0,
      "loss": 0.3889,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.6875,
      "step": 16840
    },
    {
      "epoch": 0.6271984515456627,
      "grad_norm": 9.12849024725122,
      "learning_rate": 1.8344068865396922e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.546875,
      "logps/chosen": -512.0,
      "logps/rejected": -644.0,
      "loss": 0.3872,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.65625,
      "step": 16850
    },
    {
      "epoch": 0.6275706761459865,
      "grad_norm": 11.537248521229607,
      "learning_rate": 1.8312762920226731e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.515625,
      "logps/chosen": -540.0,
      "logps/rejected": -656.0,
      "loss": 0.3903,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.8125,
      "step": 16860
    },
    {
      "epoch": 0.6279429007463103,
      "grad_norm": 11.96518532066874,
      "learning_rate": 1.8281468265361125e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.515625,
      "logps/chosen": -528.0,
      "logps/rejected": -680.0,
      "loss": 0.4121,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.8125,
      "step": 16870
    },
    {
      "epoch": 0.6283151253466341,
      "grad_norm": 11.999963973204524,
      "learning_rate": 1.8250184953636e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.5,
      "logps/chosen": -524.0,
      "logps/rejected": -668.0,
      "loss": 0.4138,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.65625,
      "step": 16880
    },
    {
      "epoch": 0.628687349946958,
      "grad_norm": 13.447501530342143,
      "learning_rate": 1.8218913037868102e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.546875,
      "logps/chosen": -552.0,
      "logps/rejected": -704.0,
      "loss": 0.3798,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.03125,
      "step": 16890
    },
    {
      "epoch": 0.6290595745472818,
      "grad_norm": 13.97747486719105,
      "learning_rate": 1.8187652570854945e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -552.0,
      "logps/rejected": -680.0,
      "loss": 0.3758,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.84375,
      "step": 16900
    },
    {
      "epoch": 0.6294317991476057,
      "grad_norm": 10.519501998631196,
      "learning_rate": 1.81564036053747e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.28125,
      "logps/chosen": -536.0,
      "logps/rejected": -684.0,
      "loss": 0.4006,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -4.96875,
      "step": 16910
    },
    {
      "epoch": 0.6298040237479295,
      "grad_norm": 10.838536951897954,
      "learning_rate": 1.812516619418613e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.421875,
      "logps/chosen": -524.0,
      "logps/rejected": -632.0,
      "loss": 0.4015,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.28125,
      "rewards/rejected": -4.53125,
      "step": 16920
    },
    {
      "epoch": 0.6301762483482534,
      "grad_norm": 10.276157361472475,
      "learning_rate": 1.8093940390028483e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.625,
      "logps/chosen": -524.0,
      "logps/rejected": -664.0,
      "loss": 0.3988,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.65625,
      "step": 16930
    },
    {
      "epoch": 0.6305484729485772,
      "grad_norm": 9.527081242893685,
      "learning_rate": 1.806272624562143e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.625,
      "logps/chosen": -568.0,
      "logps/rejected": -708.0,
      "loss": 0.387,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.9375,
      "step": 16940
    },
    {
      "epoch": 0.630920697548901,
      "grad_norm": 10.289019910195165,
      "learning_rate": 1.8031523813664923e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.609375,
      "logps/chosen": -536.0,
      "logps/rejected": -660.0,
      "loss": 0.3857,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.546875,
      "rewards/rejected": -4.78125,
      "step": 16950
    },
    {
      "epoch": 0.6312929221492248,
      "grad_norm": 12.458153041552102,
      "learning_rate": 1.800033314683917e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.546875,
      "logps/chosen": -568.0,
      "logps/rejected": -712.0,
      "loss": 0.4135,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.96875,
      "step": 16960
    },
    {
      "epoch": 0.6316651467495487,
      "grad_norm": 13.508309158195498,
      "learning_rate": 1.7969154297804507e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.515625,
      "logps/chosen": -540.0,
      "logps/rejected": -684.0,
      "loss": 0.4082,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.875,
      "step": 16970
    },
    {
      "epoch": 0.6320373713498725,
      "grad_norm": 12.606242137617944,
      "learning_rate": 1.7937987319201297e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.53125,
      "logps/chosen": -500.0,
      "logps/rejected": -648.0,
      "loss": 0.3815,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.0,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.5625,
      "step": 16980
    },
    {
      "epoch": 0.6324095959501963,
      "grad_norm": 12.540112633413608,
      "learning_rate": 1.79068322636499e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.515625,
      "logps/chosen": -556.0,
      "logps/rejected": -704.0,
      "loss": 0.3921,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.6875,
      "rewards/rejected": -5.03125,
      "step": 16990
    },
    {
      "epoch": 0.6327818205505202,
      "grad_norm": 11.79392745828706,
      "learning_rate": 1.7875689183750504e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.375,
      "logps/chosen": -536.0,
      "logps/rejected": -656.0,
      "loss": 0.4018,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.65625,
      "step": 17000
    },
    {
      "epoch": 0.6331540451508441,
      "grad_norm": 11.304235354935756,
      "learning_rate": 1.7844558132083117e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.453125,
      "logps/chosen": -516.0,
      "logps/rejected": -644.0,
      "loss": 0.4148,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.625,
      "step": 17010
    },
    {
      "epoch": 0.6335262697511679,
      "grad_norm": 10.708249486952791,
      "learning_rate": 1.7813439161207412e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.265625,
      "logps/chosen": -520.0,
      "logps/rejected": -656.0,
      "loss": 0.3906,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.59375,
      "rewards/rejected": -4.71875,
      "step": 17020
    },
    {
      "epoch": 0.6338984943514917,
      "grad_norm": 12.164199299519527,
      "learning_rate": 1.778233232366268e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -524.0,
      "logps/rejected": -660.0,
      "loss": 0.3769,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.75,
      "step": 17030
    },
    {
      "epoch": 0.6342707189518155,
      "grad_norm": 10.773335650624206,
      "learning_rate": 1.7751237671967707e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.453125,
      "logps/chosen": -516.0,
      "logps/rejected": -668.0,
      "loss": 0.3751,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.078125,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -4.90625,
      "step": 17040
    },
    {
      "epoch": 0.6346429435521393,
      "grad_norm": 12.117089602984226,
      "learning_rate": 1.772015525862073e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -524.0,
      "logps/rejected": -664.0,
      "loss": 0.3904,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.75,
      "step": 17050
    },
    {
      "epoch": 0.6350151681524632,
      "grad_norm": 12.206156311406057,
      "learning_rate": 1.7689085136099325e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -524.0,
      "logps/rejected": -696.0,
      "loss": 0.3688,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.03125,
      "step": 17060
    },
    {
      "epoch": 0.635387392752787,
      "grad_norm": 9.446892850638676,
      "learning_rate": 1.7658027356860284e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.671875,
      "logps/chosen": -548.0,
      "logps/rejected": -692.0,
      "loss": 0.3936,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.9375,
      "step": 17070
    },
    {
      "epoch": 0.6357596173531108,
      "grad_norm": 14.79557295683498,
      "learning_rate": 1.7626981973339597e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.671875,
      "logps/chosen": -540.0,
      "logps/rejected": -664.0,
      "loss": 0.392,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.90625,
      "step": 17080
    },
    {
      "epoch": 0.6361318419534348,
      "grad_norm": 9.263405589444734,
      "learning_rate": 1.7595949037952298e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.515625,
      "logps/chosen": -556.0,
      "logps/rejected": -688.0,
      "loss": 0.3829,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -4.96875,
      "step": 17090
    },
    {
      "epoch": 0.6365040665537586,
      "grad_norm": 11.502733958062342,
      "learning_rate": 1.7564928603092432e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.609375,
      "logps/chosen": -520.0,
      "logps/rejected": -692.0,
      "loss": 0.4065,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.8125,
      "rewards/rejected": -4.9375,
      "step": 17100
    },
    {
      "epoch": 0.6368762911540824,
      "grad_norm": 10.055557366150495,
      "learning_rate": 1.7533920721132894e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5,
      "logps/chosen": -508.0,
      "logps/rejected": -676.0,
      "loss": 0.3747,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.65625,
      "rewards/rejected": -4.8125,
      "step": 17110
    },
    {
      "epoch": 0.6372485157544062,
      "grad_norm": 10.659229002757895,
      "learning_rate": 1.7502925444425448e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.34375,
      "logps/chosen": -532.0,
      "logps/rejected": -676.0,
      "loss": 0.3887,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.84375,
      "step": 17120
    },
    {
      "epoch": 0.63762074035473,
      "grad_norm": 13.240007431924052,
      "learning_rate": 1.7471942825300512e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.375,
      "logps/chosen": -592.0,
      "logps/rejected": -720.0,
      "loss": 0.4036,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.09375,
      "step": 17130
    },
    {
      "epoch": 0.6379929649550539,
      "grad_norm": 11.242320832422104,
      "learning_rate": 1.744097291606718e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.578125,
      "logps/chosen": -532.0,
      "logps/rejected": -672.0,
      "loss": 0.3613,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.65625,
      "rewards/rejected": -4.75,
      "step": 17140
    },
    {
      "epoch": 0.6383651895553777,
      "grad_norm": 11.105199432862797,
      "learning_rate": 1.741001576901308e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.453125,
      "logps/chosen": -528.0,
      "logps/rejected": -680.0,
      "loss": 0.3624,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.71875,
      "rewards/rejected": -4.90625,
      "step": 17150
    },
    {
      "epoch": 0.6387374141557015,
      "grad_norm": 12.508812022878777,
      "learning_rate": 1.7379071436404268e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.484375,
      "logps/chosen": -560.0,
      "logps/rejected": -684.0,
      "loss": 0.3842,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -5.09375,
      "step": 17160
    },
    {
      "epoch": 0.6391096387560253,
      "grad_norm": 11.62128166326793,
      "learning_rate": 1.73481399704852e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5,
      "logps/chosen": -524.0,
      "logps/rejected": -708.0,
      "loss": 0.4026,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.796875,
      "rewards/rejected": -5.0625,
      "step": 17170
    },
    {
      "epoch": 0.6394818633563493,
      "grad_norm": 11.046556948540116,
      "learning_rate": 1.7317221423478556e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -532.0,
      "logps/rejected": -664.0,
      "loss": 0.3868,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.546875,
      "rewards/rejected": -4.78125,
      "step": 17180
    },
    {
      "epoch": 0.6398540879566731,
      "grad_norm": 15.028024968143566,
      "learning_rate": 1.728631584758528e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.515625,
      "logps/chosen": -544.0,
      "logps/rejected": -668.0,
      "loss": 0.3959,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.875,
      "step": 17190
    },
    {
      "epoch": 0.6402263125569969,
      "grad_norm": 12.630700137505862,
      "learning_rate": 1.7255423294984342e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.515625,
      "logps/chosen": -544.0,
      "logps/rejected": -648.0,
      "loss": 0.3892,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.328125,
      "rewards/rejected": -4.59375,
      "step": 17200
    },
    {
      "epoch": 0.6405985371573207,
      "grad_norm": 10.933950917194888,
      "learning_rate": 1.7224543817832777e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.5625,
      "logps/chosen": -540.0,
      "logps/rejected": -688.0,
      "loss": 0.3911,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.75,
      "rewards/rejected": -4.90625,
      "step": 17210
    },
    {
      "epoch": 0.6409707617576446,
      "grad_norm": 11.911589660593462,
      "learning_rate": 1.719367746826553e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.609375,
      "logps/chosen": -504.0,
      "logps/rejected": -652.0,
      "loss": 0.3882,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.6875,
      "step": 17220
    },
    {
      "epoch": 0.6413429863579684,
      "grad_norm": 10.630298280286548,
      "learning_rate": 1.716282429839535e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.453125,
      "logps/chosen": -520.0,
      "logps/rejected": -644.0,
      "loss": 0.4005,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.75,
      "step": 17230
    },
    {
      "epoch": 0.6417152109582922,
      "grad_norm": 10.805644399529568,
      "learning_rate": 1.7131984360312795e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.640625,
      "logps/chosen": -536.0,
      "logps/rejected": -680.0,
      "loss": 0.3832,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.75,
      "step": 17240
    },
    {
      "epoch": 0.642087435558616,
      "grad_norm": 9.95859666167271,
      "learning_rate": 1.7101157706086016e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.46875,
      "logps/chosen": -516.0,
      "logps/rejected": -664.0,
      "loss": 0.3901,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -4.9375,
      "step": 17250
    },
    {
      "epoch": 0.6424596601589398,
      "grad_norm": 11.697965580567141,
      "learning_rate": 1.70703443877608e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.484375,
      "logps/chosen": -516.0,
      "logps/rejected": -664.0,
      "loss": 0.4272,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.125,
      "rewards/margins": 1.65625,
      "rewards/rejected": -4.78125,
      "step": 17260
    },
    {
      "epoch": 0.6428318847592638,
      "grad_norm": 11.794064953378221,
      "learning_rate": 1.703954445736036e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -544.0,
      "logps/rejected": -700.0,
      "loss": 0.3958,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.640625,
      "rewards/rejected": -4.90625,
      "step": 17270
    },
    {
      "epoch": 0.6432041093595876,
      "grad_norm": 13.36439532203261,
      "learning_rate": 1.7008757966885368e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -548.0,
      "logps/rejected": -716.0,
      "loss": 0.3847,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.8671875,
      "rewards/rejected": -5.1875,
      "step": 17280
    },
    {
      "epoch": 0.6435763339599114,
      "grad_norm": 12.268813931921988,
      "learning_rate": 1.6977984968313735e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5625,
      "logps/chosen": -536.0,
      "logps/rejected": -704.0,
      "loss": 0.3726,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.0625,
      "step": 17290
    },
    {
      "epoch": 0.6439485585602353,
      "grad_norm": 14.30724469165668,
      "learning_rate": 1.6947225513600646e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.40625,
      "logps/chosen": -532.0,
      "logps/rejected": -708.0,
      "loss": 0.3922,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.828125,
      "rewards/rejected": -5.0,
      "step": 17300
    },
    {
      "epoch": 0.6443207831605591,
      "grad_norm": 11.93403989892117,
      "learning_rate": 1.6916479654678417e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.578125,
      "logps/chosen": -532.0,
      "logps/rejected": -668.0,
      "loss": 0.3827,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.75,
      "step": 17310
    },
    {
      "epoch": 0.6446930077608829,
      "grad_norm": 10.07825879621007,
      "learning_rate": 1.6885747443456362e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.53125,
      "logps/chosen": -532.0,
      "logps/rejected": -656.0,
      "loss": 0.3734,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.65625,
      "step": 17320
    },
    {
      "epoch": 0.6450652323612067,
      "grad_norm": 12.66208308616062,
      "learning_rate": 1.6855028931820826e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.609375,
      "logps/chosen": -544.0,
      "logps/rejected": -708.0,
      "loss": 0.3599,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.78125,
      "rewards/rejected": -5.03125,
      "step": 17330
    },
    {
      "epoch": 0.6454374569615305,
      "grad_norm": 12.477715852961811,
      "learning_rate": 1.6824324171634946e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.546875,
      "logps/chosen": -568.0,
      "logps/rejected": -732.0,
      "loss": 0.379,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.8359375,
      "rewards/rejected": -5.3125,
      "step": 17340
    },
    {
      "epoch": 0.6458096815618545,
      "grad_norm": 10.341039232211411,
      "learning_rate": 1.679363321473871e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.59375,
      "logps/chosen": -564.0,
      "logps/rejected": -744.0,
      "loss": 0.4022,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.3125,
      "step": 17350
    },
    {
      "epoch": 0.6461819061621783,
      "grad_norm": 15.170728943302171,
      "learning_rate": 1.6762956112948756e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.65625,
      "logps/chosen": -556.0,
      "logps/rejected": -684.0,
      "loss": 0.4085,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.125,
      "step": 17360
    },
    {
      "epoch": 0.6465541307625021,
      "grad_norm": 11.490449289330058,
      "learning_rate": 1.6732292918058356e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.46875,
      "logps/chosen": -532.0,
      "logps/rejected": -696.0,
      "loss": 0.3955,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.03125,
      "step": 17370
    },
    {
      "epoch": 0.6469263553628259,
      "grad_norm": 10.028724016391232,
      "learning_rate": 1.670164368183728e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.546875,
      "logps/chosen": -520.0,
      "logps/rejected": -672.0,
      "loss": 0.3713,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -4.9375,
      "step": 17380
    },
    {
      "epoch": 0.6472985799631498,
      "grad_norm": 8.741796708062667,
      "learning_rate": 1.6671008456031746e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.453125,
      "logps/chosen": -510.0,
      "logps/rejected": -664.0,
      "loss": 0.3879,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.625,
      "rewards/rejected": -4.78125,
      "step": 17390
    },
    {
      "epoch": 0.6476708045634736,
      "grad_norm": 10.737139317703354,
      "learning_rate": 1.6640387292364337e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.609375,
      "logps/chosen": -536.0,
      "logps/rejected": -640.0,
      "loss": 0.4157,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.5625,
      "step": 17400
    },
    {
      "epoch": 0.6480430291637974,
      "grad_norm": 11.040259749136776,
      "learning_rate": 1.6609780242533845e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -540.0,
      "logps/rejected": -680.0,
      "loss": 0.374,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -4.96875,
      "step": 17410
    },
    {
      "epoch": 0.6484152537641212,
      "grad_norm": 11.305431946774412,
      "learning_rate": 1.657918735821528e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.65625,
      "logps/chosen": -528.0,
      "logps/rejected": -676.0,
      "loss": 0.4015,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.9375,
      "step": 17420
    },
    {
      "epoch": 0.6487874783644451,
      "grad_norm": 11.472797877792559,
      "learning_rate": 1.6548608691059702e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.578125,
      "logps/chosen": -552.0,
      "logps/rejected": -704.0,
      "loss": 0.4118,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.03125,
      "step": 17430
    },
    {
      "epoch": 0.649159702964769,
      "grad_norm": 10.417707329555203,
      "learning_rate": 1.6518044292694192e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.6875,
      "logps/chosen": -552.0,
      "logps/rejected": -680.0,
      "loss": 0.3933,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -4.78125,
      "step": 17440
    },
    {
      "epoch": 0.6495319275650928,
      "grad_norm": 10.026534087321862,
      "learning_rate": 1.6487494214721726e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.671875,
      "logps/chosen": -544.0,
      "logps/rejected": -716.0,
      "loss": 0.3988,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.1875,
      "step": 17450
    },
    {
      "epoch": 0.6499041521654166,
      "grad_norm": 11.12725206047671,
      "learning_rate": 1.6456958508721102e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5625,
      "logps/chosen": -536.0,
      "logps/rejected": -704.0,
      "loss": 0.3764,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.890625,
      "rewards/rejected": -5.1875,
      "step": 17460
    },
    {
      "epoch": 0.6502763767657405,
      "grad_norm": 12.454124019623547,
      "learning_rate": 1.6426437226246885e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.75,
      "logps/chosen": -548.0,
      "logps/rejected": -720.0,
      "loss": 0.4014,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.8671875,
      "rewards/rejected": -5.25,
      "step": 17470
    },
    {
      "epoch": 0.6506486013660643,
      "grad_norm": 14.903026533640439,
      "learning_rate": 1.6395930418829228e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.640625,
      "logps/chosen": -564.0,
      "logps/rejected": -700.0,
      "loss": 0.3966,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -5.125,
      "step": 17480
    },
    {
      "epoch": 0.6510208259663881,
      "grad_norm": 12.339449462373711,
      "learning_rate": 1.63654381379739e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5,
      "logps/chosen": -540.0,
      "logps/rejected": -696.0,
      "loss": 0.387,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.09375,
      "step": 17490
    },
    {
      "epoch": 0.6513930505667119,
      "grad_norm": 11.782816385523233,
      "learning_rate": 1.6334960435162112e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.625,
      "logps/chosen": -536.0,
      "logps/rejected": -688.0,
      "loss": 0.392,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.78125,
      "rewards/rejected": -4.9375,
      "step": 17500
    },
    {
      "epoch": 0.6517652751670358,
      "grad_norm": 11.703400235087102,
      "learning_rate": 1.6304497361850483e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -536.0,
      "logps/rejected": -704.0,
      "loss": 0.3957,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.890625,
      "rewards/rejected": -5.21875,
      "step": 17510
    },
    {
      "epoch": 0.6521374997673596,
      "grad_norm": 11.986552926799808,
      "learning_rate": 1.6274048969470912e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.65625,
      "logps/chosen": -520.0,
      "logps/rejected": -676.0,
      "loss": 0.3875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -4.90625,
      "step": 17520
    },
    {
      "epoch": 0.6525097243676835,
      "grad_norm": 11.886513115315028,
      "learning_rate": 1.6243615309430535e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -520.0,
      "logps/rejected": -656.0,
      "loss": 0.3746,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.71875,
      "rewards/rejected": -4.9375,
      "step": 17530
    },
    {
      "epoch": 0.6528819489680073,
      "grad_norm": 12.413952662572362,
      "learning_rate": 1.621319643311159e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.65625,
      "logps/chosen": -568.0,
      "logps/rejected": -708.0,
      "loss": 0.3889,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -5.125,
      "step": 17540
    },
    {
      "epoch": 0.6532541735683312,
      "grad_norm": 12.341486765301559,
      "learning_rate": 1.618279239187138e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.609375,
      "logps/chosen": -564.0,
      "logps/rejected": -692.0,
      "loss": 0.3937,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.96875,
      "step": 17550
    },
    {
      "epoch": 0.653626398168655,
      "grad_norm": 10.862780384479622,
      "learning_rate": 1.6152403237042145e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.28125,
      "logps/chosen": -512.0,
      "logps/rejected": -656.0,
      "loss": 0.4092,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.6875,
      "step": 17560
    },
    {
      "epoch": 0.6539986227689788,
      "grad_norm": 13.12071394216856,
      "learning_rate": 1.6122029019930987e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.40625,
      "logps/chosen": -520.0,
      "logps/rejected": -704.0,
      "loss": 0.3761,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.90625,
      "rewards/rejected": -5.125,
      "step": 17570
    },
    {
      "epoch": 0.6543708473693026,
      "grad_norm": 11.629256728671239,
      "learning_rate": 1.609166979181981e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.625,
      "logps/chosen": -548.0,
      "logps/rejected": -704.0,
      "loss": 0.3845,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.15625,
      "step": 17580
    },
    {
      "epoch": 0.6547430719696264,
      "grad_norm": 11.494363285975151,
      "learning_rate": 1.6061325603965193e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.671875,
      "logps/chosen": -604.0,
      "logps/rejected": -732.0,
      "loss": 0.3839,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.21875,
      "step": 17590
    },
    {
      "epoch": 0.6551152965699503,
      "grad_norm": 12.585856794033653,
      "learning_rate": 1.6030996507598327e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.40625,
      "logps/chosen": -552.0,
      "logps/rejected": -704.0,
      "loss": 0.3855,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.90625,
      "step": 17600
    },
    {
      "epoch": 0.6554875211702741,
      "grad_norm": 13.604802688252091,
      "learning_rate": 1.6000682553924922e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.59375,
      "logps/chosen": -540.0,
      "logps/rejected": -652.0,
      "loss": 0.3856,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.75,
      "step": 17610
    },
    {
      "epoch": 0.655859745770598,
      "grad_norm": 13.51762553661304,
      "learning_rate": 1.5970383794125128e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.609375,
      "logps/chosen": -568.0,
      "logps/rejected": -720.0,
      "loss": 0.3858,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.25,
      "step": 17620
    },
    {
      "epoch": 0.6562319703709218,
      "grad_norm": 10.887105062725135,
      "learning_rate": 1.5940100279353434e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.390625,
      "logps/chosen": -584.0,
      "logps/rejected": -712.0,
      "loss": 0.3805,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.484375,
      "rewards/rejected": -5.125,
      "step": 17630
    },
    {
      "epoch": 0.6566041949712457,
      "grad_norm": 10.824518744420605,
      "learning_rate": 1.5909832060738592e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.71875,
      "logps/chosen": -564.0,
      "logps/rejected": -692.0,
      "loss": 0.3899,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.125,
      "step": 17640
    },
    {
      "epoch": 0.6569764195715695,
      "grad_norm": 12.07069269585763,
      "learning_rate": 1.5879579189383549e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5625,
      "logps/chosen": -544.0,
      "logps/rejected": -712.0,
      "loss": 0.3598,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -5.21875,
      "step": 17650
    },
    {
      "epoch": 0.6573486441718933,
      "grad_norm": 11.069666943244695,
      "learning_rate": 1.5849341716365306e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.484375,
      "logps/chosen": -560.0,
      "logps/rejected": -704.0,
      "loss": 0.409,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.125,
      "step": 17660
    },
    {
      "epoch": 0.6577208687722171,
      "grad_norm": 10.51710869720502,
      "learning_rate": 1.5819119692734892e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.65625,
      "logps/chosen": -536.0,
      "logps/rejected": -676.0,
      "loss": 0.3853,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -4.96875,
      "step": 17670
    },
    {
      "epoch": 0.658093093372541,
      "grad_norm": 12.701178009523527,
      "learning_rate": 1.5788913169517237e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.640625,
      "logps/chosen": -576.0,
      "logps/rejected": -712.0,
      "loss": 0.3777,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.125,
      "step": 17680
    },
    {
      "epoch": 0.6584653179728648,
      "grad_norm": 12.166116623035679,
      "learning_rate": 1.5758722197711122e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.671875,
      "logps/chosen": -560.0,
      "logps/rejected": -728.0,
      "loss": 0.3917,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 2.015625,
      "rewards/rejected": -5.4375,
      "step": 17690
    },
    {
      "epoch": 0.6588375425731887,
      "grad_norm": 9.851053991137098,
      "learning_rate": 1.5728546828289042e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.671875,
      "logps/chosen": -524.0,
      "logps/rejected": -680.0,
      "loss": 0.3778,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -4.96875,
      "step": 17700
    },
    {
      "epoch": 0.6592097671735125,
      "grad_norm": 9.722901563002967,
      "learning_rate": 1.5698387112197177e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.515625,
      "logps/chosen": -510.0,
      "logps/rejected": -700.0,
      "loss": 0.3885,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.109375,
      "rewards/margins": 2.0625,
      "rewards/rejected": -5.15625,
      "step": 17710
    },
    {
      "epoch": 0.6595819917738364,
      "grad_norm": 10.527125784027737,
      "learning_rate": 1.5668243100355262e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.46875,
      "logps/chosen": -528.0,
      "logps/rejected": -652.0,
      "loss": 0.3831,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.75,
      "step": 17720
    },
    {
      "epoch": 0.6599542163741602,
      "grad_norm": 10.176727874911846,
      "learning_rate": 1.5638114843656524e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.78125,
      "logps/chosen": -524.0,
      "logps/rejected": -664.0,
      "loss": 0.3735,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.6875,
      "step": 17730
    },
    {
      "epoch": 0.660326440974484,
      "grad_norm": 10.368192200199704,
      "learning_rate": 1.5608002392967603e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.78125,
      "logps/chosen": -560.0,
      "logps/rejected": -720.0,
      "loss": 0.3834,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.25,
      "step": 17740
    },
    {
      "epoch": 0.6606986655748078,
      "grad_norm": 12.173539515689434,
      "learning_rate": 1.5577905799128422e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.65625,
      "logps/chosen": -516.0,
      "logps/rejected": -668.0,
      "loss": 0.4039,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.875,
      "step": 17750
    },
    {
      "epoch": 0.6610708901751317,
      "grad_norm": 11.44773268632018,
      "learning_rate": 1.5547825112952166e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5625,
      "logps/chosen": -520.0,
      "logps/rejected": -640.0,
      "loss": 0.398,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.625,
      "step": 17760
    },
    {
      "epoch": 0.6614431147754555,
      "grad_norm": 12.233410439937714,
      "learning_rate": 1.551776038522513e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.71875,
      "logps/chosen": -576.0,
      "logps/rejected": -724.0,
      "loss": 0.401,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.515625,
      "rewards/rejected": -5.15625,
      "step": 17770
    },
    {
      "epoch": 0.6618153393757793,
      "grad_norm": 12.190066794660686,
      "learning_rate": 1.54877116667067e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.625,
      "logps/chosen": -548.0,
      "logps/rejected": -708.0,
      "loss": 0.3987,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.0625,
      "step": 17780
    },
    {
      "epoch": 0.6621875639761032,
      "grad_norm": 10.107083189569188,
      "learning_rate": 1.5457679008129203e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.421875,
      "logps/chosen": -552.0,
      "logps/rejected": -696.0,
      "loss": 0.3929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.03125,
      "step": 17790
    },
    {
      "epoch": 0.6625597885764271,
      "grad_norm": 11.076824045387054,
      "learning_rate": 1.542766246019787e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.484375,
      "logps/chosen": -500.0,
      "logps/rejected": -656.0,
      "loss": 0.3903,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.6875,
      "step": 17800
    },
    {
      "epoch": 0.6629320131767509,
      "grad_norm": 10.16424816507088,
      "learning_rate": 1.539766207359073e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.640625,
      "logps/chosen": -560.0,
      "logps/rejected": -696.0,
      "loss": 0.3857,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.78125,
      "step": 17810
    },
    {
      "epoch": 0.6633042377770747,
      "grad_norm": 10.226853281026012,
      "learning_rate": 1.5367677898958515e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.625,
      "logps/chosen": -576.0,
      "logps/rejected": -712.0,
      "loss": 0.3916,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.0,
      "step": 17820
    },
    {
      "epoch": 0.6636764623773985,
      "grad_norm": 13.293106056961738,
      "learning_rate": 1.53377099869246e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.625,
      "logps/chosen": -572.0,
      "logps/rejected": -704.0,
      "loss": 0.3982,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -5.0,
      "step": 17830
    },
    {
      "epoch": 0.6640486869777223,
      "grad_norm": 11.527983305718358,
      "learning_rate": 1.5307758388084884e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.75,
      "logps/chosen": -548.0,
      "logps/rejected": -700.0,
      "loss": 0.393,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.9140625,
      "rewards/rejected": -5.1875,
      "step": 17840
    },
    {
      "epoch": 0.6644209115780462,
      "grad_norm": 12.256789179314866,
      "learning_rate": 1.5277823153007754e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.703125,
      "logps/chosen": -520.0,
      "logps/rejected": -648.0,
      "loss": 0.4081,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.75,
      "step": 17850
    },
    {
      "epoch": 0.66479313617837,
      "grad_norm": 10.493549045225729,
      "learning_rate": 1.5247904332233936e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.734375,
      "logps/chosen": -540.0,
      "logps/rejected": -664.0,
      "loss": 0.3852,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.84375,
      "step": 17860
    },
    {
      "epoch": 0.6651653607786938,
      "grad_norm": 11.551033439860317,
      "learning_rate": 1.521800197627647e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -532.0,
      "logps/rejected": -704.0,
      "loss": 0.3857,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.0625,
      "step": 17870
    },
    {
      "epoch": 0.6655375853790177,
      "grad_norm": 11.677026513379552,
      "learning_rate": 1.5188116135620577e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.625,
      "logps/chosen": -560.0,
      "logps/rejected": -696.0,
      "loss": 0.4288,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.875,
      "step": 17880
    },
    {
      "epoch": 0.6659098099793416,
      "grad_norm": 10.469132511292617,
      "learning_rate": 1.515824686072361e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.609375,
      "logps/chosen": -536.0,
      "logps/rejected": -672.0,
      "loss": 0.3919,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.75,
      "step": 17890
    },
    {
      "epoch": 0.6662820345796654,
      "grad_norm": 10.440616485956781,
      "learning_rate": 1.5128394202014948e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.421875,
      "logps/chosen": -552.0,
      "logps/rejected": -688.0,
      "loss": 0.3653,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.78125,
      "step": 17900
    },
    {
      "epoch": 0.6666542591799892,
      "grad_norm": 12.783175715916622,
      "learning_rate": 1.5098558209895914e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.515625,
      "logps/chosen": -568.0,
      "logps/rejected": -720.0,
      "loss": 0.372,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.796875,
      "rewards/rejected": -5.28125,
      "step": 17910
    },
    {
      "epoch": 0.667026483780313,
      "grad_norm": 10.912352284172448,
      "learning_rate": 1.50687389347397e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -560.0,
      "logps/rejected": -704.0,
      "loss": 0.3896,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -5.03125,
      "step": 17920
    },
    {
      "epoch": 0.6673987083806369,
      "grad_norm": 11.103070773755555,
      "learning_rate": 1.5038936426891259e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.625,
      "logps/chosen": -560.0,
      "logps/rejected": -700.0,
      "loss": 0.3775,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -5.03125,
      "step": 17930
    },
    {
      "epoch": 0.6677709329809607,
      "grad_norm": 11.573850285607623,
      "learning_rate": 1.5009150736667247e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.46875,
      "logps/chosen": -532.0,
      "logps/rejected": -720.0,
      "loss": 0.3605,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -5.0625,
      "step": 17940
    },
    {
      "epoch": 0.6681431575812845,
      "grad_norm": 11.584354356025058,
      "learning_rate": 1.4979381914355928e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.65625,
      "logps/chosen": -572.0,
      "logps/rejected": -728.0,
      "loss": 0.3759,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.34375,
      "step": 17950
    },
    {
      "epoch": 0.6685153821816083,
      "grad_norm": 12.500467256163573,
      "learning_rate": 1.4949630010217087e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.71875,
      "logps/chosen": -560.0,
      "logps/rejected": -708.0,
      "loss": 0.4141,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.0625,
      "step": 17960
    },
    {
      "epoch": 0.6688876067819323,
      "grad_norm": 11.579962940913955,
      "learning_rate": 1.491989507448192e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.53125,
      "logps/chosen": -544.0,
      "logps/rejected": -680.0,
      "loss": 0.39,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.703125,
      "rewards/rejected": -4.90625,
      "step": 17970
    },
    {
      "epoch": 0.6692598313822561,
      "grad_norm": 10.216050597708897,
      "learning_rate": 1.4890177157353017e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -532.0,
      "logps/rejected": -672.0,
      "loss": 0.3731,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.65625,
      "rewards/rejected": -4.9375,
      "step": 17980
    },
    {
      "epoch": 0.6696320559825799,
      "grad_norm": 11.770164630894367,
      "learning_rate": 1.486047630900421e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -548.0,
      "logps/rejected": -700.0,
      "loss": 0.4076,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -4.9375,
      "step": 17990
    },
    {
      "epoch": 0.6700042805829037,
      "grad_norm": 12.600605827685495,
      "learning_rate": 1.4830792579580514e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.65625,
      "logps/chosen": -524.0,
      "logps/rejected": -644.0,
      "loss": 0.3951,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.71875,
      "step": 18000
    },
    {
      "epoch": 0.6703765051832276,
      "grad_norm": 12.686730792329778,
      "learning_rate": 1.4801126019198045e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -528.0,
      "logps/rejected": -668.0,
      "loss": 0.3944,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.9375,
      "step": 18010
    },
    {
      "epoch": 0.6707487297835514,
      "grad_norm": 11.733355103604072,
      "learning_rate": 1.4771476677943934e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.375,
      "logps/chosen": -556.0,
      "logps/rejected": -712.0,
      "loss": 0.3851,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.125,
      "step": 18020
    },
    {
      "epoch": 0.6711209543838752,
      "grad_norm": 11.508653238797978,
      "learning_rate": 1.4741844605876241e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.734375,
      "logps/chosen": -556.0,
      "logps/rejected": -676.0,
      "loss": 0.3702,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.546875,
      "rewards/rejected": -4.90625,
      "step": 18030
    },
    {
      "epoch": 0.671493178984199,
      "grad_norm": 11.58368622840693,
      "learning_rate": 1.4712229853023845e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.5625,
      "logps/chosen": -548.0,
      "logps/rejected": -704.0,
      "loss": 0.3797,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.96875,
      "step": 18040
    },
    {
      "epoch": 0.6718654035845228,
      "grad_norm": 12.267508628368823,
      "learning_rate": 1.468263246938643e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.515625,
      "logps/chosen": -568.0,
      "logps/rejected": -708.0,
      "loss": 0.392,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.15625,
      "step": 18050
    },
    {
      "epoch": 0.6722376281848468,
      "grad_norm": 12.879506370516529,
      "learning_rate": 1.4653052504934327e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -564.0,
      "logps/rejected": -724.0,
      "loss": 0.3912,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.40625,
      "step": 18060
    },
    {
      "epoch": 0.6726098527851706,
      "grad_norm": 11.479699647589294,
      "learning_rate": 1.4623490009608453e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.484375,
      "logps/chosen": -572.0,
      "logps/rejected": -692.0,
      "loss": 0.3776,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.875,
      "step": 18070
    },
    {
      "epoch": 0.6729820773854944,
      "grad_norm": 11.540269669242292,
      "learning_rate": 1.4593945033320252e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.34375,
      "logps/chosen": -544.0,
      "logps/rejected": -680.0,
      "loss": 0.4082,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.90625,
      "step": 18080
    },
    {
      "epoch": 0.6733543019858182,
      "grad_norm": 12.272868826039923,
      "learning_rate": 1.4564417625951558e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.765625,
      "logps/chosen": -560.0,
      "logps/rejected": -704.0,
      "loss": 0.3809,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -5.15625,
      "step": 18090
    },
    {
      "epoch": 0.6737265265861421,
      "grad_norm": 13.702057522667664,
      "learning_rate": 1.4534907837354577e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5,
      "logps/chosen": -560.0,
      "logps/rejected": -724.0,
      "loss": 0.3784,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.8828125,
      "rewards/rejected": -5.40625,
      "step": 18100
    },
    {
      "epoch": 0.6740987511864659,
      "grad_norm": 11.190713932294873,
      "learning_rate": 1.4505415717351752e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.484375,
      "logps/chosen": -552.0,
      "logps/rejected": -728.0,
      "loss": 0.3833,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.859375,
      "rewards/rejected": -5.4375,
      "step": 18110
    },
    {
      "epoch": 0.6744709757867897,
      "grad_norm": 14.415085929297911,
      "learning_rate": 1.4475941315735704e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.546875,
      "logps/chosen": -532.0,
      "logps/rejected": -672.0,
      "loss": 0.3807,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.5,
      "rewards/rejected": -4.875,
      "step": 18120
    },
    {
      "epoch": 0.6748432003871135,
      "grad_norm": 11.449436667442168,
      "learning_rate": 1.4446484682269117e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5,
      "logps/chosen": -536.0,
      "logps/rejected": -692.0,
      "loss": 0.3821,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.03125,
      "step": 18130
    },
    {
      "epoch": 0.6752154249874375,
      "grad_norm": 10.318722766717462,
      "learning_rate": 1.4417045866684698e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.71875,
      "logps/chosen": -572.0,
      "logps/rejected": -724.0,
      "loss": 0.3971,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.8125,
      "rewards/rejected": -5.21875,
      "step": 18140
    },
    {
      "epoch": 0.6755876495877613,
      "grad_norm": 13.564738428430756,
      "learning_rate": 1.4387624918685072e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -584.0,
      "logps/rejected": -716.0,
      "loss": 0.4071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -5.1875,
      "step": 18150
    },
    {
      "epoch": 0.6759598741880851,
      "grad_norm": 11.150811237197654,
      "learning_rate": 1.4358221887942684e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.75,
      "logps/chosen": -584.0,
      "logps/rejected": -728.0,
      "loss": 0.3775,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.28125,
      "step": 18160
    },
    {
      "epoch": 0.6763320987884089,
      "grad_norm": 12.653294790467257,
      "learning_rate": 1.432883682409975e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.671875,
      "logps/chosen": -540.0,
      "logps/rejected": -672.0,
      "loss": 0.3811,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.59375,
      "rewards/rejected": -4.9375,
      "step": 18170
    },
    {
      "epoch": 0.6767043233887328,
      "grad_norm": 11.734390642713516,
      "learning_rate": 1.4299469776768116e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.515625,
      "logps/chosen": -588.0,
      "logps/rejected": -716.0,
      "loss": 0.4071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.5625,
      "rewards/rejected": -5.15625,
      "step": 18180
    },
    {
      "epoch": 0.6770765479890566,
      "grad_norm": 10.621785148909552,
      "learning_rate": 1.4270120795529245e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.578125,
      "logps/chosen": -516.0,
      "logps/rejected": -664.0,
      "loss": 0.3889,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.171875,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -4.84375,
      "step": 18190
    },
    {
      "epoch": 0.6774487725893804,
      "grad_norm": 12.740723497349816,
      "learning_rate": 1.424078992993408e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.578125,
      "logps/chosen": -540.0,
      "logps/rejected": -676.0,
      "loss": 0.397,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.875,
      "step": 18200
    },
    {
      "epoch": 0.6778209971897042,
      "grad_norm": 10.867902521380703,
      "learning_rate": 1.4211477229502996e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.625,
      "logps/chosen": -520.0,
      "logps/rejected": -644.0,
      "loss": 0.3992,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.5625,
      "step": 18210
    },
    {
      "epoch": 0.6781932217900281,
      "grad_norm": 11.197778727587302,
      "learning_rate": 1.418218274372567e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.640625,
      "logps/chosen": -540.0,
      "logps/rejected": -692.0,
      "loss": 0.4027,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.84375,
      "step": 18220
    },
    {
      "epoch": 0.678565446390352,
      "grad_norm": 11.618727863795264,
      "learning_rate": 1.4152906522061047e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5,
      "logps/chosen": -548.0,
      "logps/rejected": -696.0,
      "loss": 0.396,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.0625,
      "step": 18230
    },
    {
      "epoch": 0.6789376709906758,
      "grad_norm": 12.258058765173185,
      "learning_rate": 1.412364861393724e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.734375,
      "logps/chosen": -576.0,
      "logps/rejected": -692.0,
      "loss": 0.3724,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.9375,
      "step": 18240
    },
    {
      "epoch": 0.6793098955909996,
      "grad_norm": 10.165562366013795,
      "learning_rate": 1.4094409068751428e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.6875,
      "logps/chosen": -516.0,
      "logps/rejected": -692.0,
      "loss": 0.3894,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.8046875,
      "rewards/rejected": -5.0,
      "step": 18250
    },
    {
      "epoch": 0.6796821201913235,
      "grad_norm": 11.481994382176554,
      "learning_rate": 1.406518793586981e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.65625,
      "logps/chosen": -552.0,
      "logps/rejected": -720.0,
      "loss": 0.3898,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.890625,
      "rewards/rejected": -5.28125,
      "step": 18260
    },
    {
      "epoch": 0.6800543447916473,
      "grad_norm": 13.030028852175679,
      "learning_rate": 1.4035985264627456e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -548.0,
      "logps/rejected": -700.0,
      "loss": 0.4039,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.25,
      "step": 18270
    },
    {
      "epoch": 0.6804265693919711,
      "grad_norm": 13.156299364016416,
      "learning_rate": 1.400680110432831e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5625,
      "logps/chosen": -532.0,
      "logps/rejected": -688.0,
      "loss": 0.3904,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -5.03125,
      "step": 18280
    },
    {
      "epoch": 0.6807987939922949,
      "grad_norm": 12.778955258062432,
      "learning_rate": 1.3977635504245047e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.25,
      "logps/chosen": -556.0,
      "logps/rejected": -724.0,
      "loss": 0.3574,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.1875,
      "step": 18290
    },
    {
      "epoch": 0.6811710185926187,
      "grad_norm": 12.803608079111353,
      "learning_rate": 1.394848851361901e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.390625,
      "logps/chosen": -548.0,
      "logps/rejected": -684.0,
      "loss": 0.3984,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.9375,
      "step": 18300
    },
    {
      "epoch": 0.6815432431929426,
      "grad_norm": 11.751817712491064,
      "learning_rate": 1.3919360181660107e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5625,
      "logps/chosen": -560.0,
      "logps/rejected": -688.0,
      "loss": 0.3944,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -5.0,
      "step": 18310
    },
    {
      "epoch": 0.6819154677932665,
      "grad_norm": 12.62445599530728,
      "learning_rate": 1.3890250557546763e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.484375,
      "logps/chosen": -564.0,
      "logps/rejected": -712.0,
      "loss": 0.3883,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.03125,
      "step": 18320
    },
    {
      "epoch": 0.6822876923935903,
      "grad_norm": 12.975823663870937,
      "learning_rate": 1.3861159690425806e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.515625,
      "logps/chosen": -504.0,
      "logps/rejected": -648.0,
      "loss": 0.3957,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.109375,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.5625,
      "step": 18330
    },
    {
      "epoch": 0.6826599169939142,
      "grad_norm": 12.291700356451363,
      "learning_rate": 1.3832087629412402e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.6875,
      "logps/chosen": -548.0,
      "logps/rejected": -700.0,
      "loss": 0.3835,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.0,
      "step": 18340
    },
    {
      "epoch": 0.683032141594238,
      "grad_norm": 11.562018615053058,
      "learning_rate": 1.3803034423589982e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.75,
      "logps/chosen": -548.0,
      "logps/rejected": -716.0,
      "loss": 0.3845,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.125,
      "step": 18350
    },
    {
      "epoch": 0.6834043661945618,
      "grad_norm": 12.306150417780664,
      "learning_rate": 1.37740001220101e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.5625,
      "logps/chosen": -544.0,
      "logps/rejected": -692.0,
      "loss": 0.3977,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.84375,
      "step": 18360
    },
    {
      "epoch": 0.6837765907948856,
      "grad_norm": 11.564121683988121,
      "learning_rate": 1.3744984773692425e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.421875,
      "logps/chosen": -560.0,
      "logps/rejected": -728.0,
      "loss": 0.3838,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -5.375,
      "step": 18370
    },
    {
      "epoch": 0.6841488153952094,
      "grad_norm": 10.855508788420106,
      "learning_rate": 1.3715988427624632e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.703125,
      "logps/chosen": -592.0,
      "logps/rejected": -688.0,
      "loss": 0.3915,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.9375,
      "step": 18380
    },
    {
      "epoch": 0.6845210399955333,
      "grad_norm": 13.75788129617736,
      "learning_rate": 1.3687011132762288e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -580.0,
      "logps/rejected": -732.0,
      "loss": 0.3959,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.6875,
      "rewards/rejected": -5.25,
      "step": 18390
    },
    {
      "epoch": 0.6848932645958571,
      "grad_norm": 13.312912481379044,
      "learning_rate": 1.3658052938028834e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.4375,
      "logps/chosen": -584.0,
      "logps/rejected": -704.0,
      "loss": 0.4009,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -5.15625,
      "step": 18400
    },
    {
      "epoch": 0.685265489196181,
      "grad_norm": 11.127864367668694,
      "learning_rate": 1.362911389231541e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.609375,
      "logps/chosen": -604.0,
      "logps/rejected": -724.0,
      "loss": 0.3906,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.1875,
      "step": 18410
    },
    {
      "epoch": 0.6856377137965048,
      "grad_norm": 11.834776576361818,
      "learning_rate": 1.3600194044480866e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -528.0,
      "logps/rejected": -660.0,
      "loss": 0.4099,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.8125,
      "step": 18420
    },
    {
      "epoch": 0.6860099383968287,
      "grad_norm": 14.924285125278754,
      "learning_rate": 1.3571293443351627e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.515625,
      "logps/chosen": -552.0,
      "logps/rejected": -704.0,
      "loss": 0.3988,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.21875,
      "step": 18430
    },
    {
      "epoch": 0.6863821629971525,
      "grad_norm": 13.29780304390838,
      "learning_rate": 1.3542412137721643e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.453125,
      "logps/chosen": -532.0,
      "logps/rejected": -688.0,
      "loss": 0.3907,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -5.0,
      "step": 18440
    },
    {
      "epoch": 0.6867543875974763,
      "grad_norm": 12.75527603909235,
      "learning_rate": 1.351355017635224e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.4375,
      "logps/chosen": -536.0,
      "logps/rejected": -680.0,
      "loss": 0.3896,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.0625,
      "step": 18450
    },
    {
      "epoch": 0.6871266121978001,
      "grad_norm": 11.378207680375498,
      "learning_rate": 1.3484707607972134e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.484375,
      "logps/chosen": -572.0,
      "logps/rejected": -740.0,
      "loss": 0.369,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -5.21875,
      "step": 18460
    },
    {
      "epoch": 0.687498836798124,
      "grad_norm": 10.732802275520267,
      "learning_rate": 1.3455884481277253e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.484375,
      "logps/chosen": -588.0,
      "logps/rejected": -732.0,
      "loss": 0.3754,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.34375,
      "step": 18470
    },
    {
      "epoch": 0.6878710613984478,
      "grad_norm": 14.79183367414506,
      "learning_rate": 1.342708084493075e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -560.0,
      "logps/rejected": -724.0,
      "loss": 0.3663,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.9296875,
      "rewards/rejected": -5.40625,
      "step": 18480
    },
    {
      "epoch": 0.6882432859987716,
      "grad_norm": 13.445209036464414,
      "learning_rate": 1.339829674756285e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.515625,
      "logps/chosen": -524.0,
      "logps/rejected": -692.0,
      "loss": 0.4041,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -5.125,
      "step": 18490
    },
    {
      "epoch": 0.6886155105990955,
      "grad_norm": 11.424913325755815,
      "learning_rate": 1.336953223777077e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.546875,
      "logps/chosen": -552.0,
      "logps/rejected": -688.0,
      "loss": 0.3705,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.875,
      "step": 18500
    },
    {
      "epoch": 0.6889877351994194,
      "grad_norm": 12.68667708410807,
      "learning_rate": 1.3340787364118688e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.53125,
      "logps/chosen": -506.0,
      "logps/rejected": -632.0,
      "loss": 0.412,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -4.5625,
      "step": 18510
    },
    {
      "epoch": 0.6893599597997432,
      "grad_norm": 12.538967497951768,
      "learning_rate": 1.331206217513759e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.578125,
      "logps/chosen": -520.0,
      "logps/rejected": -656.0,
      "loss": 0.401,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.71875,
      "step": 18520
    },
    {
      "epoch": 0.689732184400067,
      "grad_norm": 11.67663033697084,
      "learning_rate": 1.328335671932529e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.640625,
      "logps/chosen": -520.0,
      "logps/rejected": -672.0,
      "loss": 0.393,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -5.03125,
      "step": 18530
    },
    {
      "epoch": 0.6901044090003908,
      "grad_norm": 11.05397803908902,
      "learning_rate": 1.3254671045146222e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.546875,
      "logps/chosen": -564.0,
      "logps/rejected": -688.0,
      "loss": 0.394,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.640625,
      "rewards/rejected": -4.96875,
      "step": 18540
    },
    {
      "epoch": 0.6904766336007147,
      "grad_norm": 12.031286408603037,
      "learning_rate": 1.322600520103146e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.46875,
      "logps/chosen": -540.0,
      "logps/rejected": -732.0,
      "loss": 0.4129,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.9140625,
      "rewards/rejected": -5.3125,
      "step": 18550
    },
    {
      "epoch": 0.6908488582010385,
      "grad_norm": 12.729113058079177,
      "learning_rate": 1.319735923537857e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.59375,
      "logps/chosen": -568.0,
      "logps/rejected": -688.0,
      "loss": 0.4043,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.9375,
      "step": 18560
    },
    {
      "epoch": 0.6912210828013623,
      "grad_norm": 13.157052023475474,
      "learning_rate": 1.3168733196551596e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.46875,
      "logps/chosen": -528.0,
      "logps/rejected": -676.0,
      "loss": 0.3902,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.90625,
      "step": 18570
    },
    {
      "epoch": 0.6915933074016862,
      "grad_norm": 13.585213687919273,
      "learning_rate": 1.314012713288092e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5,
      "logps/chosen": -532.0,
      "logps/rejected": -672.0,
      "loss": 0.3713,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.65625,
      "rewards/rejected": -4.90625,
      "step": 18580
    },
    {
      "epoch": 0.69196553200201,
      "grad_norm": 11.60589700115605,
      "learning_rate": 1.3111541092663168e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.3125,
      "logps/chosen": -560.0,
      "logps/rejected": -732.0,
      "loss": 0.3922,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.78125,
      "rewards/rejected": -5.1875,
      "step": 18590
    },
    {
      "epoch": 0.6923377566023339,
      "grad_norm": 8.896417642720678,
      "learning_rate": 1.308297512416121e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -520.0,
      "logps/rejected": -660.0,
      "loss": 0.3859,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.71875,
      "step": 18600
    },
    {
      "epoch": 0.6927099812026577,
      "grad_norm": 12.226677397772525,
      "learning_rate": 1.3054429275603983e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.65625,
      "logps/chosen": -536.0,
      "logps/rejected": -688.0,
      "loss": 0.3809,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -5.09375,
      "step": 18610
    },
    {
      "epoch": 0.6930822058029815,
      "grad_norm": 12.509453984523534,
      "learning_rate": 1.302590359518651e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.671875,
      "logps/chosen": -616.0,
      "logps/rejected": -708.0,
      "loss": 0.4026,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.859375,
      "rewards/margins": 1.203125,
      "rewards/rejected": -5.0625,
      "step": 18620
    },
    {
      "epoch": 0.6934544304033053,
      "grad_norm": 10.94271668160092,
      "learning_rate": 1.29973981310697e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.671875,
      "logps/chosen": -588.0,
      "logps/rejected": -712.0,
      "loss": 0.3919,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.453125,
      "rewards/rejected": -5.125,
      "step": 18630
    },
    {
      "epoch": 0.6938266550036292,
      "grad_norm": 11.424491263913906,
      "learning_rate": 1.2968912931380375e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.46875,
      "logps/chosen": -552.0,
      "logps/rejected": -728.0,
      "loss": 0.3874,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -5.40625,
      "step": 18640
    },
    {
      "epoch": 0.694198879603953,
      "grad_norm": 12.520017545140862,
      "learning_rate": 1.2940448044211134e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.484375,
      "logps/chosen": -536.0,
      "logps/rejected": -668.0,
      "loss": 0.3871,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.90625,
      "step": 18650
    },
    {
      "epoch": 0.6945711042042768,
      "grad_norm": 11.82497147452956,
      "learning_rate": 1.2912003517620252e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.59375,
      "logps/chosen": -568.0,
      "logps/rejected": -708.0,
      "loss": 0.3701,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.03125,
      "step": 18660
    },
    {
      "epoch": 0.6949433288046007,
      "grad_norm": 12.009961273652873,
      "learning_rate": 1.2883579399631689e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.515625,
      "logps/chosen": -568.0,
      "logps/rejected": -748.0,
      "loss": 0.3845,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.8828125,
      "rewards/rejected": -5.375,
      "step": 18670
    },
    {
      "epoch": 0.6953155534049246,
      "grad_norm": 15.554393208489788,
      "learning_rate": 1.2855175738234886e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -552.0,
      "logps/rejected": -684.0,
      "loss": 0.3765,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.96875,
      "step": 18680
    },
    {
      "epoch": 0.6956877780052484,
      "grad_norm": 12.749271427866903,
      "learning_rate": 1.282679258138479e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -568.0,
      "logps/rejected": -708.0,
      "loss": 0.3706,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.546875,
      "rewards/rejected": -5.0625,
      "step": 18690
    },
    {
      "epoch": 0.6960600026055722,
      "grad_norm": 13.071596731602703,
      "learning_rate": 1.27984299770017e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.46875,
      "logps/chosen": -576.0,
      "logps/rejected": -720.0,
      "loss": 0.3735,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.28125,
      "step": 18700
    },
    {
      "epoch": 0.696432227205896,
      "grad_norm": 13.281893611515642,
      "learning_rate": 1.277008797297123e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5625,
      "logps/chosen": -556.0,
      "logps/rejected": -720.0,
      "loss": 0.3922,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.125,
      "step": 18710
    },
    {
      "epoch": 0.6968044518062199,
      "grad_norm": 10.470702376242498,
      "learning_rate": 1.2741766617144218e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -572.0,
      "logps/rejected": -744.0,
      "loss": 0.3751,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -5.5,
      "step": 18720
    },
    {
      "epoch": 0.6971766764065437,
      "grad_norm": 13.771148288614992,
      "learning_rate": 1.2713465957336633e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -584.0,
      "logps/rejected": -712.0,
      "loss": 0.3977,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -5.03125,
      "step": 18730
    },
    {
      "epoch": 0.6975489010068675,
      "grad_norm": 12.61857954810114,
      "learning_rate": 1.268518604132951e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.375,
      "logps/chosen": -564.0,
      "logps/rejected": -716.0,
      "loss": 0.396,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.96875,
      "step": 18740
    },
    {
      "epoch": 0.6979211256071913,
      "grad_norm": 13.397050190540162,
      "learning_rate": 1.2656926916868833e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -540.0,
      "logps/rejected": -692.0,
      "loss": 0.3876,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.625,
      "rewards/rejected": -4.96875,
      "step": 18750
    },
    {
      "epoch": 0.6982933502075153,
      "grad_norm": 10.690191890648265,
      "learning_rate": 1.262868863166554e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5,
      "logps/chosen": -528.0,
      "logps/rejected": -684.0,
      "loss": 0.3749,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.0625,
      "step": 18760
    },
    {
      "epoch": 0.6986655748078391,
      "grad_norm": 11.691941300212523,
      "learning_rate": 1.260047123339532e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.46875,
      "logps/chosen": -584.0,
      "logps/rejected": -732.0,
      "loss": 0.4038,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.28125,
      "step": 18770
    },
    {
      "epoch": 0.6990377994081629,
      "grad_norm": 11.666827488742731,
      "learning_rate": 1.2572274769698654e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -568.0,
      "logps/rejected": -704.0,
      "loss": 0.4128,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.34375,
      "rewards/rejected": -5.03125,
      "step": 18780
    },
    {
      "epoch": 0.6994100240084867,
      "grad_norm": 13.640528931887356,
      "learning_rate": 1.2544099288180626e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -564.0,
      "logps/rejected": -712.0,
      "loss": 0.384,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.15625,
      "step": 18790
    },
    {
      "epoch": 0.6997822486088106,
      "grad_norm": 14.048288987817221,
      "learning_rate": 1.2515944836410942e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.5,
      "logps/chosen": -588.0,
      "logps/rejected": -728.0,
      "loss": 0.4162,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.25,
      "step": 18800
    },
    {
      "epoch": 0.7001544732091344,
      "grad_norm": 11.070885878622287,
      "learning_rate": 1.248781146192377e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.390625,
      "logps/chosen": -492.0,
      "logps/rejected": -672.0,
      "loss": 0.3832,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -4.84375,
      "step": 18810
    },
    {
      "epoch": 0.7005266978094582,
      "grad_norm": 12.740990906430987,
      "learning_rate": 1.2459699212217713e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5,
      "logps/chosen": -528.0,
      "logps/rejected": -676.0,
      "loss": 0.3742,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.859375,
      "rewards/rejected": -5.0625,
      "step": 18820
    },
    {
      "epoch": 0.700898922409782,
      "grad_norm": 11.754296378198731,
      "learning_rate": 1.2431608134755712e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.28125,
      "logps/chosen": -568.0,
      "logps/rejected": -736.0,
      "loss": 0.3888,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.34375,
      "step": 18830
    },
    {
      "epoch": 0.7012711470101058,
      "grad_norm": 13.750465513567004,
      "learning_rate": 1.2403538276964926e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5,
      "logps/chosen": -568.0,
      "logps/rejected": -720.0,
      "loss": 0.3826,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.1875,
      "step": 18840
    },
    {
      "epoch": 0.7016433716104298,
      "grad_norm": 12.282321511026266,
      "learning_rate": 1.2375489686236724e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.421875,
      "logps/chosen": -544.0,
      "logps/rejected": -712.0,
      "loss": 0.3837,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -5.1875,
      "step": 18850
    },
    {
      "epoch": 0.7020155962107536,
      "grad_norm": 12.569645672940391,
      "learning_rate": 1.2347462409926556e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.484375,
      "logps/chosen": -572.0,
      "logps/rejected": -716.0,
      "loss": 0.3967,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.25,
      "step": 18860
    },
    {
      "epoch": 0.7023878208110774,
      "grad_norm": 12.03637903331165,
      "learning_rate": 1.23194564953539e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.46875,
      "logps/chosen": -572.0,
      "logps/rejected": -696.0,
      "loss": 0.3909,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -5.0625,
      "step": 18870
    },
    {
      "epoch": 0.7027600454114012,
      "grad_norm": 11.85179082754265,
      "learning_rate": 1.229147198980214e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.34375,
      "logps/chosen": -524.0,
      "logps/rejected": -688.0,
      "loss": 0.3555,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.8828125,
      "rewards/rejected": -5.09375,
      "step": 18880
    },
    {
      "epoch": 0.7031322700117251,
      "grad_norm": 12.005977659964522,
      "learning_rate": 1.2263508940518532e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.421875,
      "logps/chosen": -552.0,
      "logps/rejected": -692.0,
      "loss": 0.3964,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.0625,
      "step": 18890
    },
    {
      "epoch": 0.7035044946120489,
      "grad_norm": 12.746921287700621,
      "learning_rate": 1.2235567394714112e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.515625,
      "logps/chosen": -536.0,
      "logps/rejected": -684.0,
      "loss": 0.3825,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -5.03125,
      "step": 18900
    },
    {
      "epoch": 0.7038767192123727,
      "grad_norm": 11.296767159698005,
      "learning_rate": 1.2207647399563597e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.3125,
      "logps/chosen": -568.0,
      "logps/rejected": -732.0,
      "loss": 0.382,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.34375,
      "step": 18910
    },
    {
      "epoch": 0.7042489438126965,
      "grad_norm": 10.60971244343046,
      "learning_rate": 1.2179749002205342e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.46875,
      "logps/chosen": -580.0,
      "logps/rejected": -736.0,
      "loss": 0.4038,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.375,
      "step": 18920
    },
    {
      "epoch": 0.7046211684130205,
      "grad_norm": 10.580006814986287,
      "learning_rate": 1.21518722497412e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.546875,
      "logps/chosen": -540.0,
      "logps/rejected": -704.0,
      "loss": 0.3641,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.90625,
      "rewards/rejected": -5.34375,
      "step": 18930
    },
    {
      "epoch": 0.7049933930133443,
      "grad_norm": 12.100545818995448,
      "learning_rate": 1.212401718923651e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.421875,
      "logps/chosen": -548.0,
      "logps/rejected": -724.0,
      "loss": 0.369,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.28125,
      "step": 18940
    },
    {
      "epoch": 0.7053656176136681,
      "grad_norm": 12.83474744058547,
      "learning_rate": 1.2096183867719981e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.4375,
      "logps/chosen": -588.0,
      "logps/rejected": -756.0,
      "loss": 0.3921,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.9140625,
      "rewards/rejected": -5.53125,
      "step": 18950
    },
    {
      "epoch": 0.7057378422139919,
      "grad_norm": 11.406418034087865,
      "learning_rate": 1.206837233218363e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.46875,
      "logps/chosen": -528.0,
      "logps/rejected": -688.0,
      "loss": 0.3673,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.03125,
      "step": 18960
    },
    {
      "epoch": 0.7061100668143158,
      "grad_norm": 11.184551225463752,
      "learning_rate": 1.2040582629582658e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -576.0,
      "logps/rejected": -744.0,
      "loss": 0.3782,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.40625,
      "step": 18970
    },
    {
      "epoch": 0.7064822914146396,
      "grad_norm": 13.599088227515658,
      "learning_rate": 1.2012814806835432e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.515625,
      "logps/chosen": -536.0,
      "logps/rejected": -704.0,
      "loss": 0.4016,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.78125,
      "rewards/rejected": -5.125,
      "step": 18980
    },
    {
      "epoch": 0.7068545160149634,
      "grad_norm": 12.697452053257445,
      "learning_rate": 1.198506891082338e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.46875,
      "logps/chosen": -528.0,
      "logps/rejected": -688.0,
      "loss": 0.3941,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.0625,
      "step": 18990
    },
    {
      "epoch": 0.7072267406152872,
      "grad_norm": 11.138161992708655,
      "learning_rate": 1.1957344988390903e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.484375,
      "logps/chosen": -540.0,
      "logps/rejected": -680.0,
      "loss": 0.3875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -5.125,
      "step": 19000
    },
    {
      "epoch": 0.707598965215611,
      "grad_norm": 12.6352645100083,
      "learning_rate": 1.192964308634531e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.546875,
      "logps/chosen": -564.0,
      "logps/rejected": -720.0,
      "loss": 0.3901,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.15625,
      "step": 19010
    },
    {
      "epoch": 0.707971189815935,
      "grad_norm": 12.210720748969196,
      "learning_rate": 1.1901963251456704e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.390625,
      "logps/chosen": -584.0,
      "logps/rejected": -724.0,
      "loss": 0.3919,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.3125,
      "step": 19020
    },
    {
      "epoch": 0.7083434144162588,
      "grad_norm": 14.946774120649554,
      "learning_rate": 1.1874305530457967e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -568.0,
      "logps/rejected": -720.0,
      "loss": 0.3852,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.5625,
      "rewards/rejected": -5.0625,
      "step": 19030
    },
    {
      "epoch": 0.7087156390165826,
      "grad_norm": 11.362504850409936,
      "learning_rate": 1.1846669970044629e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -544.0,
      "logps/rejected": -676.0,
      "loss": 0.3812,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -4.96875,
      "step": 19040
    },
    {
      "epoch": 0.7090878636169065,
      "grad_norm": 13.0815950694129,
      "learning_rate": 1.181905661687482e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.578125,
      "logps/chosen": -572.0,
      "logps/rejected": -688.0,
      "loss": 0.3921,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.4375,
      "rewards/rejected": -5.0625,
      "step": 19050
    },
    {
      "epoch": 0.7094600882172303,
      "grad_norm": 10.978063374723309,
      "learning_rate": 1.179146551756914e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.546875,
      "logps/chosen": -556.0,
      "logps/rejected": -720.0,
      "loss": 0.3913,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -5.25,
      "step": 19060
    },
    {
      "epoch": 0.7098323128175541,
      "grad_norm": 11.695164462297809,
      "learning_rate": 1.1763896718710656e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.625,
      "logps/chosen": -564.0,
      "logps/rejected": -700.0,
      "loss": 0.3667,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.484375,
      "rewards/rejected": -5.03125,
      "step": 19070
    },
    {
      "epoch": 0.7102045374178779,
      "grad_norm": 11.325153651785831,
      "learning_rate": 1.1736350266844766e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.5625,
      "logps/chosen": -544.0,
      "logps/rejected": -732.0,
      "loss": 0.387,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.9453125,
      "rewards/rejected": -5.3125,
      "step": 19080
    },
    {
      "epoch": 0.7105767620182017,
      "grad_norm": 13.956737094397026,
      "learning_rate": 1.1708826208479145e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.5625,
      "logps/chosen": -568.0,
      "logps/rejected": -692.0,
      "loss": 0.4108,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.09375,
      "step": 19090
    },
    {
      "epoch": 0.7109489866185256,
      "grad_norm": 11.148688463515755,
      "learning_rate": 1.1681324590083663e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.5625,
      "logps/chosen": -604.0,
      "logps/rejected": -740.0,
      "loss": 0.3739,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.3125,
      "step": 19100
    },
    {
      "epoch": 0.7113212112188495,
      "grad_norm": 14.871574041895649,
      "learning_rate": 1.1653845458090286e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.609375,
      "logps/chosen": -596.0,
      "logps/rejected": -712.0,
      "loss": 0.3773,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.28125,
      "step": 19110
    },
    {
      "epoch": 0.7116934358191733,
      "grad_norm": 13.131442426683304,
      "learning_rate": 1.1626388858893033e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.46875,
      "logps/chosen": -560.0,
      "logps/rejected": -696.0,
      "loss": 0.3885,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.96875,
      "step": 19120
    },
    {
      "epoch": 0.7120656604194971,
      "grad_norm": 12.211279372204208,
      "learning_rate": 1.1598954838847877e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.46875,
      "logps/chosen": -536.0,
      "logps/rejected": -712.0,
      "loss": 0.4055,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.25,
      "step": 19130
    },
    {
      "epoch": 0.712437885019821,
      "grad_norm": 10.634976382418419,
      "learning_rate": 1.1571543444272674e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.640625,
      "logps/chosen": -560.0,
      "logps/rejected": -696.0,
      "loss": 0.379,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.0625,
      "step": 19140
    },
    {
      "epoch": 0.7128101096201448,
      "grad_norm": 12.074870077469717,
      "learning_rate": 1.1544154721447058e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.546875,
      "logps/chosen": -556.0,
      "logps/rejected": -720.0,
      "loss": 0.3741,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -5.3125,
      "step": 19150
    },
    {
      "epoch": 0.7131823342204686,
      "grad_norm": 11.800674169984996,
      "learning_rate": 1.1516788716612413e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.515625,
      "logps/chosen": -576.0,
      "logps/rejected": -708.0,
      "loss": 0.3713,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.53125,
      "rewards/rejected": -5.15625,
      "step": 19160
    },
    {
      "epoch": 0.7135545588207924,
      "grad_norm": 12.283422071004857,
      "learning_rate": 1.1489445475971751e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.390625,
      "logps/chosen": -568.0,
      "logps/rejected": -708.0,
      "loss": 0.4004,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -4.875,
      "step": 19170
    },
    {
      "epoch": 0.7139267834211163,
      "grad_norm": 10.981301044985653,
      "learning_rate": 1.1462125045689661e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.609375,
      "logps/chosen": -532.0,
      "logps/rejected": -700.0,
      "loss": 0.3827,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.65625,
      "rewards/rejected": -4.875,
      "step": 19180
    },
    {
      "epoch": 0.7142990080214401,
      "grad_norm": 13.063170183579048,
      "learning_rate": 1.1434827471892222e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.609375,
      "logps/chosen": -556.0,
      "logps/rejected": -712.0,
      "loss": 0.3719,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.03125,
      "step": 19190
    },
    {
      "epoch": 0.714671232621764,
      "grad_norm": 14.42055155421332,
      "learning_rate": 1.1407552800666895e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5,
      "logps/chosen": -552.0,
      "logps/rejected": -716.0,
      "loss": 0.3928,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -5.34375,
      "step": 19200
    },
    {
      "epoch": 0.7150434572220878,
      "grad_norm": 12.827781213766551,
      "learning_rate": 1.138030107806251e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -596.0,
      "logps/rejected": -788.0,
      "loss": 0.3966,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.796875,
      "rewards/margins": 1.921875,
      "rewards/rejected": -5.71875,
      "step": 19210
    },
    {
      "epoch": 0.7154156818224117,
      "grad_norm": 13.379356766819962,
      "learning_rate": 1.1353072350089135e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.6875,
      "logps/chosen": -580.0,
      "logps/rejected": -728.0,
      "loss": 0.41,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.1875,
      "step": 19220
    },
    {
      "epoch": 0.7157879064227355,
      "grad_norm": 12.689745442852363,
      "learning_rate": 1.1325866662718025e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.453125,
      "logps/chosen": -564.0,
      "logps/rejected": -716.0,
      "loss": 0.3978,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.25,
      "step": 19230
    },
    {
      "epoch": 0.7161601310230593,
      "grad_norm": 11.065270871734814,
      "learning_rate": 1.129868406188151e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.484375,
      "logps/chosen": -556.0,
      "logps/rejected": -700.0,
      "loss": 0.3924,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -5.0,
      "step": 19240
    },
    {
      "epoch": 0.7165323556233831,
      "grad_norm": 11.685444943817098,
      "learning_rate": 1.1271524593472973e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.625,
      "logps/chosen": -616.0,
      "logps/rejected": -744.0,
      "loss": 0.383,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.34375,
      "step": 19250
    },
    {
      "epoch": 0.716904580223707,
      "grad_norm": 13.330798455452904,
      "learning_rate": 1.1244388303346722e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.53125,
      "logps/chosen": -552.0,
      "logps/rejected": -716.0,
      "loss": 0.4052,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -5.09375,
      "step": 19260
    },
    {
      "epoch": 0.7172768048240308,
      "grad_norm": 10.652494156675647,
      "learning_rate": 1.1217275237317942e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -540.0,
      "logps/rejected": -692.0,
      "loss": 0.3991,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -5.0,
      "step": 19270
    },
    {
      "epoch": 0.7176490294243546,
      "grad_norm": 12.2884559296029,
      "learning_rate": 1.1190185441162613e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.703125,
      "logps/chosen": -540.0,
      "logps/rejected": -668.0,
      "loss": 0.4044,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.78125,
      "step": 19280
    },
    {
      "epoch": 0.7180212540246785,
      "grad_norm": 12.924000492918646,
      "learning_rate": 1.1163118960617402e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.484375,
      "logps/chosen": -564.0,
      "logps/rejected": -692.0,
      "loss": 0.4034,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.390625,
      "rewards/rejected": -5.0625,
      "step": 19290
    },
    {
      "epoch": 0.7183934786250024,
      "grad_norm": 13.832804773247117,
      "learning_rate": 1.1136075841379636e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.6875,
      "logps/chosen": -548.0,
      "logps/rejected": -672.0,
      "loss": 0.4013,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.875,
      "step": 19300
    },
    {
      "epoch": 0.7187657032253262,
      "grad_norm": 12.110759737425742,
      "learning_rate": 1.1109056129107192e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.6875,
      "logps/chosen": -564.0,
      "logps/rejected": -696.0,
      "loss": 0.4031,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.9375,
      "step": 19310
    },
    {
      "epoch": 0.71913792782565,
      "grad_norm": 10.918518997500788,
      "learning_rate": 1.1082059869418428e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.6875,
      "logps/chosen": -510.0,
      "logps/rejected": -672.0,
      "loss": 0.3679,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.703125,
      "rewards/rejected": -4.90625,
      "step": 19320
    },
    {
      "epoch": 0.7195101524259738,
      "grad_norm": 12.580762634589023,
      "learning_rate": 1.1055087107892123e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.75,
      "logps/chosen": -588.0,
      "logps/rejected": -712.0,
      "loss": 0.3965,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -5.0625,
      "step": 19330
    },
    {
      "epoch": 0.7198823770262976,
      "grad_norm": 11.488155895575538,
      "learning_rate": 1.102813789006734e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.59375,
      "logps/chosen": -560.0,
      "logps/rejected": -688.0,
      "loss": 0.3766,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.03125,
      "step": 19340
    },
    {
      "epoch": 0.7202546016266215,
      "grad_norm": 12.748570168043909,
      "learning_rate": 1.1001212261443435e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.609375,
      "logps/chosen": -532.0,
      "logps/rejected": -692.0,
      "loss": 0.3578,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.09375,
      "step": 19350
    },
    {
      "epoch": 0.7206268262269453,
      "grad_norm": 12.822366253307624,
      "learning_rate": 1.0974310267479919e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.78125,
      "logps/chosen": -580.0,
      "logps/rejected": -704.0,
      "loss": 0.3866,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.1875,
      "step": 19360
    },
    {
      "epoch": 0.7209990508272692,
      "grad_norm": 12.826136454455673,
      "learning_rate": 1.0947431953596412e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -572.0,
      "logps/rejected": -728.0,
      "loss": 0.3898,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -5.25,
      "step": 19370
    },
    {
      "epoch": 0.721371275427593,
      "grad_norm": 13.378423355642822,
      "learning_rate": 1.0920577365172528e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.703125,
      "logps/chosen": -560.0,
      "logps/rejected": -700.0,
      "loss": 0.3945,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.15625,
      "step": 19380
    },
    {
      "epoch": 0.7217435000279169,
      "grad_norm": 15.029299602779826,
      "learning_rate": 1.0893746547547861e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.453125,
      "logps/chosen": -508.0,
      "logps/rejected": -672.0,
      "loss": 0.3896,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.1875,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -4.875,
      "step": 19390
    },
    {
      "epoch": 0.7221157246282407,
      "grad_norm": 10.85274418695595,
      "learning_rate": 1.0866939546021822e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -540.0,
      "logps/rejected": -696.0,
      "loss": 0.4066,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.03125,
      "step": 19400
    },
    {
      "epoch": 0.7224879492285645,
      "grad_norm": 11.14491165585267,
      "learning_rate": 1.084015640585367e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5,
      "logps/chosen": -528.0,
      "logps/rejected": -696.0,
      "loss": 0.3747,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.796875,
      "rewards/rejected": -5.09375,
      "step": 19410
    },
    {
      "epoch": 0.7228601738288883,
      "grad_norm": 12.54980372265579,
      "learning_rate": 1.0813397172262354e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.765625,
      "logps/chosen": -544.0,
      "logps/rejected": -708.0,
      "loss": 0.389,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.1875,
      "step": 19420
    },
    {
      "epoch": 0.7232323984292122,
      "grad_norm": 11.359205147794558,
      "learning_rate": 1.0786661890426436e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.640625,
      "logps/chosen": -540.0,
      "logps/rejected": -700.0,
      "loss": 0.3814,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.09375,
      "step": 19430
    },
    {
      "epoch": 0.723604623029536,
      "grad_norm": 12.586228897577458,
      "learning_rate": 1.0759950605484078e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.5625,
      "logps/chosen": -580.0,
      "logps/rejected": -720.0,
      "loss": 0.3791,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.21875,
      "step": 19440
    },
    {
      "epoch": 0.7239768476298598,
      "grad_norm": 11.897586543881495,
      "learning_rate": 1.0733263362532885e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.625,
      "logps/chosen": -540.0,
      "logps/rejected": -704.0,
      "loss": 0.4041,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.8125,
      "rewards/rejected": -5.1875,
      "step": 19450
    },
    {
      "epoch": 0.7243490722301837,
      "grad_norm": 12.720659066220904,
      "learning_rate": 1.0706600206629931e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.515625,
      "logps/chosen": -524.0,
      "logps/rejected": -700.0,
      "loss": 0.3991,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.125,
      "step": 19460
    },
    {
      "epoch": 0.7247212968305076,
      "grad_norm": 12.052635970703696,
      "learning_rate": 1.0679961182791561e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.75,
      "logps/chosen": -576.0,
      "logps/rejected": -716.0,
      "loss": 0.3881,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.15625,
      "step": 19470
    },
    {
      "epoch": 0.7250935214308314,
      "grad_norm": 11.937084447815044,
      "learning_rate": 1.0653346335993424e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.625,
      "logps/chosen": -568.0,
      "logps/rejected": -720.0,
      "loss": 0.3962,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.34375,
      "step": 19480
    },
    {
      "epoch": 0.7254657460311552,
      "grad_norm": 12.90511844598658,
      "learning_rate": 1.0626755711170302e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.640625,
      "logps/chosen": -556.0,
      "logps/rejected": -684.0,
      "loss": 0.3903,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.90625,
      "step": 19490
    },
    {
      "epoch": 0.725837970631479,
      "grad_norm": 16.454380234105553,
      "learning_rate": 1.0600189353216132e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5625,
      "logps/chosen": -520.0,
      "logps/rejected": -680.0,
      "loss": 0.3772,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.59375,
      "rewards/rejected": -4.9375,
      "step": 19500
    },
    {
      "epoch": 0.7262101952318029,
      "grad_norm": 14.34948006278167,
      "learning_rate": 1.0573647306983866e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.6875,
      "logps/chosen": -564.0,
      "logps/rejected": -696.0,
      "loss": 0.3991,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -5.125,
      "step": 19510
    },
    {
      "epoch": 0.7265824198321267,
      "grad_norm": 10.340162380856672,
      "learning_rate": 1.0547129617285377e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5625,
      "logps/chosen": -576.0,
      "logps/rejected": -704.0,
      "loss": 0.3933,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.390625,
      "rewards/rejected": -5.125,
      "step": 19520
    },
    {
      "epoch": 0.7269546444324505,
      "grad_norm": 12.25693525604877,
      "learning_rate": 1.0520636328891467e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.65625,
      "logps/chosen": -584.0,
      "logps/rejected": -736.0,
      "loss": 0.3655,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.34375,
      "step": 19530
    },
    {
      "epoch": 0.7273268690327743,
      "grad_norm": 12.97327982261667,
      "learning_rate": 1.0494167486531677e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.4375,
      "logps/chosen": -556.0,
      "logps/rejected": -724.0,
      "loss": 0.3682,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.953125,
      "rewards/rejected": -5.34375,
      "step": 19540
    },
    {
      "epoch": 0.7276990936330983,
      "grad_norm": 12.92604564886456,
      "learning_rate": 1.0467723134894357e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.796875,
      "logps/chosen": -568.0,
      "logps/rejected": -720.0,
      "loss": 0.3852,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.25,
      "step": 19550
    },
    {
      "epoch": 0.7280713182334221,
      "grad_norm": 12.121962034815096,
      "learning_rate": 1.0441303318626434e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.796875,
      "logps/chosen": -564.0,
      "logps/rejected": -720.0,
      "loss": 0.3876,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.28125,
      "step": 19560
    },
    {
      "epoch": 0.7284435428337459,
      "grad_norm": 11.938636882360628,
      "learning_rate": 1.0414908082333451e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.609375,
      "logps/chosen": -532.0,
      "logps/rejected": -720.0,
      "loss": 0.3882,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.953125,
      "rewards/rejected": -5.21875,
      "step": 19570
    },
    {
      "epoch": 0.7288157674340697,
      "grad_norm": 14.015451483855795,
      "learning_rate": 1.0388537470579448e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -540.0,
      "logps/rejected": -696.0,
      "loss": 0.4006,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.125,
      "step": 19580
    },
    {
      "epoch": 0.7291879920343936,
      "grad_norm": 11.909700122418966,
      "learning_rate": 1.0362191527886857e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -564.0,
      "logps/rejected": -736.0,
      "loss": 0.3844,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.90625,
      "rewards/rejected": -5.40625,
      "step": 19590
    },
    {
      "epoch": 0.7295602166347174,
      "grad_norm": 11.884233491293402,
      "learning_rate": 1.0335870298736526e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.46875,
      "logps/chosen": -560.0,
      "logps/rejected": -664.0,
      "loss": 0.3906,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.3125,
      "rewards/rejected": -4.71875,
      "step": 19600
    },
    {
      "epoch": 0.7299324412350412,
      "grad_norm": 13.396776797464067,
      "learning_rate": 1.030957382756751e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.625,
      "logps/chosen": -580.0,
      "logps/rejected": -728.0,
      "loss": 0.4154,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.3125,
      "step": 19610
    },
    {
      "epoch": 0.730304665835365,
      "grad_norm": 12.309917574813117,
      "learning_rate": 1.0283302158777113e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.4375,
      "logps/chosen": -536.0,
      "logps/rejected": -672.0,
      "loss": 0.3898,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.875,
      "step": 19620
    },
    {
      "epoch": 0.7306768904356888,
      "grad_norm": 10.771815615533795,
      "learning_rate": 1.0257055336720726e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.46875,
      "logps/chosen": -548.0,
      "logps/rejected": -708.0,
      "loss": 0.3763,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -4.96875,
      "step": 19630
    },
    {
      "epoch": 0.7310491150360128,
      "grad_norm": 11.290676151720662,
      "learning_rate": 1.0230833405711822e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.703125,
      "logps/chosen": -520.0,
      "logps/rejected": -680.0,
      "loss": 0.3662,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.75,
      "rewards/rejected": -5.0625,
      "step": 19640
    },
    {
      "epoch": 0.7314213396363366,
      "grad_norm": 10.925957720531969,
      "learning_rate": 1.020463641002183e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.609375,
      "logps/chosen": -520.0,
      "logps/rejected": -652.0,
      "loss": 0.3787,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.8125,
      "step": 19650
    },
    {
      "epoch": 0.7317935642366604,
      "grad_norm": 11.069870513894104,
      "learning_rate": 1.0178464393880093e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -548.0,
      "logps/rejected": -716.0,
      "loss": 0.3835,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -5.15625,
      "step": 19660
    },
    {
      "epoch": 0.7321657888369842,
      "grad_norm": 12.125245510379026,
      "learning_rate": 1.0152317401473782e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.6875,
      "logps/chosen": -564.0,
      "logps/rejected": -704.0,
      "loss": 0.3799,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.09375,
      "step": 19670
    },
    {
      "epoch": 0.7325380134373081,
      "grad_norm": 11.920832856794695,
      "learning_rate": 1.0126195476947779e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.703125,
      "logps/chosen": -584.0,
      "logps/rejected": -716.0,
      "loss": 0.402,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.15625,
      "step": 19680
    },
    {
      "epoch": 0.7329102380376319,
      "grad_norm": 12.651371693182597,
      "learning_rate": 1.0100098664404719e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.609375,
      "logps/chosen": -536.0,
      "logps/rejected": -712.0,
      "loss": 0.3756,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.75,
      "rewards/rejected": -5.125,
      "step": 19690
    },
    {
      "epoch": 0.7332824626379557,
      "grad_norm": 11.563881350726028,
      "learning_rate": 1.0074027007904764e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.71875,
      "logps/chosen": -532.0,
      "logps/rejected": -696.0,
      "loss": 0.3857,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.78125,
      "rewards/rejected": -5.125,
      "step": 19700
    },
    {
      "epoch": 0.7336546872382795,
      "grad_norm": 10.272399217594083,
      "learning_rate": 1.0047980551465653e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -564.0,
      "logps/rejected": -664.0,
      "loss": 0.3996,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.78125,
      "step": 19710
    },
    {
      "epoch": 0.7340269118386035,
      "grad_norm": 11.54933099542085,
      "learning_rate": 1.002195933906255e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.421875,
      "logps/chosen": -576.0,
      "logps/rejected": -736.0,
      "loss": 0.3586,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.28125,
      "step": 19720
    },
    {
      "epoch": 0.7343991364389273,
      "grad_norm": 10.574366419702008,
      "learning_rate": 9.995963414628011e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.515625,
      "logps/chosen": -560.0,
      "logps/rejected": -708.0,
      "loss": 0.3866,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.453125,
      "rewards/rejected": -5.1875,
      "step": 19730
    },
    {
      "epoch": 0.7347713610392511,
      "grad_norm": 10.760705023783482,
      "learning_rate": 9.969992822051904e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.53125,
      "logps/chosen": -560.0,
      "logps/rejected": -724.0,
      "loss": 0.3843,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.8125,
      "rewards/rejected": -5.3125,
      "step": 19740
    },
    {
      "epoch": 0.7351435856395749,
      "grad_norm": 13.975460077501726,
      "learning_rate": 9.944047605181319e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -580.0,
      "logps/rejected": -720.0,
      "loss": 0.3838,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.15625,
      "step": 19750
    },
    {
      "epoch": 0.7355158102398988,
      "grad_norm": 10.4664541335143,
      "learning_rate": 9.918127807820512e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.625,
      "logps/chosen": -572.0,
      "logps/rejected": -728.0,
      "loss": 0.3731,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.8125,
      "rewards/rejected": -5.3125,
      "step": 19760
    },
    {
      "epoch": 0.7358880348402226,
      "grad_norm": 13.312888070947375,
      "learning_rate": 9.892233473730799e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.484375,
      "logps/chosen": -536.0,
      "logps/rejected": -668.0,
      "loss": 0.4019,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -5.0,
      "step": 19770
    },
    {
      "epoch": 0.7362602594405464,
      "grad_norm": 11.111311923549774,
      "learning_rate": 9.866364646630531e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.484375,
      "logps/chosen": -544.0,
      "logps/rejected": -684.0,
      "loss": 0.3817,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.875,
      "step": 19780
    },
    {
      "epoch": 0.7366324840408702,
      "grad_norm": 13.212759408323071,
      "learning_rate": 9.84052137019499e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.640625,
      "logps/chosen": -552.0,
      "logps/rejected": -688.0,
      "loss": 0.382,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -4.96875,
      "step": 19790
    },
    {
      "epoch": 0.737004708641194,
      "grad_norm": 13.862474784057436,
      "learning_rate": 9.814703688056319e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.609375,
      "logps/chosen": -564.0,
      "logps/rejected": -680.0,
      "loss": 0.3907,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.5,
      "rewards/rejected": -5.0,
      "step": 19800
    },
    {
      "epoch": 0.737376933241518,
      "grad_norm": 11.445267837762206,
      "learning_rate": 9.78891164380343e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -528.0,
      "logps/rejected": -700.0,
      "loss": 0.3819,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.78125,
      "rewards/rejected": -5.0625,
      "step": 19810
    },
    {
      "epoch": 0.7377491578418418,
      "grad_norm": 12.15291878941812,
      "learning_rate": 9.763145280981974e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5625,
      "logps/chosen": -532.0,
      "logps/rejected": -684.0,
      "loss": 0.3953,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.03125,
      "step": 19820
    },
    {
      "epoch": 0.7381213824421656,
      "grad_norm": 10.858139814298733,
      "learning_rate": 9.737404643094235e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.75,
      "logps/chosen": -508.0,
      "logps/rejected": -668.0,
      "loss": 0.3627,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.15625,
      "rewards/margins": 1.671875,
      "rewards/rejected": -4.84375,
      "step": 19830
    },
    {
      "epoch": 0.7384936070424895,
      "grad_norm": 13.103768065976293,
      "learning_rate": 9.711689773599067e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.640625,
      "logps/chosen": -552.0,
      "logps/rejected": -704.0,
      "loss": 0.3765,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -5.1875,
      "step": 19840
    },
    {
      "epoch": 0.7388658316428133,
      "grad_norm": 12.079423678332084,
      "learning_rate": 9.686000715911819e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -528.0,
      "logps/rejected": -688.0,
      "loss": 0.3791,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.734375,
      "rewards/rejected": -4.9375,
      "step": 19850
    },
    {
      "epoch": 0.7392380562431371,
      "grad_norm": 11.016710537584965,
      "learning_rate": 9.660337513404243e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.640625,
      "logps/chosen": -564.0,
      "logps/rejected": -716.0,
      "loss": 0.3928,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.03125,
      "step": 19860
    },
    {
      "epoch": 0.7396102808434609,
      "grad_norm": 11.471553365479194,
      "learning_rate": 9.634700209404464e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.734375,
      "logps/chosen": -568.0,
      "logps/rejected": -692.0,
      "loss": 0.3964,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.25,
      "step": 19870
    },
    {
      "epoch": 0.7399825054437847,
      "grad_norm": 12.91028323610654,
      "learning_rate": 9.609088847196866e-08,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.765625,
      "logps/chosen": -572.0,
      "logps/rejected": -728.0,
      "loss": 0.3795,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.21875,
      "step": 19880
    },
    {
      "epoch": 0.7403547300441086,
      "grad_norm": 12.104951362148515,
      "learning_rate": 9.583503470022053e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -564.0,
      "logps/rejected": -708.0,
      "loss": 0.3895,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.8046875,
      "rewards/rejected": -5.3125,
      "step": 19890
    },
    {
      "epoch": 0.7407269546444325,
      "grad_norm": 10.503784845817904,
      "learning_rate": 9.557944121076724e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5,
      "logps/chosen": -540.0,
      "logps/rejected": -696.0,
      "loss": 0.3615,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.15625,
      "step": 19900
    },
    {
      "epoch": 0.7410991792447563,
      "grad_norm": 10.880375467909179,
      "learning_rate": 9.532410843513666e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.640625,
      "logps/chosen": -572.0,
      "logps/rejected": -736.0,
      "loss": 0.3735,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.21875,
      "step": 19910
    },
    {
      "epoch": 0.7414714038450801,
      "grad_norm": 14.984508269930217,
      "learning_rate": 9.50690368044163e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.625,
      "logps/chosen": -540.0,
      "logps/rejected": -696.0,
      "loss": 0.3946,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.125,
      "step": 19920
    },
    {
      "epoch": 0.741843628445404,
      "grad_norm": 10.374754652751042,
      "learning_rate": 9.481422674925288e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.6875,
      "logps/chosen": -572.0,
      "logps/rejected": -740.0,
      "loss": 0.3832,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.15625,
      "step": 19930
    },
    {
      "epoch": 0.7422158530457278,
      "grad_norm": 11.84550682682538,
      "learning_rate": 9.455967869985151e-08,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.65625,
      "logps/chosen": -552.0,
      "logps/rejected": -716.0,
      "loss": 0.3724,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.84375,
      "rewards/rejected": -5.25,
      "step": 19940
    },
    {
      "epoch": 0.7425880776460516,
      "grad_norm": 12.86322774838239,
      "learning_rate": 9.430539308597474e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -532.0,
      "logps/rejected": -696.0,
      "loss": 0.3888,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.96875,
      "step": 19950
    },
    {
      "epoch": 0.7429603022463754,
      "grad_norm": 13.056970661989865,
      "learning_rate": 9.405137033694222e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.46875,
      "logps/chosen": -580.0,
      "logps/rejected": -736.0,
      "loss": 0.3791,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.3125,
      "step": 19960
    },
    {
      "epoch": 0.7433325268466993,
      "grad_norm": 13.19389597626943,
      "learning_rate": 9.37976108816298e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -556.0,
      "logps/rejected": -684.0,
      "loss": 0.3909,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.9375,
      "step": 19970
    },
    {
      "epoch": 0.7437047514470231,
      "grad_norm": 11.783886060706829,
      "learning_rate": 9.354411514846882e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.609375,
      "logps/chosen": -540.0,
      "logps/rejected": -704.0,
      "loss": 0.3909,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -5.3125,
      "step": 19980
    },
    {
      "epoch": 0.744076976047347,
      "grad_norm": 11.072759376493678,
      "learning_rate": 9.329088356544518e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.546875,
      "logps/chosen": -572.0,
      "logps/rejected": -696.0,
      "loss": 0.3759,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.125,
      "step": 19990
    },
    {
      "epoch": 0.7444492006476708,
      "grad_norm": 10.364635865998578,
      "learning_rate": 9.303791656009896e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.65625,
      "logps/chosen": -580.0,
      "logps/rejected": -744.0,
      "loss": 0.3992,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.3125,
      "step": 20000
    },
    {
      "epoch": 0.7444492006476708,
      "eval_logits/chosen": -2.625,
      "eval_logits/rejected": -2.546875,
      "eval_logps/chosen": -604.0,
      "eval_logps/rejected": -704.0,
      "eval_loss": 0.48871660232543945,
      "eval_rewards/accuracies": 0.7410415410995483,
      "eval_rewards/chosen": -3.765625,
      "eval_rewards/margins": 1.3046875,
      "eval_rewards/rejected": -5.0625,
      "eval_runtime": 5138.9915,
      "eval_samples_per_second": 37.178,
      "eval_steps_per_second": 0.581,
      "step": 20000
    },
    {
      "epoch": 0.7448214252479947,
      "grad_norm": 13.65951026177011,
      "learning_rate": 9.278521455952354e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.53125,
      "logps/chosen": -536.0,
      "logps/rejected": -700.0,
      "loss": 0.3896,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.75,
      "rewards/rejected": -5.0,
      "step": 20010
    },
    {
      "epoch": 0.7451936498483185,
      "grad_norm": 11.444019460427052,
      "learning_rate": 9.253277799036488e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.46875,
      "logps/chosen": -552.0,
      "logps/rejected": -680.0,
      "loss": 0.3967,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.5625,
      "rewards/rejected": -5.03125,
      "step": 20020
    },
    {
      "epoch": 0.7455658744486423,
      "grad_norm": 11.78201908587726,
      "learning_rate": 9.228060727882083e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.484375,
      "logps/chosen": -560.0,
      "logps/rejected": -728.0,
      "loss": 0.3846,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.75,
      "rewards/rejected": -5.21875,
      "step": 20030
    },
    {
      "epoch": 0.7459380990489661,
      "grad_norm": 11.458805477489083,
      "learning_rate": 9.202870285064021e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.421875,
      "logps/chosen": -568.0,
      "logps/rejected": -700.0,
      "loss": 0.381,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.84375,
      "step": 20040
    },
    {
      "epoch": 0.74631032364929,
      "grad_norm": 11.287462498743539,
      "learning_rate": 9.177706513112246e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -544.0,
      "logps/rejected": -684.0,
      "loss": 0.3819,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.8125,
      "step": 20050
    },
    {
      "epoch": 0.7466825482496138,
      "grad_norm": 13.86418391667636,
      "learning_rate": 9.152569454511663e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -576.0,
      "logps/rejected": -728.0,
      "loss": 0.3846,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.28125,
      "step": 20060
    },
    {
      "epoch": 0.7470547728499376,
      "grad_norm": 11.304964497424969,
      "learning_rate": 9.12745915170209e-08,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.53125,
      "logps/chosen": -544.0,
      "logps/rejected": -664.0,
      "loss": 0.392,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.78125,
      "step": 20070
    },
    {
      "epoch": 0.7474269974502615,
      "grad_norm": 10.175501239711895,
      "learning_rate": 9.102375647078145e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.65625,
      "logps/chosen": -548.0,
      "logps/rejected": -692.0,
      "loss": 0.3911,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -5.0,
      "step": 20080
    },
    {
      "epoch": 0.7477992220505854,
      "grad_norm": 14.93203446443659,
      "learning_rate": 9.077318982989222e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.703125,
      "logps/chosen": -516.0,
      "logps/rejected": -676.0,
      "loss": 0.3581,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.140625,
      "rewards/margins": 1.8359375,
      "rewards/rejected": -4.96875,
      "step": 20090
    },
    {
      "epoch": 0.7481714466509092,
      "grad_norm": 11.183339054872745,
      "learning_rate": 9.052289201739397e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.390625,
      "logps/chosen": -544.0,
      "logps/rejected": -692.0,
      "loss": 0.392,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.96875,
      "step": 20100
    },
    {
      "epoch": 0.748543671251233,
      "grad_norm": 14.412936271663268,
      "learning_rate": 9.027286345587354e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -576.0,
      "logps/rejected": -704.0,
      "loss": 0.3804,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.453125,
      "rewards/rejected": -5.125,
      "step": 20110
    },
    {
      "epoch": 0.7489158958515568,
      "grad_norm": 12.678890953499621,
      "learning_rate": 9.002310456746335e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.65625,
      "logps/chosen": -552.0,
      "logps/rejected": -672.0,
      "loss": 0.3904,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.8125,
      "step": 20120
    },
    {
      "epoch": 0.7492881204518806,
      "grad_norm": 11.457836566293674,
      "learning_rate": 8.977361577384013e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.546875,
      "logps/chosen": -544.0,
      "logps/rejected": -684.0,
      "loss": 0.3819,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.8125,
      "step": 20130
    },
    {
      "epoch": 0.7496603450522045,
      "grad_norm": 10.992799892259088,
      "learning_rate": 8.952439749622497e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5625,
      "logps/chosen": -556.0,
      "logps/rejected": -692.0,
      "loss": 0.3766,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.0,
      "step": 20140
    },
    {
      "epoch": 0.7500325696525283,
      "grad_norm": 12.506684559796602,
      "learning_rate": 8.927545015538212e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.671875,
      "logps/chosen": -560.0,
      "logps/rejected": -684.0,
      "loss": 0.3916,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -5.0,
      "step": 20150
    },
    {
      "epoch": 0.7504047942528522,
      "grad_norm": 12.60143179690306,
      "learning_rate": 8.902677417161839e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.53125,
      "logps/chosen": -544.0,
      "logps/rejected": -728.0,
      "loss": 0.3781,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.9140625,
      "rewards/rejected": -5.21875,
      "step": 20160
    },
    {
      "epoch": 0.750777018853176,
      "grad_norm": 12.831618909363998,
      "learning_rate": 8.877836996478252e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -564.0,
      "logps/rejected": -712.0,
      "loss": 0.3714,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.25,
      "step": 20170
    },
    {
      "epoch": 0.7511492434534999,
      "grad_norm": 11.755694058881284,
      "learning_rate": 8.853023795426421e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.40625,
      "logps/chosen": -556.0,
      "logps/rejected": -720.0,
      "loss": 0.3753,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.25,
      "step": 20180
    },
    {
      "epoch": 0.7515214680538237,
      "grad_norm": 13.485197097412467,
      "learning_rate": 8.828237855899373e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5625,
      "logps/chosen": -584.0,
      "logps/rejected": -732.0,
      "loss": 0.3895,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -5.28125,
      "step": 20190
    },
    {
      "epoch": 0.7518936926541475,
      "grad_norm": 13.796346984185115,
      "learning_rate": 8.803479219744111e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5625,
      "logps/chosen": -564.0,
      "logps/rejected": -684.0,
      "loss": 0.374,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.5625,
      "rewards/rejected": -5.03125,
      "step": 20200
    },
    {
      "epoch": 0.7522659172544713,
      "grad_norm": 12.925767193265921,
      "learning_rate": 8.778747928761548e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.546875,
      "logps/chosen": -552.0,
      "logps/rejected": -712.0,
      "loss": 0.3771,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.84375,
      "rewards/rejected": -5.25,
      "step": 20210
    },
    {
      "epoch": 0.7526381418547952,
      "grad_norm": 11.054609138360044,
      "learning_rate": 8.7540440247064e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.375,
      "logps/chosen": -544.0,
      "logps/rejected": -724.0,
      "loss": 0.3868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.859375,
      "rewards/rejected": -5.28125,
      "step": 20220
    },
    {
      "epoch": 0.753010366455119,
      "grad_norm": 11.258742425868741,
      "learning_rate": 8.729367549287168e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -556.0,
      "logps/rejected": -716.0,
      "loss": 0.3811,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.125,
      "step": 20230
    },
    {
      "epoch": 0.7533825910554428,
      "grad_norm": 12.020572205524427,
      "learning_rate": 8.704718544166046e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.34375,
      "logps/chosen": -560.0,
      "logps/rejected": -700.0,
      "loss": 0.379,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.15625,
      "step": 20240
    },
    {
      "epoch": 0.7537548156557667,
      "grad_norm": 10.454541817971979,
      "learning_rate": 8.680097050958834e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.546875,
      "logps/chosen": -568.0,
      "logps/rejected": -708.0,
      "loss": 0.4048,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.1875,
      "step": 20250
    },
    {
      "epoch": 0.7541270402560906,
      "grad_norm": 11.167070176702783,
      "learning_rate": 8.655503111234905e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -572.0,
      "logps/rejected": -732.0,
      "loss": 0.3923,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.25,
      "step": 20260
    },
    {
      "epoch": 0.7544992648564144,
      "grad_norm": 11.79691388056195,
      "learning_rate": 8.630936766517081e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.515625,
      "logps/chosen": -532.0,
      "logps/rejected": -660.0,
      "loss": 0.3814,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.875,
      "step": 20270
    },
    {
      "epoch": 0.7548714894567382,
      "grad_norm": 14.82735090428095,
      "learning_rate": 8.606398058281619e-08,
      "logits/chosen": -2.5,
      "logits/rejected": -2.40625,
      "logps/chosen": -560.0,
      "logps/rejected": -708.0,
      "loss": 0.3852,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.25,
      "step": 20280
    },
    {
      "epoch": 0.755243714057062,
      "grad_norm": 12.827238234959896,
      "learning_rate": 8.581887027958107e-08,
      "logits/chosen": -2.5,
      "logits/rejected": -2.453125,
      "logps/chosen": -588.0,
      "logps/rejected": -736.0,
      "loss": 0.3773,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.3125,
      "step": 20290
    },
    {
      "epoch": 0.7556159386573859,
      "grad_norm": 11.100075823911054,
      "learning_rate": 8.557403716929418e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.34375,
      "logps/chosen": -604.0,
      "logps/rejected": -752.0,
      "loss": 0.4,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.4375,
      "step": 20300
    },
    {
      "epoch": 0.7559881632577097,
      "grad_norm": 12.331974978638824,
      "learning_rate": 8.532948166531592e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.46875,
      "logps/chosen": -544.0,
      "logps/rejected": -700.0,
      "loss": 0.4055,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.0625,
      "step": 20310
    },
    {
      "epoch": 0.7563603878580335,
      "grad_norm": 12.498933062918681,
      "learning_rate": 8.508520418053839e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5,
      "logps/chosen": -528.0,
      "logps/rejected": -680.0,
      "loss": 0.4071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.90625,
      "step": 20320
    },
    {
      "epoch": 0.7567326124583573,
      "grad_norm": 12.252793979978218,
      "learning_rate": 8.484120512738386e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.59375,
      "logps/chosen": -504.0,
      "logps/rejected": -632.0,
      "loss": 0.3837,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.328125,
      "rewards/rejected": -4.5625,
      "step": 20330
    },
    {
      "epoch": 0.7571048370586813,
      "grad_norm": 11.58827623168046,
      "learning_rate": 8.459748491780502e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.578125,
      "logps/chosen": -552.0,
      "logps/rejected": -716.0,
      "loss": 0.368,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.3125,
      "step": 20340
    },
    {
      "epoch": 0.7574770616590051,
      "grad_norm": 13.379923820573957,
      "learning_rate": 8.435404396328355e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.5,
      "logps/chosen": -544.0,
      "logps/rejected": -676.0,
      "loss": 0.3916,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.90625,
      "step": 20350
    },
    {
      "epoch": 0.7578492862593289,
      "grad_norm": 13.357223316691416,
      "learning_rate": 8.411088267482944e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.53125,
      "logps/chosen": -524.0,
      "logps/rejected": -668.0,
      "loss": 0.3893,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -4.9375,
      "step": 20360
    },
    {
      "epoch": 0.7582215108596527,
      "grad_norm": 11.340252263322768,
      "learning_rate": 8.386800146298087e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.28125,
      "logps/chosen": -568.0,
      "logps/rejected": -716.0,
      "loss": 0.3749,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.15625,
      "step": 20370
    },
    {
      "epoch": 0.7585937354599765,
      "grad_norm": 11.207269411740267,
      "learning_rate": 8.362540073780275e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -552.0,
      "logps/rejected": -672.0,
      "loss": 0.401,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.9375,
      "step": 20380
    },
    {
      "epoch": 0.7589659600603004,
      "grad_norm": 11.842690513474803,
      "learning_rate": 8.338308090888696e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.53125,
      "logps/chosen": -564.0,
      "logps/rejected": -712.0,
      "loss": 0.3919,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.8828125,
      "rewards/rejected": -5.4375,
      "step": 20390
    },
    {
      "epoch": 0.7593381846606242,
      "grad_norm": 12.896354974611713,
      "learning_rate": 8.314104238535066e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.453125,
      "logps/chosen": -560.0,
      "logps/rejected": -704.0,
      "loss": 0.4166,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.96875,
      "step": 20400
    },
    {
      "epoch": 0.759710409260948,
      "grad_norm": 12.279680033981947,
      "learning_rate": 8.289928557583639e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.390625,
      "logps/chosen": -540.0,
      "logps/rejected": -712.0,
      "loss": 0.3722,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.8359375,
      "rewards/rejected": -5.125,
      "step": 20410
    },
    {
      "epoch": 0.7600826338612718,
      "grad_norm": 12.036136243217905,
      "learning_rate": 8.265781088851068e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.40625,
      "logps/chosen": -532.0,
      "logps/rejected": -680.0,
      "loss": 0.3777,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.8359375,
      "rewards/rejected": -5.15625,
      "step": 20420
    },
    {
      "epoch": 0.7604548584615958,
      "grad_norm": 12.790138284854152,
      "learning_rate": 8.241661873106426e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.296875,
      "logps/chosen": -584.0,
      "logps/rejected": -728.0,
      "loss": 0.3974,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -5.15625,
      "step": 20430
    },
    {
      "epoch": 0.7608270830619196,
      "grad_norm": 12.415432598063825,
      "learning_rate": 8.217570951071055e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5,
      "logps/chosen": -592.0,
      "logps/rejected": -720.0,
      "loss": 0.4039,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -5.09375,
      "step": 20440
    },
    {
      "epoch": 0.7611993076622434,
      "grad_norm": 11.698865898706341,
      "learning_rate": 8.193508363418514e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.5,
      "logps/chosen": -556.0,
      "logps/rejected": -688.0,
      "loss": 0.3725,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.609375,
      "rewards/rejected": -4.9375,
      "step": 20450
    },
    {
      "epoch": 0.7615715322625672,
      "grad_norm": 12.153057390337688,
      "learning_rate": 8.169474150774563e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -560.0,
      "logps/rejected": -736.0,
      "loss": 0.4224,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.28125,
      "step": 20460
    },
    {
      "epoch": 0.7619437568628911,
      "grad_norm": 11.806798518182667,
      "learning_rate": 8.145468353717006e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.484375,
      "logps/chosen": -560.0,
      "logps/rejected": -704.0,
      "loss": 0.3832,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.03125,
      "step": 20470
    },
    {
      "epoch": 0.7623159814632149,
      "grad_norm": 12.347917431319305,
      "learning_rate": 8.121491012775733e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.640625,
      "logps/chosen": -540.0,
      "logps/rejected": -692.0,
      "loss": 0.3815,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.53125,
      "rewards/rejected": -4.875,
      "step": 20480
    },
    {
      "epoch": 0.7626882060635387,
      "grad_norm": 11.500492930288416,
      "learning_rate": 8.09754216843253e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.375,
      "logps/chosen": -552.0,
      "logps/rejected": -716.0,
      "loss": 0.3773,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -5.125,
      "step": 20490
    },
    {
      "epoch": 0.7630604306638625,
      "grad_norm": 13.064056374020685,
      "learning_rate": 8.073621861121113e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.59375,
      "logps/chosen": -540.0,
      "logps/rejected": -684.0,
      "loss": 0.3711,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -4.9375,
      "step": 20500
    },
    {
      "epoch": 0.7634326552641864,
      "grad_norm": 11.094446327747093,
      "learning_rate": 8.049730131227005e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.59375,
      "logps/chosen": -560.0,
      "logps/rejected": -716.0,
      "loss": 0.3828,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -5.25,
      "step": 20510
    },
    {
      "epoch": 0.7638048798645103,
      "grad_norm": 11.804861096815815,
      "learning_rate": 8.02586701908746e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.484375,
      "logps/chosen": -552.0,
      "logps/rejected": -676.0,
      "loss": 0.4087,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -4.90625,
      "step": 20520
    },
    {
      "epoch": 0.7641771044648341,
      "grad_norm": 14.20143104044356,
      "learning_rate": 8.002032564991459e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.5625,
      "logps/chosen": -592.0,
      "logps/rejected": -764.0,
      "loss": 0.3993,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.890625,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.625,
      "step": 20530
    },
    {
      "epoch": 0.7645493290651579,
      "grad_norm": 12.89372599369068,
      "learning_rate": 7.978226809179558e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.53125,
      "logps/chosen": -548.0,
      "logps/rejected": -696.0,
      "loss": 0.3756,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.25,
      "step": 20540
    },
    {
      "epoch": 0.7649215536654818,
      "grad_norm": 12.904275989174671,
      "learning_rate": 7.95444979184389e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.5625,
      "logps/chosen": -592.0,
      "logps/rejected": -696.0,
      "loss": 0.4014,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.8125,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -5.15625,
      "step": 20550
    },
    {
      "epoch": 0.7652937782658056,
      "grad_norm": 13.336052837001878,
      "learning_rate": 7.930701553128036e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.453125,
      "logps/chosen": -520.0,
      "logps/rejected": -680.0,
      "loss": 0.4051,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.9375,
      "step": 20560
    },
    {
      "epoch": 0.7656660028661294,
      "grad_norm": 11.221941536466643,
      "learning_rate": 7.906982133127018e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -540.0,
      "logps/rejected": -680.0,
      "loss": 0.3705,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.90625,
      "step": 20570
    },
    {
      "epoch": 0.7660382274664532,
      "grad_norm": 13.213627325129528,
      "learning_rate": 7.883291571887196e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.515625,
      "logps/chosen": -510.0,
      "logps/rejected": -660.0,
      "loss": 0.3953,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.8125,
      "step": 20580
    },
    {
      "epoch": 0.766410452066777,
      "grad_norm": 15.113880184775573,
      "learning_rate": 7.859629909406196e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.4375,
      "logps/chosen": -572.0,
      "logps/rejected": -768.0,
      "loss": 0.3902,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.375,
      "step": 20590
    },
    {
      "epoch": 0.766782676667101,
      "grad_norm": 13.331388364213007,
      "learning_rate": 7.83599718563287e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.515625,
      "logps/chosen": -564.0,
      "logps/rejected": -696.0,
      "loss": 0.3968,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.1875,
      "step": 20600
    },
    {
      "epoch": 0.7671549012674248,
      "grad_norm": 15.36996158611169,
      "learning_rate": 7.812393440467177e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.640625,
      "logps/chosen": -548.0,
      "logps/rejected": -696.0,
      "loss": 0.3772,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.0625,
      "step": 20610
    },
    {
      "epoch": 0.7675271258677486,
      "grad_norm": 9.902012976309257,
      "learning_rate": 7.78881871376021e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.65625,
      "logps/chosen": -576.0,
      "logps/rejected": -688.0,
      "loss": 0.3738,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.9375,
      "step": 20620
    },
    {
      "epoch": 0.7678993504680725,
      "grad_norm": 9.740469254460912,
      "learning_rate": 7.765273045314005e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.609375,
      "logps/chosen": -584.0,
      "logps/rejected": -736.0,
      "loss": 0.3725,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.828125,
      "rewards/rejected": -5.375,
      "step": 20630
    },
    {
      "epoch": 0.7682715750683963,
      "grad_norm": 11.888326345944101,
      "learning_rate": 7.741756474881588e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.515625,
      "logps/chosen": -564.0,
      "logps/rejected": -704.0,
      "loss": 0.3912,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -5.0,
      "step": 20640
    },
    {
      "epoch": 0.7686437996687201,
      "grad_norm": 11.800811746565708,
      "learning_rate": 7.718269042166817e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.46875,
      "logps/chosen": -564.0,
      "logps/rejected": -688.0,
      "loss": 0.3743,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.90625,
      "step": 20650
    },
    {
      "epoch": 0.7690160242690439,
      "grad_norm": 9.987679746541057,
      "learning_rate": 7.694810786824388e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.671875,
      "logps/chosen": -576.0,
      "logps/rejected": -724.0,
      "loss": 0.3831,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.28125,
      "step": 20660
    },
    {
      "epoch": 0.7693882488693677,
      "grad_norm": 11.124181968463821,
      "learning_rate": 7.671381748459715e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -544.0,
      "logps/rejected": -704.0,
      "loss": 0.3876,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.125,
      "step": 20670
    },
    {
      "epoch": 0.7697604734696916,
      "grad_norm": 10.300236749734024,
      "learning_rate": 7.647981966628899e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.484375,
      "logps/chosen": -520.0,
      "logps/rejected": -676.0,
      "loss": 0.3687,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -4.9375,
      "step": 20680
    },
    {
      "epoch": 0.7701326980700155,
      "grad_norm": 11.531342569443986,
      "learning_rate": 7.624611480838647e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.59375,
      "logps/chosen": -576.0,
      "logps/rejected": -740.0,
      "loss": 0.359,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.1875,
      "step": 20690
    },
    {
      "epoch": 0.7705049226703393,
      "grad_norm": 13.562953048216412,
      "learning_rate": 7.601270330546177e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -540.0,
      "logps/rejected": -692.0,
      "loss": 0.3885,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.09375,
      "step": 20700
    },
    {
      "epoch": 0.7708771472706631,
      "grad_norm": 11.705607995915264,
      "learning_rate": 7.577958555159209e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.484375,
      "logps/chosen": -532.0,
      "logps/rejected": -688.0,
      "loss": 0.3794,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.0625,
      "step": 20710
    },
    {
      "epoch": 0.771249371870987,
      "grad_norm": 13.190067294784816,
      "learning_rate": 7.554676194035859e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.609375,
      "logps/chosen": -592.0,
      "logps/rejected": -736.0,
      "loss": 0.4039,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.75,
      "rewards/rejected": -5.4375,
      "step": 20720
    },
    {
      "epoch": 0.7716215964713108,
      "grad_norm": 12.652872072180656,
      "learning_rate": 7.531423286484584e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.65625,
      "logps/chosen": -564.0,
      "logps/rejected": -712.0,
      "loss": 0.393,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -5.28125,
      "step": 20730
    },
    {
      "epoch": 0.7719938210716346,
      "grad_norm": 11.604687077704975,
      "learning_rate": 7.508199871764098e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.59375,
      "logps/chosen": -560.0,
      "logps/rejected": -728.0,
      "loss": 0.3748,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.25,
      "step": 20740
    },
    {
      "epoch": 0.7723660456719584,
      "grad_norm": 13.966849930182086,
      "learning_rate": 7.48500598908334e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.546875,
      "logps/chosen": -544.0,
      "logps/rejected": -688.0,
      "loss": 0.3886,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.0625,
      "step": 20750
    },
    {
      "epoch": 0.7727382702722823,
      "grad_norm": 10.49317925652736,
      "learning_rate": 7.46184167760138e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5,
      "logps/chosen": -556.0,
      "logps/rejected": -676.0,
      "loss": 0.3982,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.125,
      "rewards/rejected": -4.6875,
      "step": 20760
    },
    {
      "epoch": 0.7731104948726061,
      "grad_norm": 10.446435486847806,
      "learning_rate": 7.43870697642737e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.625,
      "logps/chosen": -524.0,
      "logps/rejected": -700.0,
      "loss": 0.3832,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.859375,
      "rewards/rejected": -5.21875,
      "step": 20770
    },
    {
      "epoch": 0.77348271947293,
      "grad_norm": 12.420267550492694,
      "learning_rate": 7.415601924620465e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.484375,
      "logps/chosen": -580.0,
      "logps/rejected": -684.0,
      "loss": 0.397,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.546875,
      "rewards/rejected": -5.125,
      "step": 20780
    },
    {
      "epoch": 0.7738549440732538,
      "grad_norm": 9.640605605152462,
      "learning_rate": 7.392526561189752e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.4375,
      "logps/chosen": -576.0,
      "logps/rejected": -704.0,
      "loss": 0.3554,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.515625,
      "rewards/rejected": -5.0625,
      "step": 20790
    },
    {
      "epoch": 0.7742271686735777,
      "grad_norm": 13.636013330875443,
      "learning_rate": 7.369480925094204e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.546875,
      "logps/chosen": -572.0,
      "logps/rejected": -676.0,
      "loss": 0.4146,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.453125,
      "rewards/rejected": -5.0625,
      "step": 20800
    },
    {
      "epoch": 0.7745993932739015,
      "grad_norm": 11.242946628139153,
      "learning_rate": 7.346465055242606e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.6875,
      "logps/chosen": -564.0,
      "logps/rejected": -696.0,
      "loss": 0.3724,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.90625,
      "step": 20810
    },
    {
      "epoch": 0.7749716178742253,
      "grad_norm": 13.914473865544933,
      "learning_rate": 7.323478990493495e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.625,
      "logps/chosen": -588.0,
      "logps/rejected": -720.0,
      "loss": 0.3993,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -5.03125,
      "step": 20820
    },
    {
      "epoch": 0.7753438424745491,
      "grad_norm": 12.028671256799791,
      "learning_rate": 7.30052276965506e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.65625,
      "logps/chosen": -556.0,
      "logps/rejected": -708.0,
      "loss": 0.3802,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.15625,
      "step": 20830
    },
    {
      "epoch": 0.775716067074873,
      "grad_norm": 11.539778427252248,
      "learning_rate": 7.277596431485126e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.3125,
      "logps/chosen": -544.0,
      "logps/rejected": -704.0,
      "loss": 0.3932,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.25,
      "step": 20840
    },
    {
      "epoch": 0.7760882916751968,
      "grad_norm": 10.480661986810599,
      "learning_rate": 7.254700014691059e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -584.0,
      "logps/rejected": -704.0,
      "loss": 0.3891,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.5,
      "rewards/rejected": -5.15625,
      "step": 20850
    },
    {
      "epoch": 0.7764605162755206,
      "grad_norm": 13.958709728398025,
      "learning_rate": 7.231833557929712e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -612.0,
      "logps/rejected": -748.0,
      "loss": 0.3785,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.78125,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.40625,
      "step": 20860
    },
    {
      "epoch": 0.7768327408758445,
      "grad_norm": 13.095314193135842,
      "learning_rate": 7.208997099807356e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.515625,
      "logps/chosen": -580.0,
      "logps/rejected": -716.0,
      "loss": 0.3908,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -5.1875,
      "step": 20870
    },
    {
      "epoch": 0.7772049654761684,
      "grad_norm": 11.94802107412685,
      "learning_rate": 7.186190678879603e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.515625,
      "logps/chosen": -552.0,
      "logps/rejected": -716.0,
      "loss": 0.3934,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.96875,
      "step": 20880
    },
    {
      "epoch": 0.7775771900764922,
      "grad_norm": 13.915582210640569,
      "learning_rate": 7.163414333651357e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.578125,
      "logps/chosen": -552.0,
      "logps/rejected": -712.0,
      "loss": 0.3858,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -5.1875,
      "step": 20890
    },
    {
      "epoch": 0.777949414676816,
      "grad_norm": 12.292254146677136,
      "learning_rate": 7.140668102576756e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -568.0,
      "logps/rejected": -704.0,
      "loss": 0.3944,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -5.0,
      "step": 20900
    },
    {
      "epoch": 0.7783216392771398,
      "grad_norm": 11.855075107780769,
      "learning_rate": 7.117952024059084e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -580.0,
      "logps/rejected": -720.0,
      "loss": 0.3877,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -5.21875,
      "step": 20910
    },
    {
      "epoch": 0.7786938638774636,
      "grad_norm": 14.885317562224627,
      "learning_rate": 7.095266136450714e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.625,
      "logps/chosen": -544.0,
      "logps/rejected": -692.0,
      "loss": 0.3952,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -5.0625,
      "step": 20920
    },
    {
      "epoch": 0.7790660884777875,
      "grad_norm": 9.731856326950306,
      "learning_rate": 7.072610478053051e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.265625,
      "logps/chosen": -536.0,
      "logps/rejected": -708.0,
      "loss": 0.3704,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.875,
      "rewards/rejected": -5.25,
      "step": 20930
    },
    {
      "epoch": 0.7794383130781113,
      "grad_norm": 12.126417007305971,
      "learning_rate": 7.049985087116469e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.59375,
      "logps/chosen": -588.0,
      "logps/rejected": -708.0,
      "loss": 0.3857,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -5.03125,
      "step": 20940
    },
    {
      "epoch": 0.7798105376784352,
      "grad_norm": 10.672441850040984,
      "learning_rate": 7.02739000184023e-08,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.59375,
      "logps/chosen": -592.0,
      "logps/rejected": -720.0,
      "loss": 0.3798,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.3125,
      "step": 20950
    },
    {
      "epoch": 0.780182762278759,
      "grad_norm": 9.795785249919993,
      "learning_rate": 7.004825260372449e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.5625,
      "logps/chosen": -540.0,
      "logps/rejected": -712.0,
      "loss": 0.3859,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.0625,
      "step": 20960
    },
    {
      "epoch": 0.7805549868790829,
      "grad_norm": 11.664747146241504,
      "learning_rate": 6.982290900809976e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.640625,
      "logps/chosen": -580.0,
      "logps/rejected": -684.0,
      "loss": 0.3931,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -5.0,
      "step": 20970
    },
    {
      "epoch": 0.7809272114794067,
      "grad_norm": 19.350291626081294,
      "learning_rate": 6.959786961198396e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -572.0,
      "logps/rejected": -692.0,
      "loss": 0.4225,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.5625,
      "rewards/rejected": -5.0625,
      "step": 20980
    },
    {
      "epoch": 0.7812994360797305,
      "grad_norm": 10.874837658602198,
      "learning_rate": 6.937313479531925e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.296875,
      "logps/chosen": -564.0,
      "logps/rejected": -728.0,
      "loss": 0.3746,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.28125,
      "step": 20990
    },
    {
      "epoch": 0.7816716606800543,
      "grad_norm": 10.676876393964413,
      "learning_rate": 6.914870493753363e-08,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.5,
      "logps/chosen": -512.0,
      "logps/rejected": -672.0,
      "loss": 0.361,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -4.8125,
      "step": 21000
    },
    {
      "epoch": 0.7820438852803782,
      "grad_norm": 12.347550554155347,
      "learning_rate": 6.892458041754001e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -576.0,
      "logps/rejected": -744.0,
      "loss": 0.3844,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.25,
      "step": 21010
    },
    {
      "epoch": 0.782416109880702,
      "grad_norm": 12.472672224924795,
      "learning_rate": 6.870076161373603e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -572.0,
      "logps/rejected": -692.0,
      "loss": 0.3929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.28125,
      "rewards/rejected": -4.9375,
      "step": 21020
    },
    {
      "epoch": 0.7827883344810258,
      "grad_norm": 10.829092074826002,
      "learning_rate": 6.847724890400306e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.453125,
      "logps/chosen": -552.0,
      "logps/rejected": -720.0,
      "loss": 0.3994,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.28125,
      "step": 21030
    },
    {
      "epoch": 0.7831605590813497,
      "grad_norm": 11.834043779154687,
      "learning_rate": 6.825404266570572e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.34375,
      "logps/chosen": -528.0,
      "logps/rejected": -672.0,
      "loss": 0.3933,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.90625,
      "step": 21040
    },
    {
      "epoch": 0.7835327836816736,
      "grad_norm": 12.644101819007359,
      "learning_rate": 6.803114327569126e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.59375,
      "logps/chosen": -544.0,
      "logps/rejected": -680.0,
      "loss": 0.3913,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.875,
      "step": 21050
    },
    {
      "epoch": 0.7839050082819974,
      "grad_norm": 11.908675623634357,
      "learning_rate": 6.780855111028863e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -556.0,
      "logps/rejected": -704.0,
      "loss": 0.402,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.75,
      "rewards/rejected": -5.34375,
      "step": 21060
    },
    {
      "epoch": 0.7842772328823212,
      "grad_norm": 10.521957776190105,
      "learning_rate": 6.75862665453083e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.625,
      "logps/chosen": -576.0,
      "logps/rejected": -712.0,
      "loss": 0.4004,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -5.25,
      "step": 21070
    },
    {
      "epoch": 0.784649457482645,
      "grad_norm": 10.745758870104442,
      "learning_rate": 6.736428995604143e-08,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.53125,
      "logps/chosen": -544.0,
      "logps/rejected": -712.0,
      "loss": 0.3738,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.796875,
      "rewards/rejected": -5.15625,
      "step": 21080
    },
    {
      "epoch": 0.7850216820829689,
      "grad_norm": 16.118264554949555,
      "learning_rate": 6.714262171725904e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5625,
      "logps/chosen": -572.0,
      "logps/rejected": -688.0,
      "loss": 0.4071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.96875,
      "step": 21090
    },
    {
      "epoch": 0.7853939066832927,
      "grad_norm": 13.809275997930945,
      "learning_rate": 6.692126220321181e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.625,
      "logps/chosen": -548.0,
      "logps/rejected": -672.0,
      "loss": 0.3874,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.578125,
      "rewards/rejected": -4.90625,
      "step": 21100
    },
    {
      "epoch": 0.7857661312836165,
      "grad_norm": 11.453238567601954,
      "learning_rate": 6.67002117876288e-08,
      "logits/chosen": -2.5,
      "logits/rejected": -2.4375,
      "logps/chosen": -544.0,
      "logps/rejected": -704.0,
      "loss": 0.4036,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.8125,
      "rewards/rejected": -5.09375,
      "step": 21110
    },
    {
      "epoch": 0.7861383558839403,
      "grad_norm": 10.728787380449743,
      "learning_rate": 6.647947084371755e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.5625,
      "logps/chosen": -568.0,
      "logps/rejected": -672.0,
      "loss": 0.3787,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.40625,
      "rewards/rejected": -4.78125,
      "step": 21120
    },
    {
      "epoch": 0.7865105804842643,
      "grad_norm": 12.065606049982536,
      "learning_rate": 6.6259039744163e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -556.0,
      "logps/rejected": -696.0,
      "loss": 0.3751,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.53125,
      "rewards/rejected": -5.0,
      "step": 21130
    },
    {
      "epoch": 0.7868828050845881,
      "grad_norm": 11.86994927267534,
      "learning_rate": 6.6038918861127e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.65625,
      "logps/chosen": -568.0,
      "logps/rejected": -688.0,
      "loss": 0.3969,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.90625,
      "step": 21140
    },
    {
      "epoch": 0.7872550296849119,
      "grad_norm": 11.641207590615274,
      "learning_rate": 6.581910856624748e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.40625,
      "logps/chosen": -516.0,
      "logps/rejected": -684.0,
      "loss": 0.3707,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.8125,
      "rewards/rejected": -5.03125,
      "step": 21150
    },
    {
      "epoch": 0.7876272542852357,
      "grad_norm": 10.809791203774358,
      "learning_rate": 6.559960923063829e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -568.0,
      "logps/rejected": -724.0,
      "loss": 0.3689,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.125,
      "step": 21160
    },
    {
      "epoch": 0.7879994788855595,
      "grad_norm": 11.613258066753447,
      "learning_rate": 6.538042122488786e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.578125,
      "logps/chosen": -564.0,
      "logps/rejected": -720.0,
      "loss": 0.3947,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.34375,
      "step": 21170
    },
    {
      "epoch": 0.7883717034858834,
      "grad_norm": 13.855958758367603,
      "learning_rate": 6.516154491905943e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.53125,
      "logps/chosen": -568.0,
      "logps/rejected": -700.0,
      "loss": 0.3873,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.1875,
      "step": 21180
    },
    {
      "epoch": 0.7887439280862072,
      "grad_norm": 13.924224788425986,
      "learning_rate": 6.494298068268986e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.4375,
      "logps/chosen": -564.0,
      "logps/rejected": -708.0,
      "loss": 0.3891,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.125,
      "step": 21190
    },
    {
      "epoch": 0.789116152686531,
      "grad_norm": 11.439668185432664,
      "learning_rate": 6.472472888478888e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -568.0,
      "logps/rejected": -708.0,
      "loss": 0.3737,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.6875,
      "rewards/rejected": -5.09375,
      "step": 21200
    },
    {
      "epoch": 0.7894883772868548,
      "grad_norm": 12.0927499673929,
      "learning_rate": 6.450678989383901e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.640625,
      "logps/chosen": -568.0,
      "logps/rejected": -700.0,
      "loss": 0.3917,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.25,
      "step": 21210
    },
    {
      "epoch": 0.7898606018871788,
      "grad_norm": 12.960810079771896,
      "learning_rate": 6.428916407779452e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5625,
      "logps/chosen": -592.0,
      "logps/rejected": -736.0,
      "loss": 0.3947,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.78125,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.375,
      "step": 21220
    },
    {
      "epoch": 0.7902328264875026,
      "grad_norm": 14.716992425265936,
      "learning_rate": 6.407185180408104e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.578125,
      "logps/chosen": -592.0,
      "logps/rejected": -756.0,
      "loss": 0.3813,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.796875,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.53125,
      "step": 21230
    },
    {
      "epoch": 0.7906050510878264,
      "grad_norm": 11.19252955667562,
      "learning_rate": 6.385485343959457e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.65625,
      "logps/chosen": -608.0,
      "logps/rejected": -728.0,
      "loss": 0.3831,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.765625,
      "rewards/margins": 1.546875,
      "rewards/rejected": -5.3125,
      "step": 21240
    },
    {
      "epoch": 0.7909772756881502,
      "grad_norm": 14.217626101327312,
      "learning_rate": 6.363816935070152e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.515625,
      "logps/chosen": -592.0,
      "logps/rejected": -736.0,
      "loss": 0.3903,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.3125,
      "step": 21250
    },
    {
      "epoch": 0.7913495002884741,
      "grad_norm": 14.24261865822829,
      "learning_rate": 6.34217999032372e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -544.0,
      "logps/rejected": -708.0,
      "loss": 0.378,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.03125,
      "step": 21260
    },
    {
      "epoch": 0.7917217248887979,
      "grad_norm": 11.448650787940975,
      "learning_rate": 6.320574546250627e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.578125,
      "logps/chosen": -584.0,
      "logps/rejected": -696.0,
      "loss": 0.4085,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.375,
      "rewards/rejected": -5.0,
      "step": 21270
    },
    {
      "epoch": 0.7920939494891217,
      "grad_norm": 12.023925295411559,
      "learning_rate": 6.29900063932812e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.46875,
      "logps/chosen": -540.0,
      "logps/rejected": -680.0,
      "loss": 0.3911,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.03125,
      "step": 21280
    },
    {
      "epoch": 0.7924661740894455,
      "grad_norm": 12.914309620858873,
      "learning_rate": 6.277458305980198e-08,
      "logits/chosen": -2.5,
      "logits/rejected": -2.421875,
      "logps/chosen": -536.0,
      "logps/rejected": -680.0,
      "loss": 0.385,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.9375,
      "step": 21290
    },
    {
      "epoch": 0.7928383986897694,
      "grad_norm": 11.399269945009738,
      "learning_rate": 6.255947582577576e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -576.0,
      "logps/rejected": -708.0,
      "loss": 0.3681,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.09375,
      "step": 21300
    },
    {
      "epoch": 0.7932106232900933,
      "grad_norm": 12.318690228182676,
      "learning_rate": 6.234468505437565e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -572.0,
      "logps/rejected": -732.0,
      "loss": 0.383,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.0625,
      "step": 21310
    },
    {
      "epoch": 0.7935828478904171,
      "grad_norm": 13.141876597724547,
      "learning_rate": 6.213021110824099e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.4375,
      "logps/chosen": -560.0,
      "logps/rejected": -728.0,
      "loss": 0.3738,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -5.3125,
      "step": 21320
    },
    {
      "epoch": 0.7939550724907409,
      "grad_norm": 13.742380011396532,
      "learning_rate": 6.191605434947566e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5625,
      "logps/chosen": -560.0,
      "logps/rejected": -708.0,
      "loss": 0.3604,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.6875,
      "rewards/rejected": -5.28125,
      "step": 21330
    },
    {
      "epoch": 0.7943272970910648,
      "grad_norm": 13.902538576150146,
      "learning_rate": 6.170221513964841e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5625,
      "logps/chosen": -532.0,
      "logps/rejected": -668.0,
      "loss": 0.3843,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.75,
      "step": 21340
    },
    {
      "epoch": 0.7946995216913886,
      "grad_norm": 15.56108677873087,
      "learning_rate": 6.148869383979172e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.640625,
      "logps/chosen": -552.0,
      "logps/rejected": -692.0,
      "loss": 0.3953,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.0625,
      "step": 21350
    },
    {
      "epoch": 0.7950717462917124,
      "grad_norm": 12.432940288460905,
      "learning_rate": 6.127549081040117e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.546875,
      "logps/chosen": -612.0,
      "logps/rejected": -788.0,
      "loss": 0.3769,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.796875,
      "rewards/margins": 1.890625,
      "rewards/rejected": -5.6875,
      "step": 21360
    },
    {
      "epoch": 0.7954439708920362,
      "grad_norm": 11.63884956854925,
      "learning_rate": 6.106260641143546e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.609375,
      "logps/chosen": -608.0,
      "logps/rejected": -756.0,
      "loss": 0.3739,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.8125,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.375,
      "step": 21370
    },
    {
      "epoch": 0.79581619549236,
      "grad_norm": 12.431963769351732,
      "learning_rate": 6.085004100231475e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -576.0,
      "logps/rejected": -720.0,
      "loss": 0.3799,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.25,
      "step": 21380
    },
    {
      "epoch": 0.796188420092684,
      "grad_norm": 9.906279212396157,
      "learning_rate": 6.063779494192112e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.59375,
      "logps/chosen": -588.0,
      "logps/rejected": -736.0,
      "loss": 0.3849,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.78125,
      "rewards/margins": 1.546875,
      "rewards/rejected": -5.3125,
      "step": 21390
    },
    {
      "epoch": 0.7965606446930078,
      "grad_norm": 12.274175847777064,
      "learning_rate": 6.042586858859703e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.546875,
      "logps/chosen": -576.0,
      "logps/rejected": -724.0,
      "loss": 0.3961,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.28125,
      "step": 21400
    },
    {
      "epoch": 0.7969328692933316,
      "grad_norm": 14.12930918824295,
      "learning_rate": 6.021426230014572e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5,
      "logps/chosen": -556.0,
      "logps/rejected": -724.0,
      "loss": 0.3658,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.21875,
      "step": 21410
    },
    {
      "epoch": 0.7973050938936554,
      "grad_norm": 10.981529777538402,
      "learning_rate": 6.000297643382957e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.484375,
      "logps/chosen": -544.0,
      "logps/rejected": -712.0,
      "loss": 0.3657,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.796875,
      "rewards/rejected": -5.1875,
      "step": 21420
    },
    {
      "epoch": 0.7976773184939793,
      "grad_norm": 12.068582594731557,
      "learning_rate": 5.979201134637015e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5,
      "logps/chosen": -560.0,
      "logps/rejected": -720.0,
      "loss": 0.3974,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -5.09375,
      "step": 21430
    },
    {
      "epoch": 0.7980495430943031,
      "grad_norm": 12.118975144689378,
      "learning_rate": 5.9581367393947616e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5,
      "logps/chosen": -564.0,
      "logps/rejected": -736.0,
      "loss": 0.3705,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.9140625,
      "rewards/rejected": -5.34375,
      "step": 21440
    },
    {
      "epoch": 0.7984217676946269,
      "grad_norm": 14.159328577460954,
      "learning_rate": 5.937104493219952e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.6875,
      "logps/chosen": -580.0,
      "logps/rejected": -692.0,
      "loss": 0.3701,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.421875,
      "rewards/rejected": -5.0,
      "step": 21450
    },
    {
      "epoch": 0.7987939922949507,
      "grad_norm": 15.443122953518337,
      "learning_rate": 5.916104431622121e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.421875,
      "logps/chosen": -576.0,
      "logps/rejected": -728.0,
      "loss": 0.3932,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.34375,
      "step": 21460
    },
    {
      "epoch": 0.7991662168952746,
      "grad_norm": 12.03801752214029,
      "learning_rate": 5.895136590056407e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.53125,
      "logps/chosen": -576.0,
      "logps/rejected": -744.0,
      "loss": 0.3934,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.921875,
      "rewards/rejected": -5.4375,
      "step": 21470
    },
    {
      "epoch": 0.7995384414955985,
      "grad_norm": 10.789519464567736,
      "learning_rate": 5.8742010039235957e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.671875,
      "logps/chosen": -576.0,
      "logps/rejected": -728.0,
      "loss": 0.3676,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.28125,
      "step": 21480
    },
    {
      "epoch": 0.7999106660959223,
      "grad_norm": 11.917869052897672,
      "learning_rate": 5.853297708569979e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.578125,
      "logps/chosen": -576.0,
      "logps/rejected": -752.0,
      "loss": 0.3735,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.515625,
      "rewards/margins": 2.046875,
      "rewards/rejected": -5.5625,
      "step": 21490
    },
    {
      "epoch": 0.8002828906962461,
      "grad_norm": 12.001076088870468,
      "learning_rate": 5.832426739287355e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.578125,
      "logps/chosen": -592.0,
      "logps/rejected": -752.0,
      "loss": 0.383,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.828125,
      "rewards/rejected": -5.53125,
      "step": 21500
    },
    {
      "epoch": 0.80065511529657,
      "grad_norm": 11.808770894367486,
      "learning_rate": 5.811588131312936e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.390625,
      "logps/chosen": -568.0,
      "logps/rejected": -736.0,
      "loss": 0.3682,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -5.375,
      "step": 21510
    },
    {
      "epoch": 0.8010273398968938,
      "grad_norm": 11.741387034059217,
      "learning_rate": 5.790781919829299e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -580.0,
      "logps/rejected": -728.0,
      "loss": 0.3857,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -5.25,
      "step": 21520
    },
    {
      "epoch": 0.8013995644972176,
      "grad_norm": 12.278939056924045,
      "learning_rate": 5.770008139964333e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.328125,
      "logps/chosen": -568.0,
      "logps/rejected": -760.0,
      "loss": 0.3659,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.796875,
      "rewards/rejected": -5.5,
      "step": 21530
    },
    {
      "epoch": 0.8017717890975414,
      "grad_norm": 13.278689318652594,
      "learning_rate": 5.749266826791146e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.5,
      "logps/chosen": -564.0,
      "logps/rejected": -708.0,
      "loss": 0.3991,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.1875,
      "step": 21540
    },
    {
      "epoch": 0.8021440136978653,
      "grad_norm": 12.573639189108091,
      "learning_rate": 5.728558015328061e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.5625,
      "logps/chosen": -596.0,
      "logps/rejected": -736.0,
      "loss": 0.3963,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.4375,
      "step": 21550
    },
    {
      "epoch": 0.8025162382981891,
      "grad_norm": 12.448242473938565,
      "learning_rate": 5.707881740538509e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.546875,
      "logps/chosen": -560.0,
      "logps/rejected": -696.0,
      "loss": 0.3968,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.375,
      "rewards/rejected": -5.0625,
      "step": 21560
    },
    {
      "epoch": 0.802888462898513,
      "grad_norm": 11.908418786015003,
      "learning_rate": 5.687238037331002e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.484375,
      "logps/chosen": -544.0,
      "logps/rejected": -692.0,
      "loss": 0.3813,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.5,
      "rewards/rejected": -5.03125,
      "step": 21570
    },
    {
      "epoch": 0.8032606874988368,
      "grad_norm": 11.549390306281527,
      "learning_rate": 5.666626940559038e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.5,
      "logps/chosen": -548.0,
      "logps/rejected": -668.0,
      "loss": 0.3732,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.390625,
      "rewards/rejected": -5.0,
      "step": 21580
    },
    {
      "epoch": 0.8036329120991607,
      "grad_norm": 13.003892279793233,
      "learning_rate": 5.646048485021085e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.296875,
      "logps/chosen": -584.0,
      "logps/rejected": -740.0,
      "loss": 0.3933,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.34375,
      "step": 21590
    },
    {
      "epoch": 0.8040051366994845,
      "grad_norm": 11.58847283018414,
      "learning_rate": 5.625502705460497e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.546875,
      "logps/chosen": -556.0,
      "logps/rejected": -692.0,
      "loss": 0.4013,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -4.90625,
      "step": 21600
    },
    {
      "epoch": 0.8043773612998083,
      "grad_norm": 13.666622754961057,
      "learning_rate": 5.6049896365654577e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.5,
      "logps/chosen": -584.0,
      "logps/rejected": -708.0,
      "loss": 0.3966,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.1875,
      "step": 21610
    },
    {
      "epoch": 0.8047495859001321,
      "grad_norm": 12.5478944471836,
      "learning_rate": 5.584509312968927e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -532.0,
      "logps/rejected": -660.0,
      "loss": 0.3826,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.453125,
      "rewards/rejected": -4.78125,
      "step": 21620
    },
    {
      "epoch": 0.805121810500456,
      "grad_norm": 13.091850266847578,
      "learning_rate": 5.5640617692485654e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.4375,
      "logps/chosen": -556.0,
      "logps/rejected": -716.0,
      "loss": 0.3693,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.796875,
      "rewards/rejected": -5.25,
      "step": 21630
    },
    {
      "epoch": 0.8054940351007798,
      "grad_norm": 11.129728208066647,
      "learning_rate": 5.5436470399267103e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.453125,
      "logps/chosen": -564.0,
      "logps/rejected": -692.0,
      "loss": 0.3919,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.5,
      "rewards/rejected": -5.09375,
      "step": 21640
    },
    {
      "epoch": 0.8058662597011036,
      "grad_norm": 14.337325029943838,
      "learning_rate": 5.523265159470289e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.625,
      "logps/chosen": -556.0,
      "logps/rejected": -700.0,
      "loss": 0.3998,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.15625,
      "step": 21650
    },
    {
      "epoch": 0.8062384843014275,
      "grad_norm": 12.804831060879497,
      "learning_rate": 5.5029161622907716e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.546875,
      "logps/chosen": -576.0,
      "logps/rejected": -704.0,
      "loss": 0.3809,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.5,
      "rewards/rejected": -5.125,
      "step": 21660
    },
    {
      "epoch": 0.8066107089017513,
      "grad_norm": 12.486899035835336,
      "learning_rate": 5.482600082744096e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.484375,
      "logps/chosen": -520.0,
      "logps/rejected": -676.0,
      "loss": 0.3663,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 1.703125,
      "rewards/rejected": -4.96875,
      "step": 21670
    },
    {
      "epoch": 0.8069829335020752,
      "grad_norm": 10.510784009881172,
      "learning_rate": 5.4623169551306445e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.484375,
      "logps/chosen": -536.0,
      "logps/rejected": -708.0,
      "loss": 0.3644,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.125,
      "step": 21680
    },
    {
      "epoch": 0.807355158102399,
      "grad_norm": 12.722373950190255,
      "learning_rate": 5.442066813695151e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.328125,
      "logps/chosen": -576.0,
      "logps/rejected": -736.0,
      "loss": 0.389,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.78125,
      "rewards/rejected": -5.375,
      "step": 21690
    },
    {
      "epoch": 0.8077273827027228,
      "grad_norm": 14.174716743920195,
      "learning_rate": 5.421849692626665e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -572.0,
      "logps/rejected": -712.0,
      "loss": 0.3931,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.6875,
      "rewards/rejected": -5.3125,
      "step": 21700
    },
    {
      "epoch": 0.8080996073030466,
      "grad_norm": 10.584191309531956,
      "learning_rate": 5.401665626058491e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -560.0,
      "logps/rejected": -740.0,
      "loss": 0.3805,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.828125,
      "rewards/rejected": -5.375,
      "step": 21710
    },
    {
      "epoch": 0.8084718319033705,
      "grad_norm": 13.083052520948192,
      "learning_rate": 5.3815146480681056e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -564.0,
      "logps/rejected": -712.0,
      "loss": 0.3875,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -5.21875,
      "step": 21720
    },
    {
      "epoch": 0.8088440565036943,
      "grad_norm": 10.734596875565872,
      "learning_rate": 5.361396792677142e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -528.0,
      "logps/rejected": -672.0,
      "loss": 0.3626,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.875,
      "step": 21730
    },
    {
      "epoch": 0.8092162811040181,
      "grad_norm": 11.257729668366297,
      "learning_rate": 5.341312093851305e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.421875,
      "logps/chosen": -552.0,
      "logps/rejected": -716.0,
      "loss": 0.394,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.125,
      "step": 21740
    },
    {
      "epoch": 0.809588505704342,
      "grad_norm": 12.294151667870702,
      "learning_rate": 5.3212605855003254e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.453125,
      "logps/chosen": -536.0,
      "logps/rejected": -688.0,
      "loss": 0.384,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.09375,
      "step": 21750
    },
    {
      "epoch": 0.8099607303046659,
      "grad_norm": 13.20282345725394,
      "learning_rate": 5.301242301477879e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5625,
      "logps/chosen": -548.0,
      "logps/rejected": -680.0,
      "loss": 0.385,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.5625,
      "rewards/rejected": -4.96875,
      "step": 21760
    },
    {
      "epoch": 0.8103329549049897,
      "grad_norm": 11.253786396038596,
      "learning_rate": 5.281257275581563e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.4375,
      "logps/chosen": -576.0,
      "logps/rejected": -732.0,
      "loss": 0.3927,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -5.375,
      "step": 21770
    },
    {
      "epoch": 0.8107051795053135,
      "grad_norm": 12.128986244660735,
      "learning_rate": 5.261305541552824e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.609375,
      "logps/chosen": -584.0,
      "logps/rejected": -728.0,
      "loss": 0.3775,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.78125,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.40625,
      "step": 21780
    },
    {
      "epoch": 0.8110774041056373,
      "grad_norm": 13.045422106776277,
      "learning_rate": 5.241387133076894e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.453125,
      "logps/chosen": -572.0,
      "logps/rejected": -708.0,
      "loss": 0.389,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.40625,
      "rewards/rejected": -5.09375,
      "step": 21790
    },
    {
      "epoch": 0.8114496287059612,
      "grad_norm": 13.685399826416015,
      "learning_rate": 5.2215020837827527e-08,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.578125,
      "logps/chosen": -548.0,
      "logps/rejected": -736.0,
      "loss": 0.3734,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.953125,
      "rewards/rejected": -5.34375,
      "step": 21800
    },
    {
      "epoch": 0.811821853306285,
      "grad_norm": 11.259988830074265,
      "learning_rate": 5.201650427243034e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.625,
      "logps/chosen": -580.0,
      "logps/rejected": -704.0,
      "loss": 0.3975,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.53125,
      "rewards/rejected": -5.15625,
      "step": 21810
    },
    {
      "epoch": 0.8121940779066088,
      "grad_norm": 13.51802805794457,
      "learning_rate": 5.181832196974015e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.625,
      "logps/chosen": -568.0,
      "logps/rejected": -732.0,
      "loss": 0.3884,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.796875,
      "rewards/rejected": -5.3125,
      "step": 21820
    },
    {
      "epoch": 0.8125663025069327,
      "grad_norm": 11.064137613634106,
      "learning_rate": 5.162047426435537e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.53125,
      "logps/chosen": -572.0,
      "logps/rejected": -728.0,
      "loss": 0.3813,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.28125,
      "step": 21830
    },
    {
      "epoch": 0.8129385271072566,
      "grad_norm": 11.881478956144235,
      "learning_rate": 5.142296149030945e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.546875,
      "logps/chosen": -596.0,
      "logps/rejected": -752.0,
      "loss": 0.3888,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.734375,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.34375,
      "step": 21840
    },
    {
      "epoch": 0.8133107517075804,
      "grad_norm": 16.487385457371175,
      "learning_rate": 5.1225783981070245e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.640625,
      "logps/chosen": -564.0,
      "logps/rejected": -720.0,
      "loss": 0.3561,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.75,
      "rewards/rejected": -5.375,
      "step": 21850
    },
    {
      "epoch": 0.8136829763079042,
      "grad_norm": 12.00767887160296,
      "learning_rate": 5.102894206953975e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -596.0,
      "logps/rejected": -736.0,
      "loss": 0.3723,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.78125,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.375,
      "step": 21860
    },
    {
      "epoch": 0.814055200908228,
      "grad_norm": 13.49896503057509,
      "learning_rate": 5.083243608805332e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.609375,
      "logps/chosen": -548.0,
      "logps/rejected": -676.0,
      "loss": 0.3897,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.421875,
      "rewards/rejected": -4.96875,
      "step": 21870
    },
    {
      "epoch": 0.8144274255085518,
      "grad_norm": 14.017234814171555,
      "learning_rate": 5.0636266368379085e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.515625,
      "logps/chosen": -572.0,
      "logps/rejected": -720.0,
      "loss": 0.3903,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.25,
      "step": 21880
    },
    {
      "epoch": 0.8147996501088757,
      "grad_norm": 11.520607217521277,
      "learning_rate": 5.0440433241717596e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -564.0,
      "logps/rejected": -712.0,
      "loss": 0.3916,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.15625,
      "step": 21890
    },
    {
      "epoch": 0.8151718747091995,
      "grad_norm": 16.240371009954128,
      "learning_rate": 5.024493703870084e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -588.0,
      "logps/rejected": -724.0,
      "loss": 0.4024,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -5.25,
      "step": 21900
    },
    {
      "epoch": 0.8155440993095233,
      "grad_norm": 12.077317416778056,
      "learning_rate": 5.004977808939223e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -580.0,
      "logps/rejected": -728.0,
      "loss": 0.3933,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.6875,
      "rewards/rejected": -5.25,
      "step": 21910
    },
    {
      "epoch": 0.8159163239098473,
      "grad_norm": 11.031561414395298,
      "learning_rate": 4.9854956723285687e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.4375,
      "logps/chosen": -560.0,
      "logps/rejected": -736.0,
      "loss": 0.3729,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -5.40625,
      "step": 21920
    },
    {
      "epoch": 0.8162885485101711,
      "grad_norm": 13.083974491135145,
      "learning_rate": 4.966047326930517e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.578125,
      "logps/chosen": -560.0,
      "logps/rejected": -708.0,
      "loss": 0.3724,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.25,
      "step": 21930
    },
    {
      "epoch": 0.8166607731104949,
      "grad_norm": 12.516230752937252,
      "learning_rate": 4.94663280558042e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -580.0,
      "logps/rejected": -740.0,
      "loss": 0.3903,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.8125,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.5625,
      "step": 21940
    },
    {
      "epoch": 0.8170329977108187,
      "grad_norm": 10.527602988108878,
      "learning_rate": 4.9272521410565065e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -600.0,
      "logps/rejected": -736.0,
      "loss": 0.3663,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.828125,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -5.34375,
      "step": 21950
    },
    {
      "epoch": 0.8174052223111425,
      "grad_norm": 12.142828669056286,
      "learning_rate": 4.907905366079859e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -584.0,
      "logps/rejected": -728.0,
      "loss": 0.403,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -5.34375,
      "step": 21960
    },
    {
      "epoch": 0.8177774469114664,
      "grad_norm": 16.227075488925312,
      "learning_rate": 4.888592513314338e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -540.0,
      "logps/rejected": -680.0,
      "loss": 0.3884,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.03125,
      "step": 21970
    },
    {
      "epoch": 0.8181496715117902,
      "grad_norm": 10.61597472085945,
      "learning_rate": 4.869313615366541e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -576.0,
      "logps/rejected": -708.0,
      "loss": 0.4164,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -5.15625,
      "step": 21980
    },
    {
      "epoch": 0.818521896112114,
      "grad_norm": 12.605996603245043,
      "learning_rate": 4.8500687047857145e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.609375,
      "logps/chosen": -584.0,
      "logps/rejected": -712.0,
      "loss": 0.401,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -5.03125,
      "step": 21990
    },
    {
      "epoch": 0.8188941207124378,
      "grad_norm": 11.0102790422409,
      "learning_rate": 4.830857814063744e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -592.0,
      "logps/rejected": -680.0,
      "loss": 0.3994,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -4.9375,
      "step": 22000
    },
    {
      "epoch": 0.8192663453127618,
      "grad_norm": 13.462223643222096,
      "learning_rate": 4.8116809756350767e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.6875,
      "logps/chosen": -588.0,
      "logps/rejected": -712.0,
      "loss": 0.4081,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.4375,
      "rewards/rejected": -5.09375,
      "step": 22010
    },
    {
      "epoch": 0.8196385699130856,
      "grad_norm": 11.080641309188787,
      "learning_rate": 4.7925382218766604e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.5,
      "logps/chosen": -572.0,
      "logps/rejected": -704.0,
      "loss": 0.3964,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.546875,
      "rewards/rejected": -5.0,
      "step": 22020
    },
    {
      "epoch": 0.8200107945134094,
      "grad_norm": 11.230044363553404,
      "learning_rate": 4.7734295851079046e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.453125,
      "logps/chosen": -536.0,
      "logps/rejected": -672.0,
      "loss": 0.3805,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.96875,
      "step": 22030
    },
    {
      "epoch": 0.8203830191137332,
      "grad_norm": 12.981927414234645,
      "learning_rate": 4.7543550975906013e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.53125,
      "logps/chosen": -588.0,
      "logps/rejected": -756.0,
      "loss": 0.3771,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.875,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.625,
      "step": 22040
    },
    {
      "epoch": 0.8207552437140571,
      "grad_norm": 12.568079885836104,
      "learning_rate": 4.735314791528908e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.46875,
      "logps/chosen": -524.0,
      "logps/rejected": -684.0,
      "loss": 0.3894,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -5.03125,
      "step": 22050
    },
    {
      "epoch": 0.8211274683143809,
      "grad_norm": 12.494834318596924,
      "learning_rate": 4.716308699069257e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -548.0,
      "logps/rejected": -700.0,
      "loss": 0.3985,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.21875,
      "step": 22060
    },
    {
      "epoch": 0.8214996929147047,
      "grad_norm": 11.987563983800355,
      "learning_rate": 4.6973368523003376e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.3125,
      "logps/chosen": -544.0,
      "logps/rejected": -708.0,
      "loss": 0.3752,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.1875,
      "step": 22070
    },
    {
      "epoch": 0.8218719175150285,
      "grad_norm": 12.319815639006155,
      "learning_rate": 4.6783992832529845e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.375,
      "logps/chosen": -552.0,
      "logps/rejected": -736.0,
      "loss": 0.3875,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.84375,
      "rewards/rejected": -5.375,
      "step": 22080
    },
    {
      "epoch": 0.8222441421153523,
      "grad_norm": 11.960960921555209,
      "learning_rate": 4.6594960239001975e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.59375,
      "logps/chosen": -576.0,
      "logps/rejected": -732.0,
      "loss": 0.3959,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.375,
      "step": 22090
    },
    {
      "epoch": 0.8226163667156763,
      "grad_norm": 13.893179025188195,
      "learning_rate": 4.6406271061570166e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.625,
      "logps/chosen": -596.0,
      "logps/rejected": -720.0,
      "loss": 0.3924,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.28125,
      "step": 22100
    },
    {
      "epoch": 0.8229885913160001,
      "grad_norm": 12.160800408172022,
      "learning_rate": 4.6217925618805356e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.703125,
      "logps/chosen": -568.0,
      "logps/rejected": -688.0,
      "loss": 0.4059,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.90625,
      "step": 22110
    },
    {
      "epoch": 0.8233608159163239,
      "grad_norm": 12.408877026246355,
      "learning_rate": 4.6029924228698e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.5,
      "logps/chosen": -560.0,
      "logps/rejected": -716.0,
      "loss": 0.3853,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.75,
      "rewards/rejected": -5.3125,
      "step": 22120
    },
    {
      "epoch": 0.8237330405166478,
      "grad_norm": 12.297120343885785,
      "learning_rate": 4.5842267208657524e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.65625,
      "logps/chosen": -596.0,
      "logps/rejected": -708.0,
      "loss": 0.3848,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.78125,
      "rewards/margins": 1.46875,
      "rewards/rejected": -5.25,
      "step": 22130
    },
    {
      "epoch": 0.8241052651169716,
      "grad_norm": 14.606182352044131,
      "learning_rate": 4.565495487551213e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.625,
      "logps/chosen": -536.0,
      "logps/rejected": -664.0,
      "loss": 0.3955,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.90625,
      "step": 22140
    },
    {
      "epoch": 0.8244774897172954,
      "grad_norm": 10.995895572805091,
      "learning_rate": 4.546798754550801e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -560.0,
      "logps/rejected": -728.0,
      "loss": 0.3821,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.9921875,
      "rewards/rejected": -5.5,
      "step": 22150
    },
    {
      "epoch": 0.8248497143176192,
      "grad_norm": 10.491734714423355,
      "learning_rate": 4.5281365534308946e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.390625,
      "logps/chosen": -556.0,
      "logps/rejected": -728.0,
      "loss": 0.3659,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.796875,
      "rewards/rejected": -5.46875,
      "step": 22160
    },
    {
      "epoch": 0.825221938917943,
      "grad_norm": 11.694170592954503,
      "learning_rate": 4.5095089156995505e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -588.0,
      "logps/rejected": -724.0,
      "loss": 0.3973,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.828125,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.375,
      "step": 22170
    },
    {
      "epoch": 0.825594163518267,
      "grad_norm": 10.885421431202932,
      "learning_rate": 4.490915872806497e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -572.0,
      "logps/rejected": -728.0,
      "loss": 0.3837,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.3125,
      "step": 22180
    },
    {
      "epoch": 0.8259663881185908,
      "grad_norm": 11.651521002608154,
      "learning_rate": 4.4723574561430246e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.734375,
      "logps/chosen": -580.0,
      "logps/rejected": -724.0,
      "loss": 0.3749,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.28125,
      "step": 22190
    },
    {
      "epoch": 0.8263386127189146,
      "grad_norm": 11.787593346146735,
      "learning_rate": 4.4538336970420004e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.578125,
      "logps/chosen": -568.0,
      "logps/rejected": -748.0,
      "loss": 0.3673,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.953125,
      "rewards/rejected": -5.46875,
      "step": 22200
    },
    {
      "epoch": 0.8267108373192384,
      "grad_norm": 12.604506115926013,
      "learning_rate": 4.435344626777754e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -560.0,
      "logps/rejected": -688.0,
      "loss": 0.3932,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -5.0,
      "step": 22210
    },
    {
      "epoch": 0.8270830619195623,
      "grad_norm": 11.105506606545426,
      "learning_rate": 4.41689027656604e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.5625,
      "logps/chosen": -560.0,
      "logps/rejected": -692.0,
      "loss": 0.3988,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -5.0,
      "step": 22220
    },
    {
      "epoch": 0.8274552865198861,
      "grad_norm": 11.49213582216142,
      "learning_rate": 4.398470677564023e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.71875,
      "logps/chosen": -572.0,
      "logps/rejected": -720.0,
      "loss": 0.3587,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.9140625,
      "rewards/rejected": -5.40625,
      "step": 22230
    },
    {
      "epoch": 0.8278275111202099,
      "grad_norm": 12.532612926355771,
      "learning_rate": 4.380085860870156e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.578125,
      "logps/chosen": -532.0,
      "logps/rejected": -696.0,
      "loss": 0.3729,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 1.796875,
      "rewards/rejected": -5.0625,
      "step": 22240
    },
    {
      "epoch": 0.8281997357205337,
      "grad_norm": 12.005386148445144,
      "learning_rate": 4.361735857524221e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.4375,
      "logps/chosen": -568.0,
      "logps/rejected": -728.0,
      "loss": 0.3934,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.21875,
      "step": 22250
    },
    {
      "epoch": 0.8285719603208576,
      "grad_norm": 11.796047811277628,
      "learning_rate": 4.3434206985071667e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -576.0,
      "logps/rejected": -736.0,
      "loss": 0.3768,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.1875,
      "step": 22260
    },
    {
      "epoch": 0.8289441849211815,
      "grad_norm": 12.064358512053508,
      "learning_rate": 4.325140414741152e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.4375,
      "logps/chosen": -548.0,
      "logps/rejected": -700.0,
      "loss": 0.402,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.125,
      "step": 22270
    },
    {
      "epoch": 0.8293164095215053,
      "grad_norm": 14.286035791290436,
      "learning_rate": 4.3068950370894476e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.46875,
      "logps/chosen": -560.0,
      "logps/rejected": -720.0,
      "loss": 0.3885,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.15625,
      "step": 22280
    },
    {
      "epoch": 0.8296886341218291,
      "grad_norm": 12.566673771552686,
      "learning_rate": 4.288684596356365e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.28125,
      "logps/chosen": -524.0,
      "logps/rejected": -724.0,
      "loss": 0.3799,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.84375,
      "rewards/rejected": -5.1875,
      "step": 22290
    },
    {
      "epoch": 0.830060858722153,
      "grad_norm": 11.059156799644821,
      "learning_rate": 4.270509123287277e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -616.0,
      "logps/rejected": -760.0,
      "loss": 0.3754,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.796875,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -5.5,
      "step": 22300
    },
    {
      "epoch": 0.8304330833224768,
      "grad_norm": 13.17127817249306,
      "learning_rate": 4.2523686485684745e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.390625,
      "logps/chosen": -556.0,
      "logps/rejected": -716.0,
      "loss": 0.3978,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.34375,
      "step": 22310
    },
    {
      "epoch": 0.8308053079228006,
      "grad_norm": 12.015124529275115,
      "learning_rate": 4.23426320282719e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.609375,
      "logps/chosen": -600.0,
      "logps/rejected": -748.0,
      "loss": 0.4118,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.765625,
      "rewards/margins": 1.546875,
      "rewards/rejected": -5.3125,
      "step": 22320
    },
    {
      "epoch": 0.8311775325231244,
      "grad_norm": 13.830673733090418,
      "learning_rate": 4.2161928166314834e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.421875,
      "logps/chosen": -556.0,
      "logps/rejected": -716.0,
      "loss": 0.3957,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.25,
      "step": 22330
    },
    {
      "epoch": 0.8315497571234483,
      "grad_norm": 11.466039242211313,
      "learning_rate": 4.1981575204902695e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.640625,
      "logps/chosen": -556.0,
      "logps/rejected": -700.0,
      "loss": 0.3726,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.125,
      "step": 22340
    },
    {
      "epoch": 0.8319219817237721,
      "grad_norm": 11.592016875667252,
      "learning_rate": 4.1801573448531744e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.546875,
      "logps/chosen": -572.0,
      "logps/rejected": -736.0,
      "loss": 0.3743,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -5.21875,
      "step": 22350
    },
    {
      "epoch": 0.832294206324096,
      "grad_norm": 12.674069546353563,
      "learning_rate": 4.1621923201105466e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.65625,
      "logps/chosen": -556.0,
      "logps/rejected": -704.0,
      "loss": 0.3738,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.09375,
      "step": 22360
    },
    {
      "epoch": 0.8326664309244198,
      "grad_norm": 12.281052953404757,
      "learning_rate": 4.144262476593402e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.484375,
      "logps/chosen": -560.0,
      "logps/rejected": -700.0,
      "loss": 0.4148,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.21875,
      "step": 22370
    },
    {
      "epoch": 0.8330386555247437,
      "grad_norm": 11.369614007373897,
      "learning_rate": 4.126367844573322e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.5625,
      "logps/chosen": -556.0,
      "logps/rejected": -704.0,
      "loss": 0.3986,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.03125,
      "step": 22380
    },
    {
      "epoch": 0.8334108801250675,
      "grad_norm": 11.557620400453917,
      "learning_rate": 4.108508454262485e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -520.0,
      "logps/rejected": -700.0,
      "loss": 0.3776,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.203125,
      "rewards/margins": 1.8046875,
      "rewards/rejected": -5.0,
      "step": 22390
    },
    {
      "epoch": 0.8337831047253913,
      "grad_norm": 12.082177795522785,
      "learning_rate": 4.090684335813532e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.515625,
      "logps/chosen": -548.0,
      "logps/rejected": -672.0,
      "loss": 0.3777,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.84375,
      "step": 22400
    },
    {
      "epoch": 0.8341553293257151,
      "grad_norm": 13.643622751215613,
      "learning_rate": 4.0728955193195803e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.578125,
      "logps/chosen": -564.0,
      "logps/rejected": -720.0,
      "loss": 0.3926,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.15625,
      "step": 22410
    },
    {
      "epoch": 0.834527553926039,
      "grad_norm": 14.325486375950675,
      "learning_rate": 4.055142034814119e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -568.0,
      "logps/rejected": -708.0,
      "loss": 0.3988,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.21875,
      "step": 22420
    },
    {
      "epoch": 0.8348997785263628,
      "grad_norm": 11.992647517082453,
      "learning_rate": 4.0374239122710015e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.734375,
      "logps/chosen": -556.0,
      "logps/rejected": -724.0,
      "loss": 0.3767,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -5.1875,
      "step": 22430
    },
    {
      "epoch": 0.8352720031266866,
      "grad_norm": 10.572565023289853,
      "learning_rate": 4.019741181604383e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.453125,
      "logps/chosen": -580.0,
      "logps/rejected": -712.0,
      "loss": 0.3912,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.25,
      "step": 22440
    },
    {
      "epoch": 0.8356442277270105,
      "grad_norm": 12.81489776219769,
      "learning_rate": 4.002093872668655e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.546875,
      "logps/chosen": -568.0,
      "logps/rejected": -720.0,
      "loss": 0.3915,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.28125,
      "step": 22450
    },
    {
      "epoch": 0.8360164523273343,
      "grad_norm": 13.346271069700059,
      "learning_rate": 3.984482015258411e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5625,
      "logps/chosen": -588.0,
      "logps/rejected": -720.0,
      "loss": 0.3917,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.25,
      "step": 22460
    },
    {
      "epoch": 0.8363886769276582,
      "grad_norm": 13.944630347313069,
      "learning_rate": 3.966905639108384e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -568.0,
      "logps/rejected": -716.0,
      "loss": 0.3919,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.453125,
      "rewards/rejected": -5.21875,
      "step": 22470
    },
    {
      "epoch": 0.836760901527982,
      "grad_norm": 12.58195607174181,
      "learning_rate": 3.9493647738934046e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.609375,
      "logps/chosen": -556.0,
      "logps/rejected": -696.0,
      "loss": 0.3906,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.484375,
      "rewards/rejected": -5.03125,
      "step": 22480
    },
    {
      "epoch": 0.8371331261283058,
      "grad_norm": 10.749912145874687,
      "learning_rate": 3.931859449228353e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.71875,
      "logps/chosen": -584.0,
      "logps/rejected": -728.0,
      "loss": 0.3939,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.78125,
      "rewards/rejected": -5.40625,
      "step": 22490
    },
    {
      "epoch": 0.8375053507286296,
      "grad_norm": 13.291187479433246,
      "learning_rate": 3.91438969466811e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.484375,
      "logps/chosen": -556.0,
      "logps/rejected": -680.0,
      "loss": 0.4047,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -4.875,
      "step": 22500
    },
    {
      "epoch": 0.8378775753289535,
      "grad_norm": 10.681299687983422,
      "learning_rate": 3.896955539707483e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -560.0,
      "logps/rejected": -676.0,
      "loss": 0.3789,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.3125,
      "rewards/rejected": -5.0,
      "step": 22510
    },
    {
      "epoch": 0.8382497999292773,
      "grad_norm": 11.322664355640686,
      "learning_rate": 3.879557013781193e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -576.0,
      "logps/rejected": -736.0,
      "loss": 0.3772,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.75,
      "rewards/rejected": -5.5,
      "step": 22520
    },
    {
      "epoch": 0.8386220245296011,
      "grad_norm": 14.378209718403205,
      "learning_rate": 3.862194146263803e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.625,
      "logps/chosen": -588.0,
      "logps/rejected": -720.0,
      "loss": 0.371,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.46875,
      "rewards/rejected": -5.21875,
      "step": 22530
    },
    {
      "epoch": 0.838994249129925,
      "grad_norm": 11.986026759623151,
      "learning_rate": 3.844866966469668e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -568.0,
      "logps/rejected": -716.0,
      "loss": 0.3945,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -5.1875,
      "step": 22540
    },
    {
      "epoch": 0.8393664737302489,
      "grad_norm": 12.102231730445801,
      "learning_rate": 3.827575503652902e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -572.0,
      "logps/rejected": -724.0,
      "loss": 0.3961,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.40625,
      "step": 22550
    },
    {
      "epoch": 0.8397386983305727,
      "grad_norm": 12.032063256333581,
      "learning_rate": 3.810319787007293e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.578125,
      "logps/chosen": -564.0,
      "logps/rejected": -700.0,
      "loss": 0.388,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.1875,
      "step": 22560
    },
    {
      "epoch": 0.8401109229308965,
      "grad_norm": 11.494561791728342,
      "learning_rate": 3.7930998456663033e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.640625,
      "logps/chosen": -576.0,
      "logps/rejected": -732.0,
      "loss": 0.3545,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.4375,
      "step": 22570
    },
    {
      "epoch": 0.8404831475312203,
      "grad_norm": 12.227804213529739,
      "learning_rate": 3.7759157087029784e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -592.0,
      "logps/rejected": -712.0,
      "loss": 0.3862,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.828125,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -5.3125,
      "step": 22580
    },
    {
      "epoch": 0.8408553721315442,
      "grad_norm": 11.915447623318638,
      "learning_rate": 3.7587674051299254e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -556.0,
      "logps/rejected": -724.0,
      "loss": 0.4029,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.921875,
      "rewards/rejected": -5.34375,
      "step": 22590
    },
    {
      "epoch": 0.841227596731868,
      "grad_norm": 13.35019562026632,
      "learning_rate": 3.7416549638992406e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.40625,
      "logps/chosen": -556.0,
      "logps/rejected": -728.0,
      "loss": 0.3916,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.8671875,
      "rewards/rejected": -5.46875,
      "step": 22600
    },
    {
      "epoch": 0.8415998213321918,
      "grad_norm": 12.65974948463106,
      "learning_rate": 3.724578413902477e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.484375,
      "logps/chosen": -584.0,
      "logps/rejected": -736.0,
      "loss": 0.3904,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.25,
      "step": 22610
    },
    {
      "epoch": 0.8419720459325157,
      "grad_norm": 11.959046316642535,
      "learning_rate": 3.7075377839705964e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.546875,
      "logps/chosen": -552.0,
      "logps/rejected": -696.0,
      "loss": 0.3744,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.484375,
      "rewards/rejected": -4.96875,
      "step": 22620
    },
    {
      "epoch": 0.8423442705328396,
      "grad_norm": 12.010187618780444,
      "learning_rate": 3.690533102873911e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.46875,
      "logps/chosen": -536.0,
      "logps/rejected": -684.0,
      "loss": 0.371,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.09375,
      "step": 22630
    },
    {
      "epoch": 0.8427164951331634,
      "grad_norm": 12.244438336303698,
      "learning_rate": 3.673564399322046e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.484375,
      "logps/chosen": -556.0,
      "logps/rejected": -736.0,
      "loss": 0.3719,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -5.5,
      "step": 22640
    },
    {
      "epoch": 0.8430887197334872,
      "grad_norm": 15.032605421024707,
      "learning_rate": 3.6566317019638634e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -584.0,
      "logps/rejected": -724.0,
      "loss": 0.4102,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.28125,
      "step": 22650
    },
    {
      "epoch": 0.843460944333811,
      "grad_norm": 9.796469530869151,
      "learning_rate": 3.639735039387462e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.421875,
      "logps/chosen": -540.0,
      "logps/rejected": -704.0,
      "loss": 0.3906,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.15625,
      "step": 22660
    },
    {
      "epoch": 0.8438331689341348,
      "grad_norm": 12.810769008911148,
      "learning_rate": 3.6228744401200826e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -564.0,
      "logps/rejected": -728.0,
      "loss": 0.3876,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.8046875,
      "rewards/rejected": -5.40625,
      "step": 22670
    },
    {
      "epoch": 0.8442053935344587,
      "grad_norm": 13.482699680117431,
      "learning_rate": 3.606049932628094e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.40625,
      "logps/chosen": -572.0,
      "logps/rejected": -716.0,
      "loss": 0.3729,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.25,
      "step": 22680
    },
    {
      "epoch": 0.8445776181347825,
      "grad_norm": 13.444673015728613,
      "learning_rate": 3.58926154531691e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.546875,
      "logps/chosen": -560.0,
      "logps/rejected": -700.0,
      "loss": 0.3823,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.28125,
      "step": 22690
    },
    {
      "epoch": 0.8449498427351063,
      "grad_norm": 11.34943255405974,
      "learning_rate": 3.572509306530977e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.578125,
      "logps/chosen": -548.0,
      "logps/rejected": -688.0,
      "loss": 0.3778,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.09375,
      "step": 22700
    },
    {
      "epoch": 0.8453220673354302,
      "grad_norm": 12.810199515126428,
      "learning_rate": 3.555793244553712e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5,
      "logps/chosen": -560.0,
      "logps/rejected": -728.0,
      "loss": 0.3751,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.8125,
      "rewards/rejected": -5.34375,
      "step": 22710
    },
    {
      "epoch": 0.8456942919357541,
      "grad_norm": 12.132088025911909,
      "learning_rate": 3.53911338760744e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.34375,
      "logps/chosen": -564.0,
      "logps/rejected": -700.0,
      "loss": 0.3868,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.484375,
      "rewards/rejected": -5.0,
      "step": 22720
    },
    {
      "epoch": 0.8460665165360779,
      "grad_norm": 10.128236900452997,
      "learning_rate": 3.522469763853381e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -568.0,
      "logps/rejected": -720.0,
      "loss": 0.3893,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.828125,
      "rewards/rejected": -5.3125,
      "step": 22730
    },
    {
      "epoch": 0.8464387411364017,
      "grad_norm": 12.329146956120132,
      "learning_rate": 3.5058624013915516e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.4375,
      "logps/chosen": -576.0,
      "logps/rejected": -712.0,
      "loss": 0.3822,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.5625,
      "rewards/rejected": -5.125,
      "step": 22740
    },
    {
      "epoch": 0.8468109657367255,
      "grad_norm": 12.397874027495227,
      "learning_rate": 3.4892913282607714e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.734375,
      "logps/chosen": -612.0,
      "logps/rejected": -740.0,
      "loss": 0.3894,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.875,
      "rewards/margins": 1.40625,
      "rewards/rejected": -5.28125,
      "step": 22750
    },
    {
      "epoch": 0.8471831903370494,
      "grad_norm": 15.352153019164769,
      "learning_rate": 3.472756572438579e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -560.0,
      "logps/rejected": -696.0,
      "loss": 0.3897,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.0,
      "step": 22760
    },
    {
      "epoch": 0.8475554149373732,
      "grad_norm": 11.422185268043664,
      "learning_rate": 3.4562581618412136e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.484375,
      "logps/chosen": -560.0,
      "logps/rejected": -692.0,
      "loss": 0.3957,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -4.84375,
      "step": 22770
    },
    {
      "epoch": 0.847927639537697,
      "grad_norm": 13.30576643768592,
      "learning_rate": 3.4397961243235244e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5625,
      "logps/chosen": -552.0,
      "logps/rejected": -724.0,
      "loss": 0.3834,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.25,
      "step": 22780
    },
    {
      "epoch": 0.8482998641380208,
      "grad_norm": 13.254959281868377,
      "learning_rate": 3.423370487678972e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -556.0,
      "logps/rejected": -692.0,
      "loss": 0.3828,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -5.09375,
      "step": 22790
    },
    {
      "epoch": 0.8486720887383448,
      "grad_norm": 10.90969434531491,
      "learning_rate": 3.406981279639554e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.515625,
      "logps/chosen": -540.0,
      "logps/rejected": -684.0,
      "loss": 0.3999,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -5.0625,
      "step": 22800
    },
    {
      "epoch": 0.8490443133386686,
      "grad_norm": 11.954694139267161,
      "learning_rate": 3.390628527875764e-08,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.71875,
      "logps/chosen": -580.0,
      "logps/rejected": -732.0,
      "loss": 0.3736,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.21875,
      "step": 22810
    },
    {
      "epoch": 0.8494165379389924,
      "grad_norm": 11.536952949156992,
      "learning_rate": 3.374312259996548e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.65625,
      "logps/chosen": -560.0,
      "logps/rejected": -684.0,
      "loss": 0.4011,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.9375,
      "step": 22820
    },
    {
      "epoch": 0.8497887625393162,
      "grad_norm": 10.695845324728168,
      "learning_rate": 3.358032503549246e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.59375,
      "logps/chosen": -536.0,
      "logps/rejected": -704.0,
      "loss": 0.3748,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -5.125,
      "step": 22830
    },
    {
      "epoch": 0.8501609871396401,
      "grad_norm": 10.231839018209024,
      "learning_rate": 3.3417892860195615e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.65625,
      "logps/chosen": -556.0,
      "logps/rejected": -696.0,
      "loss": 0.3745,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.5,
      "rewards/rejected": -5.0625,
      "step": 22840
    },
    {
      "epoch": 0.8505332117399639,
      "grad_norm": 13.860574948051843,
      "learning_rate": 3.325582634831508e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.46875,
      "logps/chosen": -572.0,
      "logps/rejected": -732.0,
      "loss": 0.3942,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.28125,
      "step": 22850
    },
    {
      "epoch": 0.8509054363402877,
      "grad_norm": 11.000448179299113,
      "learning_rate": 3.309412577347365e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.65625,
      "logps/chosen": -528.0,
      "logps/rejected": -688.0,
      "loss": 0.3594,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.8125,
      "rewards/rejected": -5.125,
      "step": 22860
    },
    {
      "epoch": 0.8512776609406115,
      "grad_norm": 12.589871762431148,
      "learning_rate": 3.2932791408676275e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -544.0,
      "logps/rejected": -716.0,
      "loss": 0.3707,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.15625,
      "step": 22870
    },
    {
      "epoch": 0.8516498855409353,
      "grad_norm": 13.453113665030735,
      "learning_rate": 3.277182352630953e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.640625,
      "logps/chosen": -580.0,
      "logps/rejected": -752.0,
      "loss": 0.3738,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -5.40625,
      "step": 22880
    },
    {
      "epoch": 0.8520221101412593,
      "grad_norm": 10.904361677300223,
      "learning_rate": 3.261122239814135e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.703125,
      "logps/chosen": -584.0,
      "logps/rejected": -728.0,
      "loss": 0.3569,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.5625,
      "rewards/rejected": -5.125,
      "step": 22890
    },
    {
      "epoch": 0.8523943347415831,
      "grad_norm": 11.260309004716225,
      "learning_rate": 3.2450988295320446e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5625,
      "logps/chosen": -556.0,
      "logps/rejected": -704.0,
      "loss": 0.3796,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.125,
      "step": 22900
    },
    {
      "epoch": 0.8527665593419069,
      "grad_norm": 10.594542706789909,
      "learning_rate": 3.229112148837593e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.6875,
      "logps/chosen": -580.0,
      "logps/rejected": -712.0,
      "loss": 0.3942,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.1875,
      "step": 22910
    },
    {
      "epoch": 0.8531387839422307,
      "grad_norm": 11.931010611199902,
      "learning_rate": 3.2131622247216576e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.546875,
      "logps/chosen": -560.0,
      "logps/rejected": -700.0,
      "loss": 0.3826,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.484375,
      "rewards/rejected": -5.09375,
      "step": 22920
    },
    {
      "epoch": 0.8535110085425546,
      "grad_norm": 10.970147270464924,
      "learning_rate": 3.197249084113082e-08,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.578125,
      "logps/chosen": -544.0,
      "logps/rejected": -688.0,
      "loss": 0.374,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -4.96875,
      "step": 22930
    },
    {
      "epoch": 0.8538832331428784,
      "grad_norm": 13.932384083625514,
      "learning_rate": 3.1813727538785943e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5625,
      "logps/chosen": -564.0,
      "logps/rejected": -708.0,
      "loss": 0.3797,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.125,
      "step": 22940
    },
    {
      "epoch": 0.8542554577432022,
      "grad_norm": 12.404701925235603,
      "learning_rate": 3.165533260822784e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.390625,
      "logps/chosen": -528.0,
      "logps/rejected": -708.0,
      "loss": 0.3771,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.84375,
      "rewards/rejected": -5.1875,
      "step": 22950
    },
    {
      "epoch": 0.854627682343526,
      "grad_norm": 11.683751792658077,
      "learning_rate": 3.1497306316880385e-08,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.515625,
      "logps/chosen": -556.0,
      "logps/rejected": -692.0,
      "loss": 0.3662,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.15625,
      "step": 22960
    },
    {
      "epoch": 0.85499990694385,
      "grad_norm": 11.96430022260537,
      "learning_rate": 3.133964893154503e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.375,
      "logps/chosen": -608.0,
      "logps/rejected": -752.0,
      "loss": 0.3726,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.8125,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.375,
      "step": 22970
    },
    {
      "epoch": 0.8553721315441738,
      "grad_norm": 11.272196428653588,
      "learning_rate": 3.118236071840052e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.65625,
      "logps/chosen": -576.0,
      "logps/rejected": -736.0,
      "loss": 0.3761,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.3125,
      "step": 22980
    },
    {
      "epoch": 0.8557443561444976,
      "grad_norm": 10.843545128369783,
      "learning_rate": 3.102544194300219e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.65625,
      "logps/chosen": -584.0,
      "logps/rejected": -744.0,
      "loss": 0.3611,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.84375,
      "rewards/rejected": -5.4375,
      "step": 22990
    },
    {
      "epoch": 0.8561165807448214,
      "grad_norm": 10.09343917389029,
      "learning_rate": 3.0868892870281774e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.46875,
      "logps/chosen": -576.0,
      "logps/rejected": -784.0,
      "loss": 0.3833,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.859375,
      "rewards/margins": 1.9609375,
      "rewards/rejected": -5.8125,
      "step": 23000
    },
    {
      "epoch": 0.8564888053451453,
      "grad_norm": 14.260248883708497,
      "learning_rate": 3.071271376454665e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -552.0,
      "logps/rejected": -712.0,
      "loss": 0.3877,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -5.3125,
      "step": 23010
    },
    {
      "epoch": 0.8568610299454691,
      "grad_norm": 12.240362865300328,
      "learning_rate": 3.0556904889479736e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -556.0,
      "logps/rejected": -712.0,
      "loss": 0.3822,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.8125,
      "rewards/rejected": -5.25,
      "step": 23020
    },
    {
      "epoch": 0.8572332545457929,
      "grad_norm": 10.599849102991858,
      "learning_rate": 3.040146650813866e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.578125,
      "logps/chosen": -544.0,
      "logps/rejected": -692.0,
      "loss": 0.4045,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -5.0,
      "step": 23030
    },
    {
      "epoch": 0.8576054791461167,
      "grad_norm": 14.148227395055002,
      "learning_rate": 3.02463988829558e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.453125,
      "logps/chosen": -556.0,
      "logps/rejected": -712.0,
      "loss": 0.394,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -5.15625,
      "step": 23040
    },
    {
      "epoch": 0.8579777037464406,
      "grad_norm": 11.142424843003683,
      "learning_rate": 3.009170227573749e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -560.0,
      "logps/rejected": -720.0,
      "loss": 0.3627,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.25,
      "step": 23050
    },
    {
      "epoch": 0.8583499283467645,
      "grad_norm": 12.833951079807822,
      "learning_rate": 2.993737694766349e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.65625,
      "logps/chosen": -548.0,
      "logps/rejected": -700.0,
      "loss": 0.3859,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.0625,
      "step": 23060
    },
    {
      "epoch": 0.8587221529470883,
      "grad_norm": 10.419902021997403,
      "learning_rate": 2.978342315928692e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.375,
      "logps/chosen": -568.0,
      "logps/rejected": -728.0,
      "loss": 0.3987,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.46875,
      "rewards/rejected": -5.09375,
      "step": 23070
    },
    {
      "epoch": 0.8590943775474121,
      "grad_norm": 10.701710886380194,
      "learning_rate": 2.9629841170533408e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.484375,
      "logps/chosen": -564.0,
      "logps/rejected": -720.0,
      "loss": 0.3858,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -5.3125,
      "step": 23080
    },
    {
      "epoch": 0.859466602147736,
      "grad_norm": 13.055168968801965,
      "learning_rate": 2.9476631240701212e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.453125,
      "logps/chosen": -580.0,
      "logps/rejected": -712.0,
      "loss": 0.3817,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.9375,
      "step": 23090
    },
    {
      "epoch": 0.8598388267480598,
      "grad_norm": 12.386666671125512,
      "learning_rate": 2.9323793628460018e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5,
      "logps/chosen": -576.0,
      "logps/rejected": -712.0,
      "loss": 0.3727,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.21875,
      "step": 23100
    },
    {
      "epoch": 0.8602110513483836,
      "grad_norm": 12.888029665781634,
      "learning_rate": 2.917132859185123e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5625,
      "logps/chosen": -540.0,
      "logps/rejected": -680.0,
      "loss": 0.4,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.6875,
      "rewards/rejected": -5.0,
      "step": 23110
    },
    {
      "epoch": 0.8605832759487074,
      "grad_norm": 12.878738510816419,
      "learning_rate": 2.9019236388286907e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.578125,
      "logps/chosen": -552.0,
      "logps/rejected": -720.0,
      "loss": 0.3833,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -5.1875,
      "step": 23120
    },
    {
      "epoch": 0.8609555005490312,
      "grad_norm": 14.123621605611671,
      "learning_rate": 2.886751727455003e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.59375,
      "logps/chosen": -568.0,
      "logps/rejected": -712.0,
      "loss": 0.4053,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.25,
      "step": 23130
    },
    {
      "epoch": 0.8613277251493551,
      "grad_norm": 10.394612807407064,
      "learning_rate": 2.8716171506793423e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -548.0,
      "logps/rejected": -680.0,
      "loss": 0.3884,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.96875,
      "step": 23140
    },
    {
      "epoch": 0.861699949749679,
      "grad_norm": 11.932535704580127,
      "learning_rate": 2.8565199340539553e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.609375,
      "logps/chosen": -540.0,
      "logps/rejected": -712.0,
      "loss": 0.37,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.1875,
      "step": 23150
    },
    {
      "epoch": 0.8620721743500028,
      "grad_norm": 13.674770662967783,
      "learning_rate": 2.8414601030680273e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.46875,
      "logps/chosen": -560.0,
      "logps/rejected": -716.0,
      "loss": 0.3791,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.28125,
      "step": 23160
    },
    {
      "epoch": 0.8624443989503267,
      "grad_norm": 12.283825941117325,
      "learning_rate": 2.8264376831476034e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.65625,
      "logps/chosen": -564.0,
      "logps/rejected": -704.0,
      "loss": 0.3808,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.28125,
      "step": 23170
    },
    {
      "epoch": 0.8628166235506505,
      "grad_norm": 12.179648728362498,
      "learning_rate": 2.8114526996556002e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.515625,
      "logps/chosen": -560.0,
      "logps/rejected": -708.0,
      "loss": 0.3864,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.28125,
      "step": 23180
    },
    {
      "epoch": 0.8631888481509743,
      "grad_norm": 12.488586429740465,
      "learning_rate": 2.796505177891692e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -580.0,
      "logps/rejected": -740.0,
      "loss": 0.3826,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -5.5,
      "step": 23190
    },
    {
      "epoch": 0.8635610727512981,
      "grad_norm": 12.75042849949888,
      "learning_rate": 2.7815951430923247e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.65625,
      "logps/chosen": -560.0,
      "logps/rejected": -692.0,
      "loss": 0.3751,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.5625,
      "rewards/rejected": -5.03125,
      "step": 23200
    },
    {
      "epoch": 0.8639332973516219,
      "grad_norm": 12.4869672995221,
      "learning_rate": 2.7667226204306588e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.65625,
      "logps/chosen": -588.0,
      "logps/rejected": -724.0,
      "loss": 0.3981,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.8125,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -5.34375,
      "step": 23210
    },
    {
      "epoch": 0.8643055219519458,
      "grad_norm": 12.711373494132829,
      "learning_rate": 2.7518876350164923e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -560.0,
      "logps/rejected": -708.0,
      "loss": 0.386,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -5.125,
      "step": 23220
    },
    {
      "epoch": 0.8646777465522696,
      "grad_norm": 13.857481635183076,
      "learning_rate": 2.737090211896295e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -568.0,
      "logps/rejected": -712.0,
      "loss": 0.4005,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -5.21875,
      "step": 23230
    },
    {
      "epoch": 0.8650499711525935,
      "grad_norm": 11.502045103713689,
      "learning_rate": 2.7223303760530726e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -564.0,
      "logps/rejected": -720.0,
      "loss": 0.3895,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.8046875,
      "rewards/rejected": -5.34375,
      "step": 23240
    },
    {
      "epoch": 0.8654221957529173,
      "grad_norm": 12.034651288288211,
      "learning_rate": 2.7076081524064016e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -568.0,
      "logps/rejected": -720.0,
      "loss": 0.3912,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.28125,
      "step": 23250
    },
    {
      "epoch": 0.8657944203532412,
      "grad_norm": 14.539135381240657,
      "learning_rate": 2.692923565812333e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.625,
      "logps/chosen": -564.0,
      "logps/rejected": -728.0,
      "loss": 0.3776,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.1875,
      "step": 23260
    },
    {
      "epoch": 0.866166644953565,
      "grad_norm": 10.786563195947812,
      "learning_rate": 2.678276641063404e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -548.0,
      "logps/rejected": -704.0,
      "loss": 0.3658,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.125,
      "step": 23270
    },
    {
      "epoch": 0.8665388695538888,
      "grad_norm": 13.696824389079703,
      "learning_rate": 2.663667402888531e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.640625,
      "logps/chosen": -552.0,
      "logps/rejected": -676.0,
      "loss": 0.3906,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -4.9375,
      "step": 23280
    },
    {
      "epoch": 0.8669110941542126,
      "grad_norm": 10.962616376787157,
      "learning_rate": 2.649095875953028e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.703125,
      "logps/chosen": -556.0,
      "logps/rejected": -692.0,
      "loss": 0.3898,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.03125,
      "step": 23290
    },
    {
      "epoch": 0.8672833187545365,
      "grad_norm": 12.91882781309777,
      "learning_rate": 2.634562084858538e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.65625,
      "logps/chosen": -564.0,
      "logps/rejected": -684.0,
      "loss": 0.3869,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -5.03125,
      "step": 23300
    },
    {
      "epoch": 0.8676555433548603,
      "grad_norm": 13.151272361526251,
      "learning_rate": 2.6200660541429692e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5,
      "logps/chosen": -576.0,
      "logps/rejected": -744.0,
      "loss": 0.3699,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.9140625,
      "rewards/rejected": -5.59375,
      "step": 23310
    },
    {
      "epoch": 0.8680277679551841,
      "grad_norm": 14.096329777021285,
      "learning_rate": 2.605607808280516e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -576.0,
      "logps/rejected": -732.0,
      "loss": 0.3929,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -5.21875,
      "step": 23320
    },
    {
      "epoch": 0.868399992555508,
      "grad_norm": 11.775697804716568,
      "learning_rate": 2.5911873716815442e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.640625,
      "logps/chosen": -548.0,
      "logps/rejected": -696.0,
      "loss": 0.3863,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.6875,
      "rewards/rejected": -5.03125,
      "step": 23330
    },
    {
      "epoch": 0.8687722171558319,
      "grad_norm": 10.539908498739742,
      "learning_rate": 2.5768047686926086e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.515625,
      "logps/chosen": -544.0,
      "logps/rejected": -720.0,
      "loss": 0.3722,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.15625,
      "step": 23340
    },
    {
      "epoch": 0.8691444417561557,
      "grad_norm": 11.760978076366166,
      "learning_rate": 2.562460023596369e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.421875,
      "logps/chosen": -536.0,
      "logps/rejected": -696.0,
      "loss": 0.3785,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.8125,
      "rewards/rejected": -5.25,
      "step": 23350
    },
    {
      "epoch": 0.8695166663564795,
      "grad_norm": 11.124139209337201,
      "learning_rate": 2.5481531606115825e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.421875,
      "logps/chosen": -568.0,
      "logps/rejected": -712.0,
      "loss": 0.392,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.21875,
      "step": 23360
    },
    {
      "epoch": 0.8698888909568033,
      "grad_norm": 14.016190315022053,
      "learning_rate": 2.533884203893044e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -536.0,
      "logps/rejected": -696.0,
      "loss": 0.3805,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.125,
      "step": 23370
    },
    {
      "epoch": 0.8702611155571272,
      "grad_norm": 12.984999792042865,
      "learning_rate": 2.5196531775315528e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.53125,
      "logps/chosen": -576.0,
      "logps/rejected": -708.0,
      "loss": 0.384,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.3125,
      "step": 23380
    },
    {
      "epoch": 0.870633340157451,
      "grad_norm": 12.884564756660241,
      "learning_rate": 2.5054601055538667e-08,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.453125,
      "logps/chosen": -560.0,
      "logps/rejected": -704.0,
      "loss": 0.3552,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -5.21875,
      "step": 23390
    },
    {
      "epoch": 0.8710055647577748,
      "grad_norm": 10.406446386320892,
      "learning_rate": 2.4913050119226563e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5625,
      "logps/chosen": -568.0,
      "logps/rejected": -712.0,
      "loss": 0.3852,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.1875,
      "step": 23400
    },
    {
      "epoch": 0.8713777893580987,
      "grad_norm": 11.377260501710019,
      "learning_rate": 2.4771879205364832e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.46875,
      "logps/chosen": -588.0,
      "logps/rejected": -740.0,
      "loss": 0.3874,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.875,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.46875,
      "step": 23410
    },
    {
      "epoch": 0.8717500139584226,
      "grad_norm": 12.223092946879728,
      "learning_rate": 2.4631088552297423e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -604.0,
      "logps/rejected": -724.0,
      "loss": 0.3813,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.78125,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -5.1875,
      "step": 23420
    },
    {
      "epoch": 0.8721222385587464,
      "grad_norm": 11.589246429096017,
      "learning_rate": 2.4490678397726396e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.46875,
      "logps/chosen": -572.0,
      "logps/rejected": -748.0,
      "loss": 0.3791,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.90625,
      "rewards/rejected": -5.625,
      "step": 23430
    },
    {
      "epoch": 0.8724944631590702,
      "grad_norm": 13.546107867890342,
      "learning_rate": 2.4350648978711142e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.46875,
      "logps/chosen": -552.0,
      "logps/rejected": -708.0,
      "loss": 0.3695,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.0625,
      "step": 23440
    },
    {
      "epoch": 0.872866687759394,
      "grad_norm": 10.969393248669302,
      "learning_rate": 2.421100053166844e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.46875,
      "logps/chosen": -584.0,
      "logps/rejected": -760.0,
      "loss": 0.3779,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.8046875,
      "rewards/rejected": -5.4375,
      "step": 23450
    },
    {
      "epoch": 0.8732389123597178,
      "grad_norm": 10.33061116761562,
      "learning_rate": 2.4071733292371944e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.46875,
      "logps/chosen": -544.0,
      "logps/rejected": -732.0,
      "loss": 0.3632,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.9921875,
      "rewards/rejected": -5.3125,
      "step": 23460
    },
    {
      "epoch": 0.8736111369600417,
      "grad_norm": 11.753206328043419,
      "learning_rate": 2.3932847495951453e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.703125,
      "logps/chosen": -596.0,
      "logps/rejected": -744.0,
      "loss": 0.3589,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.4375,
      "step": 23470
    },
    {
      "epoch": 0.8739833615603655,
      "grad_norm": 11.228602280438196,
      "learning_rate": 2.3794343376892982e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.46875,
      "logps/chosen": -510.0,
      "logps/rejected": -668.0,
      "loss": 0.3757,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.0,
      "step": 23480
    },
    {
      "epoch": 0.8743555861606893,
      "grad_norm": 11.924076776829123,
      "learning_rate": 2.3656221169037926e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -588.0,
      "logps/rejected": -736.0,
      "loss": 0.3842,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.765625,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -5.4375,
      "step": 23490
    },
    {
      "epoch": 0.8747278107610132,
      "grad_norm": 12.003483822689239,
      "learning_rate": 2.3518481105583104e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.65625,
      "logps/chosen": -576.0,
      "logps/rejected": -716.0,
      "loss": 0.3914,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.546875,
      "rewards/rejected": -5.25,
      "step": 23500
    },
    {
      "epoch": 0.8751000353613371,
      "grad_norm": 11.207209612867407,
      "learning_rate": 2.3381123419080022e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.65625,
      "logps/chosen": -604.0,
      "logps/rejected": -740.0,
      "loss": 0.3782,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -5.53125,
      "step": 23510
    },
    {
      "epoch": 0.8754722599616609,
      "grad_norm": 11.272574765356918,
      "learning_rate": 2.3244148341434707e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.484375,
      "logps/chosen": -560.0,
      "logps/rejected": -692.0,
      "loss": 0.4065,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -5.03125,
      "step": 23520
    },
    {
      "epoch": 0.8758444845619847,
      "grad_norm": 13.851650528577782,
      "learning_rate": 2.3107556103907022e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.375,
      "logps/chosen": -588.0,
      "logps/rejected": -744.0,
      "loss": 0.3818,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.40625,
      "step": 23530
    },
    {
      "epoch": 0.8762167091623085,
      "grad_norm": 12.855072491815989,
      "learning_rate": 2.2971346937110668e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.34375,
      "logps/chosen": -544.0,
      "logps/rejected": -740.0,
      "loss": 0.3938,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -5.40625,
      "step": 23540
    },
    {
      "epoch": 0.8765889337626324,
      "grad_norm": 10.807947875941341,
      "learning_rate": 2.283552107101247e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.625,
      "logps/chosen": -580.0,
      "logps/rejected": -736.0,
      "loss": 0.3747,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.734375,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.3125,
      "step": 23550
    },
    {
      "epoch": 0.8769611583629562,
      "grad_norm": 12.177408438549005,
      "learning_rate": 2.270007873493221e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.609375,
      "logps/chosen": -564.0,
      "logps/rejected": -704.0,
      "loss": 0.3977,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.96875,
      "step": 23560
    },
    {
      "epoch": 0.87733338296328,
      "grad_norm": 11.305274728744175,
      "learning_rate": 2.256502015754211e-08,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.515625,
      "logps/chosen": -568.0,
      "logps/rejected": -728.0,
      "loss": 0.4006,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.34375,
      "step": 23570
    },
    {
      "epoch": 0.8777056075636038,
      "grad_norm": 11.791262752501668,
      "learning_rate": 2.2430345566866378e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -572.0,
      "logps/rejected": -700.0,
      "loss": 0.3885,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -5.25,
      "step": 23580
    },
    {
      "epoch": 0.8780778321639278,
      "grad_norm": 11.308617605258027,
      "learning_rate": 2.2296055190281023e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -576.0,
      "logps/rejected": -716.0,
      "loss": 0.3871,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.34375,
      "step": 23590
    },
    {
      "epoch": 0.8784500567642516,
      "grad_norm": 13.571884428895302,
      "learning_rate": 2.2162149254513374e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.546875,
      "logps/chosen": -564.0,
      "logps/rejected": -700.0,
      "loss": 0.3838,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.15625,
      "step": 23600
    },
    {
      "epoch": 0.8788222813645754,
      "grad_norm": 12.466298103304705,
      "learning_rate": 2.2028627985641702e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.59375,
      "logps/chosen": -552.0,
      "logps/rejected": -704.0,
      "loss": 0.3724,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.09375,
      "step": 23610
    },
    {
      "epoch": 0.8791945059648992,
      "grad_norm": 13.147135182876884,
      "learning_rate": 2.1895491609094735e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.65625,
      "logps/chosen": -572.0,
      "logps/rejected": -692.0,
      "loss": 0.3778,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.09375,
      "step": 23620
    },
    {
      "epoch": 0.879566730565223,
      "grad_norm": 11.712424770755437,
      "learning_rate": 2.1762740349651476e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.59375,
      "logps/chosen": -520.0,
      "logps/rejected": -648.0,
      "loss": 0.3959,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.296875,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.75,
      "step": 23630
    },
    {
      "epoch": 0.8799389551655469,
      "grad_norm": 13.241241109422889,
      "learning_rate": 2.1630374431440668e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.578125,
      "logps/chosen": -560.0,
      "logps/rejected": -720.0,
      "loss": 0.3736,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.5625,
      "rewards/rejected": -5.21875,
      "step": 23640
    },
    {
      "epoch": 0.8803111797658707,
      "grad_norm": 11.074731513781721,
      "learning_rate": 2.149839407794052e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -580.0,
      "logps/rejected": -716.0,
      "loss": 0.3711,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.15625,
      "step": 23650
    },
    {
      "epoch": 0.8806834043661945,
      "grad_norm": 12.40920976577564,
      "learning_rate": 2.136679951197823e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.640625,
      "logps/chosen": -580.0,
      "logps/rejected": -724.0,
      "loss": 0.3585,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.25,
      "step": 23660
    },
    {
      "epoch": 0.8810556289665183,
      "grad_norm": 16.411941028215804,
      "learning_rate": 2.1235590955729632e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.46875,
      "logps/chosen": -552.0,
      "logps/rejected": -708.0,
      "loss": 0.3887,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.15625,
      "step": 23670
    },
    {
      "epoch": 0.8814278535668423,
      "grad_norm": 13.08037071206278,
      "learning_rate": 2.1104768630718915e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.625,
      "logps/chosen": -568.0,
      "logps/rejected": -696.0,
      "loss": 0.3972,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.4375,
      "rewards/rejected": -4.9375,
      "step": 23680
    },
    {
      "epoch": 0.8818000781671661,
      "grad_norm": 12.207385165329326,
      "learning_rate": 2.0974332757818153e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.484375,
      "logps/chosen": -560.0,
      "logps/rejected": -716.0,
      "loss": 0.3806,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.0,
      "step": 23690
    },
    {
      "epoch": 0.8821723027674899,
      "grad_norm": 12.663544905561444,
      "learning_rate": 2.084428355724699e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.671875,
      "logps/chosen": -564.0,
      "logps/rejected": -724.0,
      "loss": 0.3915,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -5.3125,
      "step": 23700
    },
    {
      "epoch": 0.8825445273678137,
      "grad_norm": 12.63680715379124,
      "learning_rate": 2.0714621248572124e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.546875,
      "logps/chosen": -564.0,
      "logps/rejected": -700.0,
      "loss": 0.3862,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -5.0625,
      "step": 23710
    },
    {
      "epoch": 0.8829167519681376,
      "grad_norm": 12.116378132912256,
      "learning_rate": 2.0585346050707138e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.390625,
      "logps/chosen": -548.0,
      "logps/rejected": -700.0,
      "loss": 0.3819,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.09375,
      "step": 23720
    },
    {
      "epoch": 0.8832889765684614,
      "grad_norm": 13.799904290476471,
      "learning_rate": 2.045645818191208e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.515625,
      "logps/chosen": -560.0,
      "logps/rejected": -688.0,
      "loss": 0.3926,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -5.03125,
      "step": 23730
    },
    {
      "epoch": 0.8836612011687852,
      "grad_norm": 12.01353041249162,
      "learning_rate": 2.0327957859792967e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.546875,
      "logps/chosen": -584.0,
      "logps/rejected": -728.0,
      "loss": 0.384,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.3125,
      "step": 23740
    },
    {
      "epoch": 0.884033425769109,
      "grad_norm": 13.709316895102674,
      "learning_rate": 2.0199845301301616e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.453125,
      "logps/chosen": -584.0,
      "logps/rejected": -744.0,
      "loss": 0.3779,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.53125,
      "rewards/rejected": -5.25,
      "step": 23750
    },
    {
      "epoch": 0.8844056503694329,
      "grad_norm": 13.093764498539544,
      "learning_rate": 2.007212072273501e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5,
      "logps/chosen": -564.0,
      "logps/rejected": -716.0,
      "loss": 0.3898,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.1875,
      "step": 23760
    },
    {
      "epoch": 0.8847778749697568,
      "grad_norm": 11.611779497069785,
      "learning_rate": 1.9944784339735232e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.6875,
      "logps/chosen": -576.0,
      "logps/rejected": -720.0,
      "loss": 0.3823,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -5.1875,
      "step": 23770
    },
    {
      "epoch": 0.8851500995700806,
      "grad_norm": 10.649101978962017,
      "learning_rate": 1.981783636728887e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -536.0,
      "logps/rejected": -700.0,
      "loss": 0.3629,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -5.15625,
      "step": 23780
    },
    {
      "epoch": 0.8855223241704044,
      "grad_norm": 11.413708937593873,
      "learning_rate": 1.9691277019726838e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.734375,
      "logps/chosen": -572.0,
      "logps/rejected": -740.0,
      "loss": 0.3811,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.875,
      "rewards/rejected": -5.40625,
      "step": 23790
    },
    {
      "epoch": 0.8858945487707283,
      "grad_norm": 10.727184745637505,
      "learning_rate": 1.956510651072385e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.46875,
      "logps/chosen": -552.0,
      "logps/rejected": -716.0,
      "loss": 0.3704,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.21875,
      "step": 23800
    },
    {
      "epoch": 0.8862667733710521,
      "grad_norm": 13.754513635064214,
      "learning_rate": 1.9439325053298123e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.609375,
      "logps/chosen": -596.0,
      "logps/rejected": -720.0,
      "loss": 0.3893,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.921875,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -5.28125,
      "step": 23810
    },
    {
      "epoch": 0.8866389979713759,
      "grad_norm": 12.306018491401295,
      "learning_rate": 1.9313932859811033e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.640625,
      "logps/chosen": -552.0,
      "logps/rejected": -704.0,
      "loss": 0.3911,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.375,
      "step": 23820
    },
    {
      "epoch": 0.8870112225716997,
      "grad_norm": 13.131195403309832,
      "learning_rate": 1.918893014196682e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.59375,
      "logps/chosen": -524.0,
      "logps/rejected": -664.0,
      "loss": 0.393,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.59375,
      "rewards/rejected": -4.875,
      "step": 23830
    },
    {
      "epoch": 0.8873834471720236,
      "grad_norm": 11.13844579345589,
      "learning_rate": 1.9064317110812106e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -564.0,
      "logps/rejected": -712.0,
      "loss": 0.3871,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.1875,
      "step": 23840
    },
    {
      "epoch": 0.8877556717723475,
      "grad_norm": 11.780598852378576,
      "learning_rate": 1.894009397673549e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -572.0,
      "logps/rejected": -720.0,
      "loss": 0.3859,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.28125,
      "step": 23850
    },
    {
      "epoch": 0.8881278963726713,
      "grad_norm": 12.85221088418439,
      "learning_rate": 1.881626094946745e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.390625,
      "logps/chosen": -580.0,
      "logps/rejected": -720.0,
      "loss": 0.3739,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.21875,
      "step": 23860
    },
    {
      "epoch": 0.8885001209729951,
      "grad_norm": 12.541458633470292,
      "learning_rate": 1.8692818238079778e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.40625,
      "logps/chosen": -556.0,
      "logps/rejected": -712.0,
      "loss": 0.4067,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.1875,
      "step": 23870
    },
    {
      "epoch": 0.888872345573319,
      "grad_norm": 15.15725841802495,
      "learning_rate": 1.8569766050985257e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.484375,
      "logps/chosen": -532.0,
      "logps/rejected": -684.0,
      "loss": 0.3837,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -4.96875,
      "step": 23880
    },
    {
      "epoch": 0.8892445701736428,
      "grad_norm": 11.983520660722853,
      "learning_rate": 1.8447104595937428e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.546875,
      "logps/chosen": -552.0,
      "logps/rejected": -748.0,
      "loss": 0.3577,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.546875,
      "rewards/margins": 2.078125,
      "rewards/rejected": -5.625,
      "step": 23890
    },
    {
      "epoch": 0.8896167947739666,
      "grad_norm": 11.772453425099208,
      "learning_rate": 1.832483408002994e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.53125,
      "logps/chosen": -552.0,
      "logps/rejected": -676.0,
      "loss": 0.3964,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.375,
      "rewards/rejected": -4.875,
      "step": 23900
    },
    {
      "epoch": 0.8899890193742904,
      "grad_norm": 11.259893205695786,
      "learning_rate": 1.82029547096966e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.609375,
      "logps/chosen": -596.0,
      "logps/rejected": -712.0,
      "loss": 0.3722,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.546875,
      "rewards/rejected": -5.3125,
      "step": 23910
    },
    {
      "epoch": 0.8903612439746142,
      "grad_norm": 13.708649789989705,
      "learning_rate": 1.8081466690710755e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.421875,
      "logps/chosen": -568.0,
      "logps/rejected": -716.0,
      "loss": 0.3912,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.25,
      "step": 23920
    },
    {
      "epoch": 0.8907334685749381,
      "grad_norm": 13.395744618461084,
      "learning_rate": 1.7960370228185083e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -552.0,
      "logps/rejected": -704.0,
      "loss": 0.3771,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.0625,
      "step": 23930
    },
    {
      "epoch": 0.891105693175262,
      "grad_norm": 11.7678943374734,
      "learning_rate": 1.7839665526571014e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.578125,
      "logps/chosen": -568.0,
      "logps/rejected": -736.0,
      "loss": 0.3822,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -5.40625,
      "step": 23940
    },
    {
      "epoch": 0.8914779177755858,
      "grad_norm": 12.955052366881228,
      "learning_rate": 1.7719352789658775e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.453125,
      "logps/chosen": -532.0,
      "logps/rejected": -692.0,
      "loss": 0.3977,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.09375,
      "step": 23950
    },
    {
      "epoch": 0.8918501423759096,
      "grad_norm": 11.172707754543932,
      "learning_rate": 1.759943222057661e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -568.0,
      "logps/rejected": -724.0,
      "loss": 0.3609,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.1875,
      "step": 23960
    },
    {
      "epoch": 0.8922223669762335,
      "grad_norm": 15.169496427917199,
      "learning_rate": 1.7479904021790863e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -596.0,
      "logps/rejected": -736.0,
      "loss": 0.3803,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.828125,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.40625,
      "step": 23970
    },
    {
      "epoch": 0.8925945915765573,
      "grad_norm": 12.660485067671308,
      "learning_rate": 1.736076839510531e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.546875,
      "logps/chosen": -556.0,
      "logps/rejected": -720.0,
      "loss": 0.3893,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.25,
      "step": 23980
    },
    {
      "epoch": 0.8929668161768811,
      "grad_norm": 11.626765311746997,
      "learning_rate": 1.7242025541660875e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.671875,
      "logps/chosen": -556.0,
      "logps/rejected": -712.0,
      "loss": 0.3869,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -5.03125,
      "step": 23990
    },
    {
      "epoch": 0.8933390407772049,
      "grad_norm": 10.037377994082629,
      "learning_rate": 1.7123675661935483e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.640625,
      "logps/chosen": -592.0,
      "logps/rejected": -740.0,
      "loss": 0.3908,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.828125,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.375,
      "step": 24000
    },
    {
      "epoch": 0.8937112653775288,
      "grad_norm": 14.36100647199057,
      "learning_rate": 1.7005718955743432e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.65625,
      "logps/chosen": -596.0,
      "logps/rejected": -728.0,
      "loss": 0.4112,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.765625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.375,
      "step": 24010
    },
    {
      "epoch": 0.8940834899778526,
      "grad_norm": 13.037795830508866,
      "learning_rate": 1.6888155622235474e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.546875,
      "logps/chosen": -592.0,
      "logps/rejected": -748.0,
      "loss": 0.3963,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -3.734375,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.375,
      "step": 24020
    },
    {
      "epoch": 0.8944557145781765,
      "grad_norm": 12.628480113887083,
      "learning_rate": 1.6770985859897863e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.390625,
      "logps/chosen": -540.0,
      "logps/rejected": -700.0,
      "loss": 0.3693,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.09375,
      "step": 24030
    },
    {
      "epoch": 0.8948279391785003,
      "grad_norm": 11.761137534229874,
      "learning_rate": 1.6654209866552638e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.671875,
      "logps/chosen": -596.0,
      "logps/rejected": -728.0,
      "loss": 0.3776,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.75,
      "rewards/rejected": -5.46875,
      "step": 24040
    },
    {
      "epoch": 0.8952001637788242,
      "grad_norm": 13.037928442911003,
      "learning_rate": 1.6537827839356923e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.40625,
      "logps/chosen": -536.0,
      "logps/rejected": -700.0,
      "loss": 0.4078,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.0625,
      "step": 24050
    },
    {
      "epoch": 0.895572388379148,
      "grad_norm": 14.708043089327633,
      "learning_rate": 1.642183997480273e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.546875,
      "logps/chosen": -560.0,
      "logps/rejected": -668.0,
      "loss": 0.4025,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.28125,
      "rewards/rejected": -4.84375,
      "step": 24060
    },
    {
      "epoch": 0.8959446129794718,
      "grad_norm": 13.049789967995112,
      "learning_rate": 1.630624646871659e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.40625,
      "logps/chosen": -552.0,
      "logps/rejected": -712.0,
      "loss": 0.3977,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.8359375,
      "rewards/rejected": -5.34375,
      "step": 24070
    },
    {
      "epoch": 0.8963168375797956,
      "grad_norm": 10.869959036181031,
      "learning_rate": 1.619104751625913e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.453125,
      "logps/chosen": -560.0,
      "logps/rejected": -720.0,
      "loss": 0.3516,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.1875,
      "step": 24080
    },
    {
      "epoch": 0.8966890621801195,
      "grad_norm": 11.911213582061011,
      "learning_rate": 1.6076243311924974e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.703125,
      "logps/chosen": -532.0,
      "logps/rejected": -656.0,
      "loss": 0.3624,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -4.875,
      "step": 24090
    },
    {
      "epoch": 0.8970612867804433,
      "grad_norm": 11.81044008004693,
      "learning_rate": 1.5961834049542154e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -584.0,
      "logps/rejected": -720.0,
      "loss": 0.408,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.734375,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.3125,
      "step": 24100
    },
    {
      "epoch": 0.8974335113807671,
      "grad_norm": 12.390901475107809,
      "learning_rate": 1.5847819922272077e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.515625,
      "logps/chosen": -552.0,
      "logps/rejected": -748.0,
      "loss": 0.3645,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 2.109375,
      "rewards/rejected": -5.46875,
      "step": 24110
    },
    {
      "epoch": 0.897805735981091,
      "grad_norm": 10.792540074008876,
      "learning_rate": 1.573420112260884e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.4375,
      "logps/chosen": -540.0,
      "logps/rejected": -688.0,
      "loss": 0.3736,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.96875,
      "step": 24120
    },
    {
      "epoch": 0.8981779605814149,
      "grad_norm": 12.79436359960845,
      "learning_rate": 1.56209778423792e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -564.0,
      "logps/rejected": -728.0,
      "loss": 0.3784,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.53125,
      "rewards/rejected": -5.0625,
      "step": 24130
    },
    {
      "epoch": 0.8985501851817387,
      "grad_norm": 11.285455994467986,
      "learning_rate": 1.550815027274216e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5,
      "logps/chosen": -532.0,
      "logps/rejected": -684.0,
      "loss": 0.3902,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.09375,
      "step": 24140
    },
    {
      "epoch": 0.8989224097820625,
      "grad_norm": 13.833806641168039,
      "learning_rate": 1.5395718604188506e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5,
      "logps/chosen": -576.0,
      "logps/rejected": -708.0,
      "loss": 0.3762,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.515625,
      "rewards/rejected": -5.03125,
      "step": 24150
    },
    {
      "epoch": 0.8992946343823863,
      "grad_norm": 11.679739424490217,
      "learning_rate": 1.528368302654079e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -556.0,
      "logps/rejected": -680.0,
      "loss": 0.3898,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.34375,
      "rewards/rejected": -4.9375,
      "step": 24160
    },
    {
      "epoch": 0.8996668589827101,
      "grad_norm": 11.622964423889034,
      "learning_rate": 1.5172043728952672e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.625,
      "logps/chosen": -580.0,
      "logps/rejected": -716.0,
      "loss": 0.3825,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -5.3125,
      "step": 24170
    },
    {
      "epoch": 0.900039083583034,
      "grad_norm": 12.548451384498948,
      "learning_rate": 1.506080089990888e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.6875,
      "logps/chosen": -564.0,
      "logps/rejected": -704.0,
      "loss": 0.3888,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.78125,
      "rewards/rejected": -5.125,
      "step": 24180
    },
    {
      "epoch": 0.9004113081833578,
      "grad_norm": 11.141462937601174,
      "learning_rate": 1.4949954727224627e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.390625,
      "logps/chosen": -536.0,
      "logps/rejected": -672.0,
      "loss": 0.3849,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.84375,
      "step": 24190
    },
    {
      "epoch": 0.9007835327836817,
      "grad_norm": 12.09495014484346,
      "learning_rate": 1.48395053980456e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.484375,
      "logps/chosen": -584.0,
      "logps/rejected": -728.0,
      "loss": 0.3923,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.6875,
      "rewards/rejected": -5.375,
      "step": 24200
    },
    {
      "epoch": 0.9011557573840056,
      "grad_norm": 10.860106283439052,
      "learning_rate": 1.472945309884735e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.65625,
      "logps/chosen": -544.0,
      "logps/rejected": -668.0,
      "loss": 0.3882,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -4.84375,
      "step": 24210
    },
    {
      "epoch": 0.9015279819843294,
      "grad_norm": 11.216059879079749,
      "learning_rate": 1.4619798015435198e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -560.0,
      "logps/rejected": -712.0,
      "loss": 0.4042,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -5.1875,
      "step": 24220
    },
    {
      "epoch": 0.9019002065846532,
      "grad_norm": 12.037075236205117,
      "learning_rate": 1.4510540332943799e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -576.0,
      "logps/rejected": -744.0,
      "loss": 0.352,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.84375,
      "rewards/rejected": -5.40625,
      "step": 24230
    },
    {
      "epoch": 0.902272431184977,
      "grad_norm": 14.101092990523979,
      "learning_rate": 1.4401680235836766e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.65625,
      "logps/chosen": -540.0,
      "logps/rejected": -716.0,
      "loss": 0.3779,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.859375,
      "rewards/rejected": -5.28125,
      "step": 24240
    },
    {
      "epoch": 0.9026446557853008,
      "grad_norm": 10.774507507874835,
      "learning_rate": 1.4293217907906663e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.53125,
      "logps/chosen": -560.0,
      "logps/rejected": -696.0,
      "loss": 0.3689,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.46875,
      "rewards/rejected": -5.125,
      "step": 24250
    },
    {
      "epoch": 0.9030168803856247,
      "grad_norm": 12.720722994901381,
      "learning_rate": 1.4185153532274313e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.609375,
      "logps/chosen": -548.0,
      "logps/rejected": -700.0,
      "loss": 0.3784,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.6875,
      "rewards/rejected": -5.1875,
      "step": 24260
    },
    {
      "epoch": 0.9033891049859485,
      "grad_norm": 10.509117688638907,
      "learning_rate": 1.4077487291388746e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.515625,
      "logps/chosen": -548.0,
      "logps/rejected": -712.0,
      "loss": 0.3749,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.1875,
      "step": 24270
    },
    {
      "epoch": 0.9037613295862723,
      "grad_norm": 12.319005315743953,
      "learning_rate": 1.3970219367026692e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.453125,
      "logps/chosen": -584.0,
      "logps/rejected": -748.0,
      "loss": 0.3869,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.8046875,
      "rewards/rejected": -5.34375,
      "step": 24280
    },
    {
      "epoch": 0.9041335541865962,
      "grad_norm": 13.379809985669096,
      "learning_rate": 1.3863349940292562e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5625,
      "logps/chosen": -548.0,
      "logps/rejected": -708.0,
      "loss": 0.3559,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.1875,
      "step": 24290
    },
    {
      "epoch": 0.9045057787869201,
      "grad_norm": 13.3334605040839,
      "learning_rate": 1.3756879191617831e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -572.0,
      "logps/rejected": -712.0,
      "loss": 0.3848,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -5.21875,
      "step": 24300
    },
    {
      "epoch": 0.9048780033872439,
      "grad_norm": 12.287515028829775,
      "learning_rate": 1.365080730076093e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.484375,
      "logps/chosen": -552.0,
      "logps/rejected": -732.0,
      "loss": 0.3751,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.3125,
      "step": 24310
    },
    {
      "epoch": 0.9052502279875677,
      "grad_norm": 11.017488045671758,
      "learning_rate": 1.3545134446806911e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.703125,
      "logps/chosen": -548.0,
      "logps/rejected": -712.0,
      "loss": 0.393,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.6875,
      "rewards/rejected": -5.25,
      "step": 24320
    },
    {
      "epoch": 0.9056224525878915,
      "grad_norm": 11.690607452709163,
      "learning_rate": 1.343986080816703e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -532.0,
      "logps/rejected": -688.0,
      "loss": 0.3801,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.75,
      "rewards/rejected": -5.0625,
      "step": 24330
    },
    {
      "epoch": 0.9059946771882154,
      "grad_norm": 12.544088569573866,
      "learning_rate": 1.3334986562578614e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -592.0,
      "logps/rejected": -704.0,
      "loss": 0.3659,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.90625,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -5.28125,
      "step": 24340
    },
    {
      "epoch": 0.9063669017885392,
      "grad_norm": 12.501074965126717,
      "learning_rate": 1.3230511887104639e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.65625,
      "logps/chosen": -552.0,
      "logps/rejected": -692.0,
      "loss": 0.3703,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.515625,
      "rewards/rejected": -4.9375,
      "step": 24350
    },
    {
      "epoch": 0.906739126388863,
      "grad_norm": 12.009260219792251,
      "learning_rate": 1.3126436958133507e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.421875,
      "logps/chosen": -536.0,
      "logps/rejected": -724.0,
      "loss": 0.3827,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.921875,
      "rewards/rejected": -5.28125,
      "step": 24360
    },
    {
      "epoch": 0.9071113509891868,
      "grad_norm": 11.456170689704978,
      "learning_rate": 1.3022761951378663e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.65625,
      "logps/chosen": -564.0,
      "logps/rejected": -720.0,
      "loss": 0.3549,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.28125,
      "step": 24370
    },
    {
      "epoch": 0.9074835755895108,
      "grad_norm": 13.530467007287132,
      "learning_rate": 1.2919487041878369e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.671875,
      "logps/chosen": -560.0,
      "logps/rejected": -680.0,
      "loss": 0.391,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.421875,
      "rewards/rejected": -4.9375,
      "step": 24380
    },
    {
      "epoch": 0.9078558001898346,
      "grad_norm": 11.592703894968546,
      "learning_rate": 1.2816612403995397e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -552.0,
      "logps/rejected": -688.0,
      "loss": 0.3809,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -5.09375,
      "step": 24390
    },
    {
      "epoch": 0.9082280247901584,
      "grad_norm": 12.115795980164735,
      "learning_rate": 1.2714138211416758e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.671875,
      "logps/chosen": -568.0,
      "logps/rejected": -700.0,
      "loss": 0.3754,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.15625,
      "step": 24400
    },
    {
      "epoch": 0.9086002493904822,
      "grad_norm": 11.826110604908537,
      "learning_rate": 1.2612064637153336e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.46875,
      "logps/chosen": -560.0,
      "logps/rejected": -700.0,
      "loss": 0.3879,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.15625,
      "step": 24410
    },
    {
      "epoch": 0.908972473990806,
      "grad_norm": 10.754130186557516,
      "learning_rate": 1.251039185353961e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.625,
      "logps/chosen": -588.0,
      "logps/rejected": -740.0,
      "loss": 0.3976,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.375,
      "step": 24420
    },
    {
      "epoch": 0.9093446985911299,
      "grad_norm": 12.290691251944443,
      "learning_rate": 1.2409120032233433e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.453125,
      "logps/chosen": -572.0,
      "logps/rejected": -732.0,
      "loss": 0.3842,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.1875,
      "step": 24430
    },
    {
      "epoch": 0.9097169231914537,
      "grad_norm": 13.893355158632453,
      "learning_rate": 1.2308249344215704e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.46875,
      "logps/chosen": -572.0,
      "logps/rejected": -724.0,
      "loss": 0.4082,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.796875,
      "rewards/margins": 1.546875,
      "rewards/rejected": -5.34375,
      "step": 24440
    },
    {
      "epoch": 0.9100891477917775,
      "grad_norm": 13.297131330050238,
      "learning_rate": 1.220777995979011e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.515625,
      "logps/chosen": -588.0,
      "logps/rejected": -748.0,
      "loss": 0.3889,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.6875,
      "rewards/rejected": -5.4375,
      "step": 24450
    },
    {
      "epoch": 0.9104613723921013,
      "grad_norm": 11.79293571484281,
      "learning_rate": 1.210771204858263e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -560.0,
      "logps/rejected": -728.0,
      "loss": 0.3621,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.25,
      "step": 24460
    },
    {
      "epoch": 0.9108335969924253,
      "grad_norm": 12.496963569111871,
      "learning_rate": 1.200804577954162e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -592.0,
      "logps/rejected": -744.0,
      "loss": 0.3858,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.765625,
      "rewards/margins": 1.78125,
      "rewards/rejected": -5.5625,
      "step": 24470
    },
    {
      "epoch": 0.9112058215927491,
      "grad_norm": 9.902335511364711,
      "learning_rate": 1.1908781320937256e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.71875,
      "logps/chosen": -568.0,
      "logps/rejected": -716.0,
      "loss": 0.3827,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.1875,
      "step": 24480
    },
    {
      "epoch": 0.9115780461930729,
      "grad_norm": 12.493851549169445,
      "learning_rate": 1.1809918840361282e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.4375,
      "logps/chosen": -560.0,
      "logps/rejected": -712.0,
      "loss": 0.3824,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.1875,
      "step": 24490
    },
    {
      "epoch": 0.9119502707933967,
      "grad_norm": 10.131703101197418,
      "learning_rate": 1.171145850472688e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -572.0,
      "logps/rejected": -712.0,
      "loss": 0.3779,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.25,
      "step": 24500
    },
    {
      "epoch": 0.9123224953937206,
      "grad_norm": 12.936368761226527,
      "learning_rate": 1.16134004802681e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.484375,
      "logps/chosen": -576.0,
      "logps/rejected": -764.0,
      "loss": 0.3604,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.8828125,
      "rewards/rejected": -5.5,
      "step": 24510
    },
    {
      "epoch": 0.9126947199940444,
      "grad_norm": 10.258089943482558,
      "learning_rate": 1.1515744932539929e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.625,
      "logps/chosen": -616.0,
      "logps/rejected": -732.0,
      "loss": 0.3598,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.859375,
      "rewards/margins": 1.5625,
      "rewards/rejected": -5.40625,
      "step": 24520
    },
    {
      "epoch": 0.9130669445943682,
      "grad_norm": 15.785881542453991,
      "learning_rate": 1.1418492026417703e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.5,
      "logps/chosen": -580.0,
      "logps/rejected": -736.0,
      "loss": 0.3985,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.375,
      "step": 24530
    },
    {
      "epoch": 0.913439169194692,
      "grad_norm": 15.024352881060366,
      "learning_rate": 1.1321641926097137e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.609375,
      "logps/chosen": -580.0,
      "logps/rejected": -708.0,
      "loss": 0.3577,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.78125,
      "rewards/margins": 1.453125,
      "rewards/rejected": -5.21875,
      "step": 24540
    },
    {
      "epoch": 0.9138113937950159,
      "grad_norm": 13.293117505490802,
      "learning_rate": 1.1225194795093623e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.671875,
      "logps/chosen": -544.0,
      "logps/rejected": -696.0,
      "loss": 0.3941,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.15625,
      "step": 24550
    },
    {
      "epoch": 0.9141836183953398,
      "grad_norm": 12.768100015710077,
      "learning_rate": 1.1129150796242436e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.546875,
      "logps/chosen": -560.0,
      "logps/rejected": -720.0,
      "loss": 0.3658,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.84375,
      "rewards/rejected": -5.34375,
      "step": 24560
    },
    {
      "epoch": 0.9145558429956636,
      "grad_norm": 12.47326405777344,
      "learning_rate": 1.1033510091698117e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -552.0,
      "logps/rejected": -704.0,
      "loss": 0.3827,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.1875,
      "step": 24570
    },
    {
      "epoch": 0.9149280675959874,
      "grad_norm": 12.772033855597655,
      "learning_rate": 1.0938272842934304e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.59375,
      "logps/chosen": -560.0,
      "logps/rejected": -720.0,
      "loss": 0.3772,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.8125,
      "rewards/rejected": -5.3125,
      "step": 24580
    },
    {
      "epoch": 0.9153002921963113,
      "grad_norm": 13.459409343618143,
      "learning_rate": 1.0843439210743572e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5,
      "logps/chosen": -568.0,
      "logps/rejected": -724.0,
      "loss": 0.3961,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -5.28125,
      "step": 24590
    },
    {
      "epoch": 0.9156725167966351,
      "grad_norm": 13.167030385558137,
      "learning_rate": 1.0749009355236898e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.640625,
      "logps/chosen": -564.0,
      "logps/rejected": -704.0,
      "loss": 0.3921,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.15625,
      "step": 24600
    },
    {
      "epoch": 0.9160447413969589,
      "grad_norm": 12.840150324638762,
      "learning_rate": 1.0654983435843645e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -556.0,
      "logps/rejected": -692.0,
      "loss": 0.3903,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -5.03125,
      "step": 24610
    },
    {
      "epoch": 0.9164169659972827,
      "grad_norm": 16.487347026626985,
      "learning_rate": 1.0561361611311188e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.546875,
      "logps/chosen": -544.0,
      "logps/rejected": -724.0,
      "loss": 0.3713,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.53125,
      "rewards/margins": 2.015625,
      "rewards/rejected": -5.5625,
      "step": 24620
    },
    {
      "epoch": 0.9167891905976066,
      "grad_norm": 12.979801227946064,
      "learning_rate": 1.046814403970464e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.78125,
      "logps/chosen": -596.0,
      "logps/rejected": -732.0,
      "loss": 0.3831,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.734375,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.40625,
      "step": 24630
    },
    {
      "epoch": 0.9171614151979305,
      "grad_norm": 13.103637325273112,
      "learning_rate": 1.037533087840664e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.65625,
      "logps/chosen": -584.0,
      "logps/rejected": -740.0,
      "loss": 0.4011,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.734375,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.34375,
      "step": 24640
    },
    {
      "epoch": 0.9175336397982543,
      "grad_norm": 13.705156859701516,
      "learning_rate": 1.0282922284116951e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.546875,
      "logps/chosen": -588.0,
      "logps/rejected": -728.0,
      "loss": 0.3843,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.78125,
      "rewards/margins": 1.5625,
      "rewards/rejected": -5.34375,
      "step": 24650
    },
    {
      "epoch": 0.9179058643985781,
      "grad_norm": 13.765040756617752,
      "learning_rate": 1.0190918412852384e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -544.0,
      "logps/rejected": -724.0,
      "loss": 0.3927,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.328125,
      "rewards/margins": 2.015625,
      "rewards/rejected": -5.34375,
      "step": 24660
    },
    {
      "epoch": 0.918278088998902,
      "grad_norm": 13.405198084472092,
      "learning_rate": 1.0099319419946406e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5625,
      "logps/chosen": -552.0,
      "logps/rejected": -736.0,
      "loss": 0.382,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.9609375,
      "rewards/rejected": -5.4375,
      "step": 24670
    },
    {
      "epoch": 0.9186503135992258,
      "grad_norm": 14.132759009098232,
      "learning_rate": 1.0008125460048921e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.546875,
      "logps/chosen": -576.0,
      "logps/rejected": -736.0,
      "loss": 0.3793,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.375,
      "step": 24680
    },
    {
      "epoch": 0.9190225381995496,
      "grad_norm": 12.186742686099032,
      "learning_rate": 9.917336687125938e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.625,
      "logps/chosen": -580.0,
      "logps/rejected": -724.0,
      "loss": 0.3766,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.1875,
      "step": 24690
    },
    {
      "epoch": 0.9193947627998734,
      "grad_norm": 9.450952160487828,
      "learning_rate": 9.826953254459508e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.625,
      "logps/chosen": -592.0,
      "logps/rejected": -736.0,
      "loss": 0.3776,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.78125,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -5.3125,
      "step": 24700
    },
    {
      "epoch": 0.9197669874001972,
      "grad_norm": 11.321042520997064,
      "learning_rate": 9.736975314647205e-09,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.59375,
      "logps/chosen": -544.0,
      "logps/rejected": -704.0,
      "loss": 0.384,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.15625,
      "step": 24710
    },
    {
      "epoch": 0.9201392120005211,
      "grad_norm": 11.205700916927768,
      "learning_rate": 9.647403019602068e-09,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -584.0,
      "logps/rejected": -704.0,
      "loss": 0.3934,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -5.21875,
      "step": 24720
    },
    {
      "epoch": 0.920511436600845,
      "grad_norm": 13.462854458943259,
      "learning_rate": 9.558236520552265e-09,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.734375,
      "logps/chosen": -580.0,
      "logps/rejected": -736.0,
      "loss": 0.3869,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.34375,
      "step": 24730
    },
    {
      "epoch": 0.9208836612011688,
      "grad_norm": 11.700893530221167,
      "learning_rate": 9.469475968040763e-09,
      "logits/chosen": -2.75,
      "logits/rejected": -2.6875,
      "logps/chosen": -584.0,
      "logps/rejected": -728.0,
      "loss": 0.3793,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.375,
      "step": 24740
    },
    {
      "epoch": 0.9212558858014926,
      "grad_norm": 15.174681284952724,
      "learning_rate": 9.381121511925243e-09,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.484375,
      "logps/chosen": -616.0,
      "logps/rejected": -764.0,
      "loss": 0.4011,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.828125,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -5.3125,
      "step": 24750
    },
    {
      "epoch": 0.9216281104018165,
      "grad_norm": 14.237287905362702,
      "learning_rate": 9.293173301377772e-09,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.796875,
      "logps/chosen": -584.0,
      "logps/rejected": -748.0,
      "loss": 0.3986,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.34375,
      "step": 24760
    },
    {
      "epoch": 0.9220003350021403,
      "grad_norm": 14.16570756866144,
      "learning_rate": 9.205631484884485e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -564.0,
      "logps/rejected": -724.0,
      "loss": 0.3824,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.40625,
      "step": 24770
    },
    {
      "epoch": 0.9223725596024641,
      "grad_norm": 13.15871707086385,
      "learning_rate": 9.11849621024538e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.4375,
      "logps/chosen": -592.0,
      "logps/rejected": -736.0,
      "loss": 0.3798,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.4375,
      "step": 24780
    },
    {
      "epoch": 0.9227447842027879,
      "grad_norm": 13.485072419520355,
      "learning_rate": 9.031767624574083e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.546875,
      "logps/chosen": -584.0,
      "logps/rejected": -732.0,
      "loss": 0.3755,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.375,
      "step": 24790
    },
    {
      "epoch": 0.9231170088031118,
      "grad_norm": 13.204177850335839,
      "learning_rate": 8.945445874297659e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -576.0,
      "logps/rejected": -736.0,
      "loss": 0.3829,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.734375,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -5.15625,
      "step": 24800
    },
    {
      "epoch": 0.9234892334034356,
      "grad_norm": 12.15403725419121,
      "learning_rate": 8.859531105156193e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.734375,
      "logps/chosen": -584.0,
      "logps/rejected": -712.0,
      "loss": 0.3618,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -5.0625,
      "step": 24810
    },
    {
      "epoch": 0.9238614580037595,
      "grad_norm": 12.309901094489803,
      "learning_rate": 8.774023462202768e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -600.0,
      "logps/rejected": -748.0,
      "loss": 0.3921,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.859375,
      "rewards/rejected": -5.5,
      "step": 24820
    },
    {
      "epoch": 0.9242336826040833,
      "grad_norm": 12.226164044109598,
      "learning_rate": 8.688923089802957e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.609375,
      "logps/chosen": -564.0,
      "logps/rejected": -720.0,
      "loss": 0.3823,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.3125,
      "step": 24830
    },
    {
      "epoch": 0.9246059072044072,
      "grad_norm": 11.372456589440237,
      "learning_rate": 8.604230131634832e-09,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.640625,
      "logps/chosen": -596.0,
      "logps/rejected": -748.0,
      "loss": 0.3784,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.828125,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.53125,
      "step": 24840
    },
    {
      "epoch": 0.924978131804731,
      "grad_norm": 12.05173108835727,
      "learning_rate": 8.519944730688594e-09,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.5,
      "logps/chosen": -588.0,
      "logps/rejected": -700.0,
      "loss": 0.3823,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -5.125,
      "step": 24850
    },
    {
      "epoch": 0.9253503564050548,
      "grad_norm": 15.924184702622446,
      "learning_rate": 8.436067029266358e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.609375,
      "logps/chosen": -580.0,
      "logps/rejected": -708.0,
      "loss": 0.401,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -5.25,
      "step": 24860
    },
    {
      "epoch": 0.9257225810053786,
      "grad_norm": 13.317671273314275,
      "learning_rate": 8.352597168981845e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -580.0,
      "logps/rejected": -712.0,
      "loss": 0.3727,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.5,
      "rewards/rejected": -5.21875,
      "step": 24870
    },
    {
      "epoch": 0.9260948056057025,
      "grad_norm": 11.963653533324406,
      "learning_rate": 8.269535290760271e-09,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.484375,
      "logps/chosen": -564.0,
      "logps/rejected": -744.0,
      "loss": 0.3577,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.8828125,
      "rewards/rejected": -5.5,
      "step": 24880
    },
    {
      "epoch": 0.9264670302060263,
      "grad_norm": 10.48018695031447,
      "learning_rate": 8.18688153483793e-09,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.421875,
      "logps/chosen": -572.0,
      "logps/rejected": -708.0,
      "loss": 0.3703,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.09375,
      "step": 24890
    },
    {
      "epoch": 0.9268392548063501,
      "grad_norm": 12.77090944668158,
      "learning_rate": 8.104636040762281e-09,
      "logits/chosen": -2.75,
      "logits/rejected": -2.59375,
      "logps/chosen": -584.0,
      "logps/rejected": -756.0,
      "loss": 0.3906,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -5.46875,
      "step": 24900
    },
    {
      "epoch": 0.927211479406674,
      "grad_norm": 12.623392678521288,
      "learning_rate": 8.022798947391302e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.453125,
      "logps/chosen": -592.0,
      "logps/rejected": -736.0,
      "loss": 0.3766,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.828125,
      "rewards/margins": 1.53125,
      "rewards/rejected": -5.375,
      "step": 24910
    },
    {
      "epoch": 0.9275837040069979,
      "grad_norm": 14.190743977086743,
      "learning_rate": 7.941370392893526e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.65625,
      "logps/chosen": -588.0,
      "logps/rejected": -724.0,
      "loss": 0.3795,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.6875,
      "rewards/rejected": -5.21875,
      "step": 24920
    },
    {
      "epoch": 0.9279559286073217,
      "grad_norm": 11.601957711152336,
      "learning_rate": 7.86035051474776e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.421875,
      "logps/chosen": -544.0,
      "logps/rejected": -668.0,
      "loss": 0.3798,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.546875,
      "rewards/rejected": -4.90625,
      "step": 24930
    },
    {
      "epoch": 0.9283281532076455,
      "grad_norm": 10.843845998323117,
      "learning_rate": 7.779739449742723e-09,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.703125,
      "logps/chosen": -604.0,
      "logps/rejected": -760.0,
      "loss": 0.3833,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.8046875,
      "rewards/rejected": -5.53125,
      "step": 24940
    },
    {
      "epoch": 0.9287003778079693,
      "grad_norm": 11.882512178411348,
      "learning_rate": 7.699537333977101e-09,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.546875,
      "logps/chosen": -580.0,
      "logps/rejected": -724.0,
      "loss": 0.3923,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.28125,
      "step": 24950
    },
    {
      "epoch": 0.9290726024082931,
      "grad_norm": 11.142629839529942,
      "learning_rate": 7.619744302858944e-09,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5,
      "logps/chosen": -552.0,
      "logps/rejected": -700.0,
      "loss": 0.3612,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.0625,
      "step": 24960
    },
    {
      "epoch": 0.929444827008617,
      "grad_norm": 16.411679996746173,
      "learning_rate": 7.540360491105763e-09,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.484375,
      "logps/chosen": -548.0,
      "logps/rejected": -684.0,
      "loss": 0.4322,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -4.875,
      "step": 24970
    },
    {
      "epoch": 0.9298170516089408,
      "grad_norm": 13.717116966590911,
      "learning_rate": 7.461386032744099e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.609375,
      "logps/chosen": -560.0,
      "logps/rejected": -724.0,
      "loss": 0.3919,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.21875,
      "step": 24980
    },
    {
      "epoch": 0.9301892762092646,
      "grad_norm": 12.320551800137768,
      "learning_rate": 7.382821061109407e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.421875,
      "logps/chosen": -548.0,
      "logps/rejected": -704.0,
      "loss": 0.3807,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.125,
      "step": 24990
    },
    {
      "epoch": 0.9305615008095885,
      "grad_norm": 10.775998154917199,
      "learning_rate": 7.304665708845803e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -572.0,
      "logps/rejected": -720.0,
      "loss": 0.3734,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.3125,
      "step": 25000
    },
    {
      "epoch": 0.9309337254099124,
      "grad_norm": 11.70920238281295,
      "learning_rate": 7.2269201079057375e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.484375,
      "logps/chosen": -572.0,
      "logps/rejected": -740.0,
      "loss": 0.378,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.78125,
      "rewards/rejected": -5.46875,
      "step": 25010
    },
    {
      "epoch": 0.9313059500102362,
      "grad_norm": 12.111459136645236,
      "learning_rate": 7.149584389549962e-09,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.625,
      "logps/chosen": -572.0,
      "logps/rejected": -728.0,
      "loss": 0.3873,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.3125,
      "step": 25020
    },
    {
      "epoch": 0.93167817461056,
      "grad_norm": 13.065522526389362,
      "learning_rate": 7.072658684347116e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.421875,
      "logps/chosen": -588.0,
      "logps/rejected": -728.0,
      "loss": 0.3867,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.375,
      "step": 25030
    },
    {
      "epoch": 0.9320503992108838,
      "grad_norm": 10.525551349945887,
      "learning_rate": 6.996143122173753e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.515625,
      "logps/chosen": -552.0,
      "logps/rejected": -728.0,
      "loss": 0.3638,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -5.3125,
      "step": 25040
    },
    {
      "epoch": 0.9324226238112077,
      "grad_norm": 12.534140962147099,
      "learning_rate": 6.920037832213788e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.671875,
      "logps/chosen": -584.0,
      "logps/rejected": -708.0,
      "loss": 0.3821,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.734375,
      "rewards/margins": 1.5625,
      "rewards/rejected": -5.3125,
      "step": 25050
    },
    {
      "epoch": 0.9327948484115315,
      "grad_norm": 17.42261288145803,
      "learning_rate": 6.8443429429585755e-09,
      "logits/chosen": -2.75,
      "logits/rejected": -2.546875,
      "logps/chosen": -560.0,
      "logps/rejected": -736.0,
      "loss": 0.3812,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.8828125,
      "rewards/rejected": -5.34375,
      "step": 25060
    },
    {
      "epoch": 0.9331670730118553,
      "grad_norm": 10.901186931691706,
      "learning_rate": 6.769058582206555e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -596.0,
      "logps/rejected": -760.0,
      "loss": 0.3726,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.890625,
      "rewards/rejected": -5.5625,
      "step": 25070
    },
    {
      "epoch": 0.9335392976121792,
      "grad_norm": 15.231260664999386,
      "learning_rate": 6.694184877062941e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -556.0,
      "logps/rejected": -724.0,
      "loss": 0.3934,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.40625,
      "step": 25080
    },
    {
      "epoch": 0.9339115222125031,
      "grad_norm": 11.342514890048706,
      "learning_rate": 6.619721953939866e-09,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.578125,
      "logps/chosen": -620.0,
      "logps/rejected": -748.0,
      "loss": 0.3999,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -4.09375,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -5.46875,
      "step": 25090
    },
    {
      "epoch": 0.9342837468128269,
      "grad_norm": 13.624199203669408,
      "learning_rate": 6.545669938555654e-09,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.484375,
      "logps/chosen": -560.0,
      "logps/rejected": -708.0,
      "loss": 0.3764,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.21875,
      "step": 25100
    },
    {
      "epoch": 0.9346559714131507,
      "grad_norm": 11.78467079545182,
      "learning_rate": 6.4720289559351e-09,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.65625,
      "logps/chosen": -612.0,
      "logps/rejected": -744.0,
      "loss": 0.3771,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.796875,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.40625,
      "step": 25110
    },
    {
      "epoch": 0.9350281960134745,
      "grad_norm": 11.686711027088293,
      "learning_rate": 6.3987991304088885e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -568.0,
      "logps/rejected": -728.0,
      "loss": 0.3484,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.34375,
      "step": 25120
    },
    {
      "epoch": 0.9354004206137984,
      "grad_norm": 12.555816251455111,
      "learning_rate": 6.325980585613594e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -576.0,
      "logps/rejected": -732.0,
      "loss": 0.3719,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.4375,
      "step": 25130
    },
    {
      "epoch": 0.9357726452141222,
      "grad_norm": 12.10040500547715,
      "learning_rate": 6.2535734444914004e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -576.0,
      "logps/rejected": -740.0,
      "loss": 0.3756,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.875,
      "rewards/rejected": -5.40625,
      "step": 25140
    },
    {
      "epoch": 0.936144869814446,
      "grad_norm": 13.27234277811694,
      "learning_rate": 6.181577829289936e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.609375,
      "logps/chosen": -600.0,
      "logps/rejected": -732.0,
      "loss": 0.3912,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.796875,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -5.28125,
      "step": 25150
    },
    {
      "epoch": 0.9365170944147698,
      "grad_norm": 14.455986309810134,
      "learning_rate": 6.109993861561969e-09,
      "logits/chosen": -2.75,
      "logits/rejected": -2.65625,
      "logps/chosen": -604.0,
      "logps/rejected": -756.0,
      "loss": 0.3962,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.828125,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -5.5625,
      "step": 25160
    },
    {
      "epoch": 0.9368893190150938,
      "grad_norm": 10.359137855957051,
      "learning_rate": 6.0388216621652945e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -584.0,
      "logps/rejected": -740.0,
      "loss": 0.385,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.375,
      "step": 25170
    },
    {
      "epoch": 0.9372615436154176,
      "grad_norm": 12.90975627288972,
      "learning_rate": 5.968061351262599e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -552.0,
      "logps/rejected": -676.0,
      "loss": 0.3639,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -4.90625,
      "step": 25180
    },
    {
      "epoch": 0.9376337682157414,
      "grad_norm": 13.020702172734332,
      "learning_rate": 5.897713048320985e-09,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.59375,
      "logps/chosen": -548.0,
      "logps/rejected": -704.0,
      "loss": 0.4018,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -5.3125,
      "step": 25190
    },
    {
      "epoch": 0.9380059928160652,
      "grad_norm": 10.966938845132187,
      "learning_rate": 5.827776872112111e-09,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.453125,
      "logps/chosen": -544.0,
      "logps/rejected": -712.0,
      "loss": 0.3843,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.21875,
      "step": 25200
    },
    {
      "epoch": 0.938378217416389,
      "grad_norm": 11.301321936443914,
      "learning_rate": 5.758252940711666e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -560.0,
      "logps/rejected": -712.0,
      "loss": 0.3655,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.21875,
      "step": 25210
    },
    {
      "epoch": 0.9387504420167129,
      "grad_norm": 11.461107763809864,
      "learning_rate": 5.689141371499479e-09,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.53125,
      "logps/chosen": -592.0,
      "logps/rejected": -716.0,
      "loss": 0.3962,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -5.28125,
      "step": 25220
    },
    {
      "epoch": 0.9391226666170367,
      "grad_norm": 10.686891271321368,
      "learning_rate": 5.6204422811591546e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -544.0,
      "logps/rejected": -696.0,
      "loss": 0.3545,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.0625,
      "step": 25230
    },
    {
      "epoch": 0.9394948912173605,
      "grad_norm": 11.941267345094671,
      "learning_rate": 5.5521557856778035e-09,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5625,
      "logps/chosen": -588.0,
      "logps/rejected": -728.0,
      "loss": 0.393,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.375,
      "step": 25240
    },
    {
      "epoch": 0.9398671158176843,
      "grad_norm": 12.331413646653864,
      "learning_rate": 5.484282000346007e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.4375,
      "logps/chosen": -532.0,
      "logps/rejected": -688.0,
      "loss": 0.3809,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.0,
      "step": 25250
    },
    {
      "epoch": 0.9402393404180083,
      "grad_norm": 13.236494470572495,
      "learning_rate": 5.416821039757491e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.671875,
      "logps/chosen": -632.0,
      "logps/rejected": -756.0,
      "loss": 0.3762,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -4.03125,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -5.5,
      "step": 25260
    },
    {
      "epoch": 0.9406115650183321,
      "grad_norm": 10.19144004696015,
      "learning_rate": 5.34977301780909e-09,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.671875,
      "logps/chosen": -544.0,
      "logps/rejected": -676.0,
      "loss": 0.3925,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -4.96875,
      "step": 25270
    },
    {
      "epoch": 0.9409837896186559,
      "grad_norm": 11.359938671648765,
      "learning_rate": 5.283138047700392e-09,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.359375,
      "logps/chosen": -548.0,
      "logps/rejected": -696.0,
      "loss": 0.3695,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.0625,
      "step": 25280
    },
    {
      "epoch": 0.9413560142189797,
      "grad_norm": 13.740770761472318,
      "learning_rate": 5.216916241933683e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -576.0,
      "logps/rejected": -764.0,
      "loss": 0.3902,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -5.5625,
      "step": 25290
    },
    {
      "epoch": 0.9417282388193036,
      "grad_norm": 14.753588608142373,
      "learning_rate": 5.151107712313529e-09,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.484375,
      "logps/chosen": -564.0,
      "logps/rejected": -732.0,
      "loss": 0.3775,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.3125,
      "step": 25300
    },
    {
      "epoch": 0.9421004634196274,
      "grad_norm": 13.009956244343648,
      "learning_rate": 5.085712569946915e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.421875,
      "logps/chosen": -568.0,
      "logps/rejected": -724.0,
      "loss": 0.408,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.1875,
      "step": 25310
    },
    {
      "epoch": 0.9424726880199512,
      "grad_norm": 11.759603215476957,
      "learning_rate": 5.020730925242828e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.453125,
      "logps/chosen": -576.0,
      "logps/rejected": -724.0,
      "loss": 0.4031,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -5.21875,
      "step": 25320
    },
    {
      "epoch": 0.942844912620275,
      "grad_norm": 11.611033786472877,
      "learning_rate": 4.956162887912146e-09,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.578125,
      "logps/chosen": -568.0,
      "logps/rejected": -740.0,
      "loss": 0.3822,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.25,
      "step": 25330
    },
    {
      "epoch": 0.9432171372205989,
      "grad_norm": 13.14082739706073,
      "learning_rate": 4.89200856696742e-09,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.65625,
      "logps/chosen": -552.0,
      "logps/rejected": -720.0,
      "loss": 0.4195,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.546875,
      "rewards/rejected": -5.15625,
      "step": 25340
    },
    {
      "epoch": 0.9435893618209228,
      "grad_norm": 13.703929661157632,
      "learning_rate": 4.828268070722674e-09,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.515625,
      "logps/chosen": -588.0,
      "logps/rejected": -764.0,
      "loss": 0.3839,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.796875,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -5.625,
      "step": 25350
    },
    {
      "epoch": 0.9439615864212466,
      "grad_norm": 13.835581355816108,
      "learning_rate": 4.764941506793324e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -556.0,
      "logps/rejected": -724.0,
      "loss": 0.3767,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.921875,
      "rewards/rejected": -5.375,
      "step": 25360
    },
    {
      "epoch": 0.9443338110215704,
      "grad_norm": 11.87422615209611,
      "learning_rate": 4.702028982095901e-09,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.5625,
      "logps/chosen": -536.0,
      "logps/rejected": -680.0,
      "loss": 0.392,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.0,
      "step": 25370
    },
    {
      "epoch": 0.9447060356218943,
      "grad_norm": 13.53480464196373,
      "learning_rate": 4.639530602847913e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -564.0,
      "logps/rejected": -720.0,
      "loss": 0.3972,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.59375,
      "rewards/rejected": -5.15625,
      "step": 25380
    },
    {
      "epoch": 0.9450782602222181,
      "grad_norm": 11.785558120501342,
      "learning_rate": 4.57744647456762e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.453125,
      "logps/chosen": -568.0,
      "logps/rejected": -700.0,
      "loss": 0.3748,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.15625,
      "step": 25390
    },
    {
      "epoch": 0.9454504848225419,
      "grad_norm": 12.837742348624172,
      "learning_rate": 4.515776702073926e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -580.0,
      "logps/rejected": -744.0,
      "loss": 0.3915,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.5,
      "step": 25400
    },
    {
      "epoch": 0.9458227094228657,
      "grad_norm": 12.15250621780422,
      "learning_rate": 4.454521389486099e-09,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.515625,
      "logps/chosen": -572.0,
      "logps/rejected": -700.0,
      "loss": 0.3856,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.453125,
      "rewards/rejected": -5.03125,
      "step": 25410
    },
    {
      "epoch": 0.9461949340231895,
      "grad_norm": 13.676310448302836,
      "learning_rate": 4.393680640223718e-09,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.65625,
      "logps/chosen": -568.0,
      "logps/rejected": -716.0,
      "loss": 0.3796,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -5.3125,
      "step": 25420
    },
    {
      "epoch": 0.9465671586235135,
      "grad_norm": 11.841354913121762,
      "learning_rate": 4.333254557006477e-09,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.3125,
      "logps/chosen": -520.0,
      "logps/rejected": -704.0,
      "loss": 0.3712,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.234375,
      "rewards/margins": 1.9765625,
      "rewards/rejected": -5.21875,
      "step": 25430
    },
    {
      "epoch": 0.9469393832238373,
      "grad_norm": 10.796103886876304,
      "learning_rate": 4.273243241853852e-09,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.359375,
      "logps/chosen": -576.0,
      "logps/rejected": -724.0,
      "loss": 0.3839,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -5.21875,
      "step": 25440
    },
    {
      "epoch": 0.9473116078241611,
      "grad_norm": 13.837690087141059,
      "learning_rate": 4.213646796085157e-09,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.5,
      "logps/chosen": -560.0,
      "logps/rejected": -720.0,
      "loss": 0.3932,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -5.34375,
      "step": 25450
    },
    {
      "epoch": 0.947683832424485,
      "grad_norm": 12.892095780562851,
      "learning_rate": 4.154465320319184e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.65625,
      "logps/chosen": -548.0,
      "logps/rejected": -712.0,
      "loss": 0.3658,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -5.21875,
      "step": 25460
    },
    {
      "epoch": 0.9480560570248088,
      "grad_norm": 11.129010123951195,
      "learning_rate": 4.095698914474255e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.46875,
      "logps/chosen": -552.0,
      "logps/rejected": -732.0,
      "loss": 0.3614,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.828125,
      "rewards/rejected": -5.3125,
      "step": 25470
    },
    {
      "epoch": 0.9484282816251326,
      "grad_norm": 13.834713757948348,
      "learning_rate": 4.037347677767728e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -552.0,
      "logps/rejected": -708.0,
      "loss": 0.3949,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.21875,
      "step": 25480
    },
    {
      "epoch": 0.9488005062254564,
      "grad_norm": 12.65041602463171,
      "learning_rate": 3.97941170871613e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -548.0,
      "logps/rejected": -708.0,
      "loss": 0.366,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -5.34375,
      "step": 25490
    },
    {
      "epoch": 0.9491727308257802,
      "grad_norm": 12.24468120059753,
      "learning_rate": 3.921891105134884e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.703125,
      "logps/chosen": -612.0,
      "logps/rejected": -712.0,
      "loss": 0.3727,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -3.796875,
      "rewards/margins": 1.171875,
      "rewards/rejected": -4.96875,
      "step": 25500
    },
    {
      "epoch": 0.9495449554261041,
      "grad_norm": 12.022663176582723,
      "learning_rate": 3.864785964138112e-09,
      "logits/chosen": -2.5,
      "logits/rejected": -2.484375,
      "logps/chosen": -576.0,
      "logps/rejected": -720.0,
      "loss": 0.3791,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.3125,
      "step": 25510
    },
    {
      "epoch": 0.949917180026428,
      "grad_norm": 13.36119211317538,
      "learning_rate": 3.80809638213847e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.40625,
      "logps/chosen": -556.0,
      "logps/rejected": -700.0,
      "loss": 0.3746,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.25,
      "step": 25520
    },
    {
      "epoch": 0.9502894046267518,
      "grad_norm": 11.454308497781364,
      "learning_rate": 3.751822454847037e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.6875,
      "logps/chosen": -556.0,
      "logps/rejected": -708.0,
      "loss": 0.3637,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.3125,
      "step": 25530
    },
    {
      "epoch": 0.9506616292270756,
      "grad_norm": 11.814789024341586,
      "learning_rate": 3.6959642772730893e-09,
      "logits/chosen": -2.75,
      "logits/rejected": -2.453125,
      "logps/chosen": -576.0,
      "logps/rejected": -716.0,
      "loss": 0.3857,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.3125,
      "step": 25540
    },
    {
      "epoch": 0.9510338538273995,
      "grad_norm": 13.582546910948654,
      "learning_rate": 3.6405219437240785e-09,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5625,
      "logps/chosen": -552.0,
      "logps/rejected": -720.0,
      "loss": 0.3604,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -5.21875,
      "step": 25550
    },
    {
      "epoch": 0.9514060784277233,
      "grad_norm": 12.22562498347341,
      "learning_rate": 3.5854955478052672e-09,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.53125,
      "logps/chosen": -552.0,
      "logps/rejected": -704.0,
      "loss": 0.398,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.09375,
      "step": 25560
    },
    {
      "epoch": 0.9517783030280471,
      "grad_norm": 12.25391190357001,
      "learning_rate": 3.5308851824197296e-09,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.40625,
      "logps/chosen": -540.0,
      "logps/rejected": -672.0,
      "loss": 0.3674,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -5.03125,
      "step": 25570
    },
    {
      "epoch": 0.9521505276283709,
      "grad_norm": 13.679961919356234,
      "learning_rate": 3.476690939768101e-09,
      "logits/chosen": -2.75,
      "logits/rejected": -2.703125,
      "logps/chosen": -592.0,
      "logps/rejected": -716.0,
      "loss": 0.3881,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.3125,
      "step": 25580
    },
    {
      "epoch": 0.9525227522286948,
      "grad_norm": 12.259675973369998,
      "learning_rate": 3.4229129113484968e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.546875,
      "logps/chosen": -544.0,
      "logps/rejected": -696.0,
      "loss": 0.3746,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.1875,
      "step": 25590
    },
    {
      "epoch": 0.9528949768290186,
      "grad_norm": 12.717563660156095,
      "learning_rate": 3.3695511879562877e-09,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.375,
      "logps/chosen": -576.0,
      "logps/rejected": -744.0,
      "loss": 0.3696,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.78125,
      "rewards/rejected": -5.4375,
      "step": 25600
    },
    {
      "epoch": 0.9532672014293425,
      "grad_norm": 15.119082805268805,
      "learning_rate": 3.316605859684074e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -556.0,
      "logps/rejected": -720.0,
      "loss": 0.3911,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.796875,
      "rewards/rejected": -5.21875,
      "step": 25610
    },
    {
      "epoch": 0.9536394260296663,
      "grad_norm": 12.65430589408343,
      "learning_rate": 3.264077015921268e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -556.0,
      "logps/rejected": -712.0,
      "loss": 0.3773,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.09375,
      "step": 25620
    },
    {
      "epoch": 0.9540116506299902,
      "grad_norm": 11.32303986088713,
      "learning_rate": 3.21196474535429e-09,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.46875,
      "logps/chosen": -576.0,
      "logps/rejected": -752.0,
      "loss": 0.3761,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.953125,
      "rewards/rejected": -5.5625,
      "step": 25630
    },
    {
      "epoch": 0.954383875230314,
      "grad_norm": 11.13461811763636,
      "learning_rate": 3.1602691359661758e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -536.0,
      "logps/rejected": -728.0,
      "loss": 0.3756,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.8828125,
      "rewards/rejected": -5.25,
      "step": 25640
    },
    {
      "epoch": 0.9547560998306378,
      "grad_norm": 11.2960503302416,
      "learning_rate": 3.108990275036444e-09,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.515625,
      "logps/chosen": -556.0,
      "logps/rejected": -732.0,
      "loss": 0.3859,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.8828125,
      "rewards/rejected": -5.375,
      "step": 25650
    },
    {
      "epoch": 0.9551283244309616,
      "grad_norm": 13.136630004490987,
      "learning_rate": 3.0581282491411176e-09,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.609375,
      "logps/chosen": -572.0,
      "logps/rejected": -740.0,
      "loss": 0.3793,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.765625,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.46875,
      "step": 25660
    },
    {
      "epoch": 0.9555005490312855,
      "grad_norm": 13.3984503263093,
      "learning_rate": 3.0076831441523677e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -576.0,
      "logps/rejected": -716.0,
      "loss": 0.377,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.375,
      "step": 25670
    },
    {
      "epoch": 0.9558727736316093,
      "grad_norm": 14.495025647278803,
      "learning_rate": 2.957655045238455e-09,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.75,
      "logps/chosen": -564.0,
      "logps/rejected": -692.0,
      "loss": 0.4014,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.40625,
      "rewards/rejected": -5.0,
      "step": 25680
    },
    {
      "epoch": 0.9562449982319331,
      "grad_norm": 14.7148899931445,
      "learning_rate": 2.9080440368636773e-09,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.46875,
      "logps/chosen": -572.0,
      "logps/rejected": -708.0,
      "loss": 0.4035,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -5.1875,
      "step": 25690
    },
    {
      "epoch": 0.956617222832257,
      "grad_norm": 13.540378474198336,
      "learning_rate": 2.8588502027881156e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -536.0,
      "logps/rejected": -680.0,
      "loss": 0.3842,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.59375,
      "rewards/rejected": -4.96875,
      "step": 25700
    },
    {
      "epoch": 0.9569894474325809,
      "grad_norm": 12.915472244440704,
      "learning_rate": 2.8100736260674442e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.609375,
      "logps/chosen": -544.0,
      "logps/rejected": -720.0,
      "loss": 0.3701,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 1.84375,
      "rewards/rejected": -5.15625,
      "step": 25710
    },
    {
      "epoch": 0.9573616720329047,
      "grad_norm": 11.680008926913045,
      "learning_rate": 2.761714389052955e-09,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5,
      "logps/chosen": -580.0,
      "logps/rejected": -708.0,
      "loss": 0.3721,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -5.125,
      "step": 25720
    },
    {
      "epoch": 0.9577338966332285,
      "grad_norm": 12.41366567460214,
      "learning_rate": 2.713772573391282e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -584.0,
      "logps/rejected": -744.0,
      "loss": 0.3769,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.828125,
      "rewards/rejected": -5.40625,
      "step": 25730
    },
    {
      "epoch": 0.9581061212335523,
      "grad_norm": 13.438196214991956,
      "learning_rate": 2.6662482600243452e-09,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.453125,
      "logps/chosen": -548.0,
      "logps/rejected": -668.0,
      "loss": 0.3962,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -4.96875,
      "step": 25740
    },
    {
      "epoch": 0.9584783458338761,
      "grad_norm": 9.309382447794043,
      "learning_rate": 2.619141529189184e-09,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.578125,
      "logps/chosen": -544.0,
      "logps/rejected": -704.0,
      "loss": 0.3793,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.125,
      "step": 25750
    },
    {
      "epoch": 0.9588505704342,
      "grad_norm": 12.684218825978139,
      "learning_rate": 2.572452460417762e-09,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.515625,
      "logps/chosen": -596.0,
      "logps/rejected": -728.0,
      "loss": 0.3851,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.25,
      "step": 25760
    },
    {
      "epoch": 0.9592227950345238,
      "grad_norm": 10.721784585127596,
      "learning_rate": 2.526181132536942e-09,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5625,
      "logps/chosen": -576.0,
      "logps/rejected": -752.0,
      "loss": 0.3595,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.1875,
      "step": 25770
    },
    {
      "epoch": 0.9595950196348476,
      "grad_norm": 12.322435580530525,
      "learning_rate": 2.480327623668288e-09,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.46875,
      "logps/chosen": -584.0,
      "logps/rejected": -740.0,
      "loss": 0.3691,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -5.34375,
      "step": 25780
    },
    {
      "epoch": 0.9599672442351715,
      "grad_norm": 12.612358237258011,
      "learning_rate": 2.434892011227929e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -588.0,
      "logps/rejected": -728.0,
      "loss": 0.3695,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.890625,
      "rewards/rejected": -5.4375,
      "step": 25790
    },
    {
      "epoch": 0.9603394688354954,
      "grad_norm": 13.951673858954178,
      "learning_rate": 2.389874371926448e-09,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.40625,
      "logps/chosen": -584.0,
      "logps/rejected": -716.0,
      "loss": 0.4079,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.25,
      "step": 25800
    },
    {
      "epoch": 0.9607116934358192,
      "grad_norm": 12.751816969626175,
      "learning_rate": 2.345274781768769e-09,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.671875,
      "logps/chosen": -568.0,
      "logps/rejected": -716.0,
      "loss": 0.3662,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.1875,
      "step": 25810
    },
    {
      "epoch": 0.961083918036143,
      "grad_norm": 12.024652095008875,
      "learning_rate": 2.3010933160539927e-09,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.515625,
      "logps/chosen": -568.0,
      "logps/rejected": -740.0,
      "loss": 0.4068,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -5.40625,
      "step": 25820
    },
    {
      "epoch": 0.9614561426364668,
      "grad_norm": 11.851612337507431,
      "learning_rate": 2.2573300493752835e-09,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.65625,
      "logps/chosen": -556.0,
      "logps/rejected": -688.0,
      "loss": 0.4014,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -5.0625,
      "step": 25830
    },
    {
      "epoch": 0.9618283672367907,
      "grad_norm": 12.765670924486257,
      "learning_rate": 2.213985055619788e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -548.0,
      "logps/rejected": -684.0,
      "loss": 0.3892,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.546875,
      "rewards/rejected": -5.03125,
      "step": 25840
    },
    {
      "epoch": 0.9622005918371145,
      "grad_norm": 12.345999231011586,
      "learning_rate": 2.171058407968357e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -540.0,
      "logps/rejected": -696.0,
      "loss": 0.3899,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.0625,
      "step": 25850
    },
    {
      "epoch": 0.9625728164374383,
      "grad_norm": 10.739304147477203,
      "learning_rate": 2.128550178895655e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -592.0,
      "logps/rejected": -744.0,
      "loss": 0.3788,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.8125,
      "rewards/rejected": -5.53125,
      "step": 25860
    },
    {
      "epoch": 0.9629450410377622,
      "grad_norm": 12.99407650583786,
      "learning_rate": 2.086460440169857e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -536.0,
      "logps/rejected": -708.0,
      "loss": 0.3904,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.28125,
      "step": 25870
    },
    {
      "epoch": 0.9633172656380861,
      "grad_norm": 13.532205401268927,
      "learning_rate": 2.044789262852592e-09,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.5625,
      "logps/chosen": -532.0,
      "logps/rejected": -672.0,
      "loss": 0.4007,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.34375,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -5.03125,
      "step": 25880
    },
    {
      "epoch": 0.9636894902384099,
      "grad_norm": 13.680021300408267,
      "learning_rate": 2.003536717298804e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.546875,
      "logps/chosen": -544.0,
      "logps/rejected": -716.0,
      "loss": 0.3697,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.4375,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.1875,
      "step": 25890
    },
    {
      "epoch": 0.9640617148387337,
      "grad_norm": 14.45652869432489,
      "learning_rate": 1.9627028731566685e-09,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.484375,
      "logps/chosen": -556.0,
      "logps/rejected": -732.0,
      "loss": 0.3851,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.8125,
      "rewards/rejected": -5.28125,
      "step": 25900
    },
    {
      "epoch": 0.9644339394390575,
      "grad_norm": 11.48054665256611,
      "learning_rate": 1.922287799367456e-09,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.640625,
      "logps/chosen": -628.0,
      "logps/rejected": -776.0,
      "loss": 0.3926,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.9375,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -5.71875,
      "step": 25910
    },
    {
      "epoch": 0.9648061640393814,
      "grad_norm": 9.703551236240848,
      "learning_rate": 1.882291564165389e-09,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5,
      "logps/chosen": -560.0,
      "logps/rejected": -724.0,
      "loss": 0.3707,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -5.375,
      "step": 25920
    },
    {
      "epoch": 0.9651783886397052,
      "grad_norm": 13.89804426535108,
      "learning_rate": 1.8427142350775638e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.28125,
      "logps/chosen": -548.0,
      "logps/rejected": -704.0,
      "loss": 0.3996,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.65625,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.3125,
      "step": 25930
    },
    {
      "epoch": 0.965550613240029,
      "grad_norm": 12.172562843388512,
      "learning_rate": 1.803555878923807e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.65625,
      "logps/chosen": -556.0,
      "logps/rejected": -712.0,
      "loss": 0.3832,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -5.25,
      "step": 25940
    },
    {
      "epoch": 0.9659228378403528,
      "grad_norm": 12.162762822712445,
      "learning_rate": 1.7648165618166233e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.453125,
      "logps/chosen": -568.0,
      "logps/rejected": -684.0,
      "loss": 0.3821,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.53125,
      "rewards/rejected": -5.125,
      "step": 25950
    },
    {
      "epoch": 0.9662950624406768,
      "grad_norm": 11.484473940907955,
      "learning_rate": 1.726496349160944e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -540.0,
      "logps/rejected": -716.0,
      "loss": 0.3604,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.9375,
      "rewards/rejected": -5.3125,
      "step": 25960
    },
    {
      "epoch": 0.9666672870410006,
      "grad_norm": 13.377562465256288,
      "learning_rate": 1.688595305654239e-09,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.453125,
      "logps/chosen": -568.0,
      "logps/rejected": -720.0,
      "loss": 0.3935,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.5625,
      "rewards/rejected": -5.09375,
      "step": 25970
    },
    {
      "epoch": 0.9670395116413244,
      "grad_norm": 11.602405239522202,
      "learning_rate": 1.651113495286155e-09,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.546875,
      "logps/chosen": -588.0,
      "logps/rejected": -728.0,
      "loss": 0.3789,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -5.25,
      "step": 25980
    },
    {
      "epoch": 0.9674117362416482,
      "grad_norm": 14.224745561049126,
      "learning_rate": 1.6140509813386273e-09,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5,
      "logps/chosen": -576.0,
      "logps/rejected": -712.0,
      "loss": 0.3735,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.640625,
      "rewards/rejected": -5.25,
      "step": 25990
    },
    {
      "epoch": 0.967783960841972,
      "grad_norm": 12.005212035327915,
      "learning_rate": 1.5774078263856571e-09,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.625,
      "logps/chosen": -556.0,
      "logps/rejected": -700.0,
      "loss": 0.3764,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.5625,
      "rewards/rejected": -5.125,
      "step": 26000
    },
    {
      "epoch": 0.9681561854422959,
      "grad_norm": 11.73830417210075,
      "learning_rate": 1.5411840922931184e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.28125,
      "logps/chosen": -540.0,
      "logps/rejected": -688.0,
      "loss": 0.3683,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.78125,
      "rewards/rejected": -5.1875,
      "step": 26010
    },
    {
      "epoch": 0.9685284100426197,
      "grad_norm": 11.993124106821906,
      "learning_rate": 1.5053798402189788e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -556.0,
      "logps/rejected": -700.0,
      "loss": 0.3929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.5625,
      "rewards/rejected": -5.03125,
      "step": 26020
    },
    {
      "epoch": 0.9689006346429435,
      "grad_norm": 12.827175653653958,
      "learning_rate": 1.4699951306127722e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.6875,
      "logps/chosen": -612.0,
      "logps/rejected": -744.0,
      "loss": 0.3617,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.921875,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.5625,
      "step": 26030
    },
    {
      "epoch": 0.9692728592432673,
      "grad_norm": 14.37676657466553,
      "learning_rate": 1.4350300232158218e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -584.0,
      "logps/rejected": -736.0,
      "loss": 0.3948,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.34375,
      "step": 26040
    },
    {
      "epoch": 0.9696450838435913,
      "grad_norm": 12.124885461454948,
      "learning_rate": 1.400484577060962e-09,
      "logits/chosen": -2.75,
      "logits/rejected": -2.703125,
      "logps/chosen": -576.0,
      "logps/rejected": -712.0,
      "loss": 0.3879,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.6875,
      "rewards/rejected": -5.3125,
      "step": 26050
    },
    {
      "epoch": 0.9700173084439151,
      "grad_norm": 10.477332532152394,
      "learning_rate": 1.3663588504725098e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -544.0,
      "logps/rejected": -680.0,
      "loss": 0.3743,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.875,
      "step": 26060
    },
    {
      "epoch": 0.9703895330442389,
      "grad_norm": 13.700159786682187,
      "learning_rate": 1.3326529010662112e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.578125,
      "logps/chosen": -576.0,
      "logps/rejected": -720.0,
      "loss": 0.3863,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.796875,
      "rewards/rejected": -5.1875,
      "step": 26070
    },
    {
      "epoch": 0.9707617576445627,
      "grad_norm": 12.26720052502932,
      "learning_rate": 1.2993667857489898e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -568.0,
      "logps/rejected": -728.0,
      "loss": 0.351,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.84375,
      "rewards/rejected": -5.34375,
      "step": 26080
    },
    {
      "epoch": 0.9711339822448866,
      "grad_norm": 10.954905256507562,
      "learning_rate": 1.2665005607190581e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.546875,
      "logps/chosen": -568.0,
      "logps/rejected": -712.0,
      "loss": 0.3783,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.15625,
      "step": 26090
    },
    {
      "epoch": 0.9715062068452104,
      "grad_norm": 12.672346786182617,
      "learning_rate": 1.2340542814655853e-09,
      "logits/chosen": -2.75,
      "logits/rejected": -2.609375,
      "logps/chosen": -596.0,
      "logps/rejected": -744.0,
      "loss": 0.3697,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.375,
      "step": 26100
    },
    {
      "epoch": 0.9718784314455342,
      "grad_norm": 13.524212988926525,
      "learning_rate": 1.2020280027688622e-09,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5625,
      "logps/chosen": -576.0,
      "logps/rejected": -704.0,
      "loss": 0.3858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.359375,
      "rewards/rejected": -5.09375,
      "step": 26110
    },
    {
      "epoch": 0.972250656045858,
      "grad_norm": 10.641979974298316,
      "learning_rate": 1.1704217786999703e-09,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.46875,
      "logps/chosen": -560.0,
      "logps/rejected": -712.0,
      "loss": 0.3547,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.34375,
      "step": 26120
    },
    {
      "epoch": 0.9726228806461819,
      "grad_norm": 12.66782460681118,
      "learning_rate": 1.139235662620891e-09,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.640625,
      "logps/chosen": -608.0,
      "logps/rejected": -768.0,
      "loss": 0.3958,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.828125,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.53125,
      "step": 26130
    },
    {
      "epoch": 0.9729951052465058,
      "grad_norm": 13.153652509007056,
      "learning_rate": 1.1084697071842008e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.484375,
      "logps/chosen": -548.0,
      "logps/rejected": -688.0,
      "loss": 0.3712,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.0,
      "step": 26140
    },
    {
      "epoch": 0.9733673298468296,
      "grad_norm": 11.323759234474716,
      "learning_rate": 1.0781239643332384e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.625,
      "logps/chosen": -580.0,
      "logps/rejected": -756.0,
      "loss": 0.3815,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.8125,
      "rewards/rejected": -5.5,
      "step": 26150
    },
    {
      "epoch": 0.9737395544471534,
      "grad_norm": 11.584768221578036,
      "learning_rate": 1.0481984853017988e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -576.0,
      "logps/rejected": -732.0,
      "loss": 0.3804,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.6015625,
      "rewards/rejected": -5.1875,
      "step": 26160
    },
    {
      "epoch": 0.9741117790474773,
      "grad_norm": 12.554395759203297,
      "learning_rate": 1.0186933206141612e-09,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.625,
      "logps/chosen": -568.0,
      "logps/rejected": -708.0,
      "loss": 0.377,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.03125,
      "step": 26170
    },
    {
      "epoch": 0.9744840036478011,
      "grad_norm": 11.223172955897878,
      "learning_rate": 9.89608520084978e-10,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.734375,
      "logps/chosen": -624.0,
      "logps/rejected": -744.0,
      "loss": 0.3858,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.875,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.4375,
      "step": 26180
    },
    {
      "epoch": 0.9748562282481249,
      "grad_norm": 12.36716153887432,
      "learning_rate": 9.609441328191082e-10,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.40625,
      "logps/chosen": -576.0,
      "logps/rejected": -736.0,
      "loss": 0.3797,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.21875,
      "step": 26190
    },
    {
      "epoch": 0.9752284528484487,
      "grad_norm": 13.165949595323742,
      "learning_rate": 9.327002072117563e-10,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5625,
      "logps/chosen": -556.0,
      "logps/rejected": -724.0,
      "loss": 0.3889,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -5.1875,
      "step": 26200
    },
    {
      "epoch": 0.9756006774487725,
      "grad_norm": 10.21699388213036,
      "learning_rate": 9.048767909480837e-10,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.46875,
      "logps/chosen": -536.0,
      "logps/rejected": -688.0,
      "loss": 0.3627,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.0,
      "step": 26210
    },
    {
      "epoch": 0.9759729020490965,
      "grad_norm": 12.27992769440028,
      "learning_rate": 8.774739310034585e-10,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.6875,
      "logps/chosen": -588.0,
      "logps/rejected": -732.0,
      "loss": 0.3709,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -5.375,
      "step": 26220
    },
    {
      "epoch": 0.9763451266494203,
      "grad_norm": 11.144733518295036,
      "learning_rate": 8.504916736430667e-10,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5625,
      "logps/chosen": -564.0,
      "logps/rejected": -716.0,
      "loss": 0.4003,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.796875,
      "rewards/rejected": -5.34375,
      "step": 26230
    },
    {
      "epoch": 0.9767173512497441,
      "grad_norm": 11.6855525822593,
      "learning_rate": 8.239300644220237e-10,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.609375,
      "logps/chosen": -616.0,
      "logps/rejected": -760.0,
      "loss": 0.3935,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.71875,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.46875,
      "step": 26240
    },
    {
      "epoch": 0.977089575850068,
      "grad_norm": 11.24018564590509,
      "learning_rate": 7.977891481852906e-10,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -564.0,
      "logps/rejected": -716.0,
      "loss": 0.3902,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.671875,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -5.28125,
      "step": 26250
    },
    {
      "epoch": 0.9774618004503918,
      "grad_norm": 12.430674739342667,
      "learning_rate": 7.720689690674798e-10,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.578125,
      "logps/chosen": -540.0,
      "logps/rejected": -692.0,
      "loss": 0.3769,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -5.03125,
      "step": 26260
    },
    {
      "epoch": 0.9778340250507156,
      "grad_norm": 13.193639927729864,
      "learning_rate": 7.467695704929666e-10,
      "logits/chosen": -2.625,
      "logits/rejected": -2.578125,
      "logps/chosen": -588.0,
      "logps/rejected": -744.0,
      "loss": 0.3564,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.375,
      "step": 26270
    },
    {
      "epoch": 0.9782062496510394,
      "grad_norm": 13.389035623408677,
      "learning_rate": 7.218909951755836e-10,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.53125,
      "logps/chosen": -568.0,
      "logps/rejected": -704.0,
      "loss": 0.4095,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.625,
      "rewards/rejected": -5.125,
      "step": 26280
    },
    {
      "epoch": 0.9785784742513632,
      "grad_norm": 13.790806771875403,
      "learning_rate": 6.974332851187592e-10,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5,
      "logps/chosen": -572.0,
      "logps/rejected": -724.0,
      "loss": 0.3921,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -5.28125,
      "step": 26290
    },
    {
      "epoch": 0.9789506988516871,
      "grad_norm": 13.045463823328681,
      "learning_rate": 6.733964816153237e-10,
      "logits/chosen": -2.75,
      "logits/rejected": -2.578125,
      "logps/chosen": -600.0,
      "logps/rejected": -736.0,
      "loss": 0.3731,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.828125,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -5.4375,
      "step": 26300
    },
    {
      "epoch": 0.979322923452011,
      "grad_norm": 12.643974076625074,
      "learning_rate": 6.497806252474814e-10,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.609375,
      "logps/chosen": -524.0,
      "logps/rejected": -672.0,
      "loss": 0.3962,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -4.875,
      "step": 26310
    },
    {
      "epoch": 0.9796951480523348,
      "grad_norm": 12.14956211193431,
      "learning_rate": 6.265857558867271e-10,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.46875,
      "logps/chosen": -592.0,
      "logps/rejected": -724.0,
      "loss": 0.3814,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.53125,
      "rewards/rejected": -5.125,
      "step": 26320
    },
    {
      "epoch": 0.9800673726526586,
      "grad_norm": 11.154179887731237,
      "learning_rate": 6.038119126938191e-10,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.59375,
      "logps/chosen": -572.0,
      "logps/rejected": -692.0,
      "loss": 0.3871,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.4375,
      "rewards/rejected": -5.03125,
      "step": 26330
    },
    {
      "epoch": 0.9804395972529825,
      "grad_norm": 12.33151041075265,
      "learning_rate": 5.814591341186392e-10,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.65625,
      "logps/chosen": -560.0,
      "logps/rejected": -704.0,
      "loss": 0.3818,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.1875,
      "step": 26340
    },
    {
      "epoch": 0.9808118218533063,
      "grad_norm": 14.70433139996672,
      "learning_rate": 5.595274579002219e-10,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -584.0,
      "logps/rejected": -736.0,
      "loss": 0.3754,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.3125,
      "step": 26350
    },
    {
      "epoch": 0.9811840464536301,
      "grad_norm": 13.840385947099549,
      "learning_rate": 5.380169210666141e-10,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.390625,
      "logps/chosen": -552.0,
      "logps/rejected": -728.0,
      "loss": 0.4028,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.78125,
      "rewards/rejected": -5.34375,
      "step": 26360
    },
    {
      "epoch": 0.9815562710539539,
      "grad_norm": 12.063981491077207,
      "learning_rate": 5.169275599347933e-10,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.734375,
      "logps/chosen": -632.0,
      "logps/rejected": -772.0,
      "loss": 0.397,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.9375,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.59375,
      "step": 26370
    },
    {
      "epoch": 0.9819284956542778,
      "grad_norm": 12.55863372575399,
      "learning_rate": 4.96259410110722e-10,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.734375,
      "logps/chosen": -560.0,
      "logps/rejected": -728.0,
      "loss": 0.354,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.6875,
      "rewards/rejected": -5.25,
      "step": 26380
    },
    {
      "epoch": 0.9823007202546016,
      "grad_norm": 13.028163104193297,
      "learning_rate": 4.760125064891818e-10,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.453125,
      "logps/chosen": -600.0,
      "logps/rejected": -732.0,
      "loss": 0.3618,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -5.3125,
      "step": 26390
    },
    {
      "epoch": 0.9826729448549255,
      "grad_norm": 14.733025435172623,
      "learning_rate": 4.561868832537452e-10,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.625,
      "logps/chosen": -568.0,
      "logps/rejected": -728.0,
      "loss": 0.3804,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.828125,
      "rewards/rejected": -5.3125,
      "step": 26400
    },
    {
      "epoch": 0.9830451694552493,
      "grad_norm": 11.060805140566606,
      "learning_rate": 4.367825738767206e-10,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -532.0,
      "logps/rejected": -672.0,
      "loss": 0.3771,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.359375,
      "rewards/margins": 1.5390625,
      "rewards/rejected": -4.90625,
      "step": 26410
    },
    {
      "epoch": 0.9834173940555732,
      "grad_norm": 10.011356998999695,
      "learning_rate": 4.1779961111915176e-10,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5,
      "logps/chosen": -572.0,
      "logps/rejected": -708.0,
      "loss": 0.3738,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -5.03125,
      "step": 26420
    },
    {
      "epoch": 0.983789618655897,
      "grad_norm": 13.074549059251957,
      "learning_rate": 3.9923802703059616e-10,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.6875,
      "logps/chosen": -532.0,
      "logps/rejected": -696.0,
      "loss": 0.3805,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.828125,
      "rewards/rejected": -5.21875,
      "step": 26430
    },
    {
      "epoch": 0.9841618432562208,
      "grad_norm": 13.471862240669035,
      "learning_rate": 3.810978529493192e-10,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -560.0,
      "logps/rejected": -712.0,
      "loss": 0.3798,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -5.25,
      "step": 26440
    },
    {
      "epoch": 0.9845340678565446,
      "grad_norm": 13.805547965192096,
      "learning_rate": 3.633791195019886e-10,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.625,
      "logps/chosen": -552.0,
      "logps/rejected": -700.0,
      "loss": 0.392,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.4765625,
      "rewards/rejected": -5.0625,
      "step": 26450
    },
    {
      "epoch": 0.9849062924568684,
      "grad_norm": 12.235282671602267,
      "learning_rate": 3.4608185660378595e-10,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -552.0,
      "logps/rejected": -676.0,
      "loss": 0.3731,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.125,
      "step": 26460
    },
    {
      "epoch": 0.9852785170571923,
      "grad_norm": 16.704575386392722,
      "learning_rate": 3.292060934582952e-10,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.625,
      "logps/chosen": -592.0,
      "logps/rejected": -748.0,
      "loss": 0.4131,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.609375,
      "rewards/rejected": -5.21875,
      "step": 26470
    },
    {
      "epoch": 0.9856507416575161,
      "grad_norm": 12.42479230514382,
      "learning_rate": 3.127518585575306e-10,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.53125,
      "logps/chosen": -584.0,
      "logps/rejected": -744.0,
      "loss": 0.3975,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.59375,
      "rewards/margins": 1.765625,
      "rewards/rejected": -5.375,
      "step": 26480
    },
    {
      "epoch": 0.98602296625784,
      "grad_norm": 12.79407285161256,
      "learning_rate": 2.9671917968171476e-10,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.609375,
      "logps/chosen": -608.0,
      "logps/rejected": -732.0,
      "loss": 0.3865,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -3.875,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -5.375,
      "step": 26490
    },
    {
      "epoch": 0.9863951908581639,
      "grad_norm": 10.58551337580507,
      "learning_rate": 2.8110808389944506e-10,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.609375,
      "logps/chosen": -572.0,
      "logps/rejected": -744.0,
      "loss": 0.3788,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.546875,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -5.40625,
      "step": 26500
    },
    {
      "epoch": 0.9867674154584877,
      "grad_norm": 13.226954905212143,
      "learning_rate": 2.6591859756749934e-10,
      "logits/chosen": -2.75,
      "logits/rejected": -2.671875,
      "logps/chosen": -592.0,
      "logps/rejected": -728.0,
      "loss": 0.3986,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.25,
      "step": 26510
    },
    {
      "epoch": 0.9871396400588115,
      "grad_norm": 13.079815429231985,
      "learning_rate": 2.511507463307805e-10,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.546875,
      "logps/chosen": -596.0,
      "logps/rejected": -748.0,
      "loss": 0.386,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.5,
      "step": 26520
    },
    {
      "epoch": 0.9875118646591353,
      "grad_norm": 11.798927089671873,
      "learning_rate": 2.3680455512242736e-10,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.53125,
      "logps/chosen": -584.0,
      "logps/rejected": -732.0,
      "loss": 0.383,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -5.3125,
      "step": 26530
    },
    {
      "epoch": 0.9878840892594591,
      "grad_norm": 12.407208716032939,
      "learning_rate": 2.2288004816364836e-10,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -612.0,
      "logps/rejected": -752.0,
      "loss": 0.3725,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.859375,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -5.40625,
      "step": 26540
    },
    {
      "epoch": 0.988256313859783,
      "grad_norm": 12.02940149523056,
      "learning_rate": 2.0937724896369358e-10,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.609375,
      "logps/chosen": -584.0,
      "logps/rejected": -740.0,
      "loss": 0.384,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.375,
      "step": 26550
    },
    {
      "epoch": 0.9886285384601068,
      "grad_norm": 12.160236309857048,
      "learning_rate": 1.9629618031977159e-10,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.65625,
      "logps/chosen": -556.0,
      "logps/rejected": -712.0,
      "loss": 0.3654,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.21875,
      "step": 26560
    },
    {
      "epoch": 0.9890007630604306,
      "grad_norm": 10.680788774044064,
      "learning_rate": 1.8363686431718817e-10,
      "logits/chosen": -2.625,
      "logits/rejected": -2.546875,
      "logps/chosen": -572.0,
      "logps/rejected": -700.0,
      "loss": 0.3657,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.578125,
      "rewards/rejected": -5.15625,
      "step": 26570
    },
    {
      "epoch": 0.9893729876607545,
      "grad_norm": 14.22248619169697,
      "learning_rate": 1.713993223290966e-10,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -580.0,
      "logps/rejected": -748.0,
      "loss": 0.3694,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -5.46875,
      "step": 26580
    },
    {
      "epoch": 0.9897452122610784,
      "grad_norm": 14.589567480124675,
      "learning_rate": 1.5958357501658082e-10,
      "logits/chosen": -2.625,
      "logits/rejected": -2.40625,
      "logps/chosen": -580.0,
      "logps/rejected": -728.0,
      "loss": 0.3743,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.25,
      "step": 26590
    },
    {
      "epoch": 0.9901174368614022,
      "grad_norm": 12.774953448403062,
      "learning_rate": 1.4818964232859997e-10,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.671875,
      "logps/chosen": -540.0,
      "logps/rejected": -704.0,
      "loss": 0.3527,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.40625,
      "rewards/margins": 1.8828125,
      "rewards/rejected": -5.28125,
      "step": 26600
    },
    {
      "epoch": 0.990489661461726,
      "grad_norm": 12.07859929572445,
      "learning_rate": 1.3721754350196068e-10,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.59375,
      "logps/chosen": -548.0,
      "logps/rejected": -704.0,
      "loss": 0.3692,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.453125,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -5.21875,
      "step": 26610
    },
    {
      "epoch": 0.9908618860620498,
      "grad_norm": 12.3429775060109,
      "learning_rate": 1.2666729706120593e-10,
      "logits/chosen": -2.625,
      "logits/rejected": -2.515625,
      "logps/chosen": -596.0,
      "logps/rejected": -716.0,
      "loss": 0.3836,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.75,
      "rewards/margins": 1.4375,
      "rewards/rejected": -5.1875,
      "step": 26620
    },
    {
      "epoch": 0.9912341106623737,
      "grad_norm": 13.162732673839601,
      "learning_rate": 1.1653892081875393e-10,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.578125,
      "logps/chosen": -552.0,
      "logps/rejected": -664.0,
      "loss": 0.4089,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.390625,
      "rewards/rejected": -4.90625,
      "step": 26630
    },
    {
      "epoch": 0.9916063352626975,
      "grad_norm": 11.568990589430394,
      "learning_rate": 1.0683243187467605e-10,
      "logits/chosen": -2.625,
      "logits/rejected": -2.640625,
      "logps/chosen": -544.0,
      "logps/rejected": -720.0,
      "loss": 0.3616,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.515625,
      "rewards/margins": 1.734375,
      "rewards/rejected": -5.25,
      "step": 26640
    },
    {
      "epoch": 0.9919785598630213,
      "grad_norm": 13.51755311897776,
      "learning_rate": 9.754784661680781e-11,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.515625,
      "logps/chosen": -568.0,
      "logps/rejected": -712.0,
      "loss": 0.4006,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.640625,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.3125,
      "step": 26650
    },
    {
      "epoch": 0.9923507844633452,
      "grad_norm": 13.44729724015547,
      "learning_rate": 8.86851807206379e-11,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.390625,
      "logps/chosen": -556.0,
      "logps/rejected": -688.0,
      "loss": 0.3812,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.0625,
      "step": 26660
    },
    {
      "epoch": 0.9927230090636691,
      "grad_norm": 13.279642630984956,
      "learning_rate": 8.024444914933592e-11,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.375,
      "logps/chosen": -552.0,
      "logps/rejected": -712.0,
      "loss": 0.3799,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -5.25,
      "step": 26670
    },
    {
      "epoch": 0.9930952336639929,
      "grad_norm": 12.015145348778157,
      "learning_rate": 7.222566615369685e-11,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.640625,
      "logps/chosen": -552.0,
      "logps/rejected": -676.0,
      "loss": 0.3917,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.421875,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -4.875,
      "step": 26680
    },
    {
      "epoch": 0.9934674582643167,
      "grad_norm": 12.984262448460166,
      "learning_rate": 6.462884527208557e-11,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5625,
      "logps/chosen": -572.0,
      "logps/rejected": -700.0,
      "loss": 0.3905,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.515625,
      "rewards/rejected": -5.09375,
      "step": 26690
    },
    {
      "epoch": 0.9938396828646405,
      "grad_norm": 12.451833436816766,
      "learning_rate": 5.745399933054784e-11,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.65625,
      "logps/chosen": -572.0,
      "logps/rejected": -704.0,
      "loss": 0.3855,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.53125,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -5.03125,
      "step": 26700
    },
    {
      "epoch": 0.9942119074649644,
      "grad_norm": 11.823937095330143,
      "learning_rate": 5.0701140442588333e-11,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -576.0,
      "logps/rejected": -712.0,
      "loss": 0.3586,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -3.578125,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -5.28125,
      "step": 26710
    },
    {
      "epoch": 0.9945841320652882,
      "grad_norm": 12.784504200740505,
      "learning_rate": 4.437028000933707e-11,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -584.0,
      "logps/rejected": -732.0,
      "loss": 0.3903,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.625,
      "rewards/margins": 1.65625,
      "rewards/rejected": -5.28125,
      "step": 26720
    },
    {
      "epoch": 0.994956356665612,
      "grad_norm": 11.512363998075505,
      "learning_rate": 3.846142871938296e-11,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.546875,
      "logps/chosen": -552.0,
      "logps/rejected": -688.0,
      "loss": 0.3993,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 1.46875,
      "rewards/rejected": -4.9375,
      "step": 26730
    },
    {
      "epoch": 0.9953285812659358,
      "grad_norm": 11.844454877718462,
      "learning_rate": 3.2974596548884795e-11,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -572.0,
      "logps/rejected": -716.0,
      "loss": 0.3625,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.46875,
      "rewards/margins": 1.671875,
      "rewards/rejected": -5.15625,
      "step": 26740
    },
    {
      "epoch": 0.9957008058662598,
      "grad_norm": 11.757942305158313,
      "learning_rate": 2.790979276143246e-11,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.640625,
      "logps/chosen": -604.0,
      "logps/rejected": -736.0,
      "loss": 0.3615,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.78125,
      "rewards/margins": 1.515625,
      "rewards/rejected": -5.3125,
      "step": 26750
    },
    {
      "epoch": 0.9960730304665836,
      "grad_norm": 13.5738539280368,
      "learning_rate": 2.3267025908130232e-11,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.59375,
      "logps/chosen": -544.0,
      "logps/rejected": -728.0,
      "loss": 0.3817,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 1.96875,
      "rewards/rejected": -5.25,
      "step": 26760
    },
    {
      "epoch": 0.9964452550669074,
      "grad_norm": 10.89626651890187,
      "learning_rate": 1.9046303827569e-11,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.421875,
      "logps/chosen": -540.0,
      "logps/rejected": -692.0,
      "loss": 0.3547,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.0625,
      "step": 26770
    },
    {
      "epoch": 0.9968174796672312,
      "grad_norm": 11.55053406711159,
      "learning_rate": 1.524763364565973e-11,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -584.0,
      "logps/rejected": -744.0,
      "loss": 0.3713,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.71875,
      "rewards/rejected": -5.40625,
      "step": 26780
    },
    {
      "epoch": 0.997189704267555,
      "grad_norm": 11.08876241088393,
      "learning_rate": 1.1871021775911039e-11,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.671875,
      "logps/chosen": -600.0,
      "logps/rejected": -740.0,
      "loss": 0.4058,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.75,
      "rewards/rejected": -5.46875,
      "step": 26790
    },
    {
      "epoch": 0.9975619288678789,
      "grad_norm": 11.547018537703437,
      "learning_rate": 8.916473919151624e-12,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.390625,
      "logps/chosen": -564.0,
      "logps/rejected": -704.0,
      "loss": 0.3819,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -5.125,
      "step": 26800
    },
    {
      "epoch": 0.9979341534682027,
      "grad_norm": 11.380145589106387,
      "learning_rate": 6.38399506364129e-12,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.625,
      "logps/chosen": -564.0,
      "logps/rejected": -716.0,
      "loss": 0.3677,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -5.28125,
      "step": 26810
    },
    {
      "epoch": 0.9983063780685265,
      "grad_norm": 14.786103464107127,
      "learning_rate": 4.27358948507095e-12,
      "logits/chosen": -2.625,
      "logits/rejected": -2.640625,
      "logps/chosen": -576.0,
      "logps/rejected": -696.0,
      "loss": 0.3839,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.6875,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -5.15625,
      "step": 26820
    },
    {
      "epoch": 0.9986786026688503,
      "grad_norm": 12.807767775328847,
      "learning_rate": 2.5852607465071118e-12,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.6875,
      "logps/chosen": -564.0,
      "logps/rejected": -724.0,
      "loss": 0.3813,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.609375,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.3125,
      "step": 26830
    },
    {
      "epoch": 0.9990508272691743,
      "grad_norm": 12.34825006377118,
      "learning_rate": 1.3190116984196365e-12,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.75,
      "logps/chosen": -580.0,
      "logps/rejected": -712.0,
      "loss": 0.3879,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -3.703125,
      "rewards/margins": 1.515625,
      "rewards/rejected": -5.21875,
      "step": 26840
    },
    {
      "epoch": 0.9994230518694981,
      "grad_norm": 14.84562613495023,
      "learning_rate": 4.748444786539796e-13,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.625,
      "logps/chosen": -588.0,
      "logps/rejected": -760.0,
      "loss": 0.3833,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.78125,
      "rewards/margins": 1.703125,
      "rewards/rejected": -5.5,
      "step": 26850
    },
    {
      "epoch": 0.9997952764698219,
      "grad_norm": 13.907509579555143,
      "learning_rate": 5.2760512486704765e-14,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -572.0,
      "logps/rejected": -708.0,
      "loss": 0.391,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 1.515625,
      "rewards/rejected": -5.03125,
      "step": 26860
    },
    {
      "epoch": 0.9999813887699838,
      "step": 26865,
      "total_flos": 0.0,
      "train_loss": 0.0,
      "train_runtime": 18.6514,
      "train_samples_per_second": 184371.069,
      "train_steps_per_second": 1440.375
    }
  ],
  "logging_steps": 10,
  "max_steps": 26865,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 10000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}