{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9999360981532366,
  "eval_steps": 10000,
  "global_step": 7824,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.00012780369352674293,
      "grad_norm": 2.160655424307234,
      "learning_rate": 6.385696040868454e-10,
      "logits/chosen": -0.578125,
      "logits/rejected": -0.400390625,
      "logps/chosen": -167.0,
      "logps/rejected": -172.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0012780369352674292,
      "grad_norm": 2.2292416921787273,
      "learning_rate": 6.3856960408684546e-09,
      "logits/chosen": -0.95703125,
      "logits/rejected": -0.40234375,
      "logps/chosen": -183.0,
      "logps/rejected": -159.0,
      "loss": 0.6921,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.000659942626953125,
      "rewards/margins": 0.0006256103515625,
      "rewards/rejected": 3.4332275390625e-05,
      "step": 10
    },
    {
      "epoch": 0.0025560738705348585,
      "grad_norm": 3.1167582888203573,
      "learning_rate": 1.2771392081736909e-08,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.546875,
      "logps/chosen": -193.0,
      "logps/rejected": -159.0,
      "loss": 0.6924,
      "rewards/accuracies": 0.26249998807907104,
      "rewards/chosen": -6.198883056640625e-05,
      "rewards/margins": -0.0005950927734375,
      "rewards/rejected": 0.0005340576171875,
      "step": 20
    },
    {
      "epoch": 0.0038341108058022877,
      "grad_norm": 2.5180662966256713,
      "learning_rate": 1.915708812260536e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.57421875,
      "logps/chosen": -197.0,
      "logps/rejected": -166.0,
      "loss": 0.6924,
      "rewards/accuracies": 0.3062500059604645,
      "rewards/chosen": 0.00054931640625,
      "rewards/margins": 0.000732421875,
      "rewards/rejected": -0.000186920166015625,
      "step": 30
    },
    {
      "epoch": 0.005112147741069717,
      "grad_norm": 2.648705784512983,
      "learning_rate": 2.5542784163473818e-08,
      "logits/chosen": -0.96484375,
      "logits/rejected": -0.51953125,
      "logps/chosen": -179.0,
      "logps/rejected": -156.0,
      "loss": 0.6924,
      "rewards/accuracies": 0.33125001192092896,
      "rewards/chosen": 0.00087738037109375,
      "rewards/margins": 0.00095367431640625,
      "rewards/rejected": -7.772445678710938e-05,
      "step": 40
    },
    {
      "epoch": 0.006390184676337146,
      "grad_norm": 2.5489666254893697,
      "learning_rate": 3.192848020434227e-08,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.345703125,
      "logps/chosen": -189.0,
      "logps/rejected": -157.0,
      "loss": 0.6927,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": 0.00032806396484375,
      "rewards/margins": -0.0002193450927734375,
      "rewards/rejected": 0.00054931640625,
      "step": 50
    },
    {
      "epoch": 0.007668221611604575,
      "grad_norm": 2.4474346682362,
      "learning_rate": 3.831417624521072e-08,
      "logits/chosen": -0.890625,
      "logits/rejected": -0.40625,
      "logps/chosen": -182.0,
      "logps/rejected": -150.0,
      "loss": 0.6924,
      "rewards/accuracies": 0.29374998807907104,
      "rewards/chosen": 0.00087738037109375,
      "rewards/margins": 0.000186920166015625,
      "rewards/rejected": 0.000690460205078125,
      "step": 60
    },
    {
      "epoch": 0.008946258546872005,
      "grad_norm": 5.290448013063906,
      "learning_rate": 4.469987228607918e-08,
      "logits/chosen": -0.92578125,
      "logits/rejected": -0.416015625,
      "logps/chosen": -190.0,
      "logps/rejected": -165.0,
      "loss": 0.6922,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.00323486328125,
      "rewards/margins": 0.0012664794921875,
      "rewards/rejected": 0.0019683837890625,
      "step": 70
    },
    {
      "epoch": 0.010224295482139434,
      "grad_norm": 2.7218889097807697,
      "learning_rate": 5.1085568326947637e-08,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.376953125,
      "logps/chosen": -187.0,
      "logps/rejected": -158.0,
      "loss": 0.6917,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": 0.004058837890625,
      "rewards/margins": 0.00098419189453125,
      "rewards/rejected": 0.0030670166015625,
      "step": 80
    },
    {
      "epoch": 0.011502332417406863,
      "grad_norm": 2.819382249297325,
      "learning_rate": 5.747126436781609e-08,
      "logits/chosen": -0.86328125,
      "logits/rejected": -0.482421875,
      "logps/chosen": -177.0,
      "logps/rejected": -164.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": 0.00738525390625,
      "rewards/margins": 0.00151824951171875,
      "rewards/rejected": 0.005859375,
      "step": 90
    },
    {
      "epoch": 0.012780369352674292,
      "grad_norm": 3.262935898452368,
      "learning_rate": 6.385696040868454e-08,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.34375,
      "logps/chosen": -181.0,
      "logps/rejected": -159.0,
      "loss": 0.6915,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.01165771484375,
      "rewards/margins": 0.0021514892578125,
      "rewards/rejected": 0.009521484375,
      "step": 100
    },
    {
      "epoch": 0.014058406287941722,
      "grad_norm": 2.731162000427312,
      "learning_rate": 7.024265644955301e-08,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.38671875,
      "logps/chosen": -175.0,
      "logps/rejected": -154.0,
      "loss": 0.6906,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 0.015625,
      "rewards/margins": 0.002227783203125,
      "rewards/rejected": 0.01348876953125,
      "step": 110
    },
    {
      "epoch": 0.01533644322320915,
      "grad_norm": 2.9218189771620175,
      "learning_rate": 7.662835249042144e-08,
      "logits/chosen": -0.9296875,
      "logits/rejected": -0.3515625,
      "logps/chosen": -175.0,
      "logps/rejected": -157.0,
      "loss": 0.6912,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 0.0228271484375,
      "rewards/margins": 0.0035247802734375,
      "rewards/rejected": 0.019287109375,
      "step": 120
    },
    {
      "epoch": 0.01661448015847658,
      "grad_norm": 2.7940700270156396,
      "learning_rate": 8.301404853128991e-08,
      "logits/chosen": -0.89453125,
      "logits/rejected": -0.4296875,
      "logps/chosen": -170.0,
      "logps/rejected": -180.0,
      "loss": 0.6891,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.036376953125,
      "rewards/margins": 0.006927490234375,
      "rewards/rejected": 0.0294189453125,
      "step": 130
    },
    {
      "epoch": 0.01789251709374401,
      "grad_norm": 3.1150003449927888,
      "learning_rate": 8.939974457215836e-08,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.52734375,
      "logps/chosen": -188.0,
      "logps/rejected": -164.0,
      "loss": 0.6877,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": 0.048583984375,
      "rewards/margins": 0.00897216796875,
      "rewards/rejected": 0.03955078125,
      "step": 140
    },
    {
      "epoch": 0.01917055402901144,
      "grad_norm": 2.733853037231787,
      "learning_rate": 9.578544061302682e-08,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.60546875,
      "logps/chosen": -177.0,
      "logps/rejected": -152.0,
      "loss": 0.6865,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.06298828125,
      "rewards/margins": 0.01251220703125,
      "rewards/rejected": 0.050537109375,
      "step": 150
    },
    {
      "epoch": 0.020448590964278868,
      "grad_norm": 2.883091974112721,
      "learning_rate": 1.0217113665389527e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.4453125,
      "logps/chosen": -181.0,
      "logps/rejected": -168.0,
      "loss": 0.6843,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0830078125,
      "rewards/margins": 0.0167236328125,
      "rewards/rejected": 0.06640625,
      "step": 160
    },
    {
      "epoch": 0.021726627899546297,
      "grad_norm": 2.311235895014099,
      "learning_rate": 1.0855683269476372e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.439453125,
      "logps/chosen": -168.0,
      "logps/rejected": -167.0,
      "loss": 0.6803,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.1142578125,
      "rewards/margins": 0.031982421875,
      "rewards/rejected": 0.08203125,
      "step": 170
    },
    {
      "epoch": 0.023004664834813726,
      "grad_norm": 2.5148259492186984,
      "learning_rate": 1.1494252873563217e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.4375,
      "logps/chosen": -168.0,
      "logps/rejected": -162.0,
      "loss": 0.6762,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.126953125,
      "rewards/margins": 0.03662109375,
      "rewards/rejected": 0.08984375,
      "step": 180
    },
    {
      "epoch": 0.024282701770081155,
      "grad_norm": 2.33623413889278,
      "learning_rate": 1.2132822477650062e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.7421875,
      "logps/chosen": -178.0,
      "logps/rejected": -158.0,
      "loss": 0.6757,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.1328125,
      "rewards/margins": 0.032470703125,
      "rewards/rejected": 0.1005859375,
      "step": 190
    },
    {
      "epoch": 0.025560738705348585,
      "grad_norm": 2.107024223153624,
      "learning_rate": 1.277139208173691e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.625,
      "logps/chosen": -141.0,
      "logps/rejected": -131.0,
      "loss": 0.6737,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.13671875,
      "rewards/margins": 0.04052734375,
      "rewards/rejected": 0.095703125,
      "step": 200
    },
    {
      "epoch": 0.026838775640616014,
      "grad_norm": 2.3134372526473785,
      "learning_rate": 1.3409961685823753e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.671875,
      "logps/chosen": -164.0,
      "logps/rejected": -150.0,
      "loss": 0.6721,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.1513671875,
      "rewards/margins": 0.041748046875,
      "rewards/rejected": 0.109375,
      "step": 210
    },
    {
      "epoch": 0.028116812575883443,
      "grad_norm": 2.339324448011463,
      "learning_rate": 1.4048531289910602e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.51953125,
      "logps/chosen": -182.0,
      "logps/rejected": -176.0,
      "loss": 0.6708,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.15625,
      "rewards/margins": 0.053466796875,
      "rewards/rejected": 0.1025390625,
      "step": 220
    },
    {
      "epoch": 0.029394849511150872,
      "grad_norm": 2.150583019983845,
      "learning_rate": 1.4687100893997445e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.640625,
      "logps/chosen": -167.0,
      "logps/rejected": -157.0,
      "loss": 0.6704,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1484375,
      "rewards/margins": 0.0517578125,
      "rewards/rejected": 0.09619140625,
      "step": 230
    },
    {
      "epoch": 0.0306728864464183,
      "grad_norm": 2.6348377634282536,
      "learning_rate": 1.532567049808429e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.68359375,
      "logps/chosen": -180.0,
      "logps/rejected": -154.0,
      "loss": 0.6675,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.154296875,
      "rewards/margins": 0.06591796875,
      "rewards/rejected": 0.087890625,
      "step": 240
    },
    {
      "epoch": 0.03195092338168573,
      "grad_norm": 2.1045818587051177,
      "learning_rate": 1.5964240102171135e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.73828125,
      "logps/chosen": -169.0,
      "logps/rejected": -157.0,
      "loss": 0.6654,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.1552734375,
      "rewards/margins": 0.059814453125,
      "rewards/rejected": 0.09521484375,
      "step": 250
    },
    {
      "epoch": 0.03322896031695316,
      "grad_norm": 2.244611524128685,
      "learning_rate": 1.6602809706257982e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.6640625,
      "logps/chosen": -172.0,
      "logps/rejected": -167.0,
      "loss": 0.6609,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.1513671875,
      "rewards/margins": 0.08935546875,
      "rewards/rejected": 0.062255859375,
      "step": 260
    },
    {
      "epoch": 0.03450699725222059,
      "grad_norm": 2.2325457664289803,
      "learning_rate": 1.7241379310344828e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.609375,
      "logps/chosen": -173.0,
      "logps/rejected": -157.0,
      "loss": 0.6591,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.1181640625,
      "rewards/margins": 0.08056640625,
      "rewards/rejected": 0.037841796875,
      "step": 270
    },
    {
      "epoch": 0.03578503418748802,
      "grad_norm": 2.21742984764838,
      "learning_rate": 1.7879948914431672e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.640625,
      "logps/chosen": -166.0,
      "logps/rejected": -163.0,
      "loss": 0.6543,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.1171875,
      "rewards/margins": 0.0859375,
      "rewards/rejected": 0.031494140625,
      "step": 280
    },
    {
      "epoch": 0.03706307112275545,
      "grad_norm": 2.375331376888913,
      "learning_rate": 1.8518518518518516e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.65625,
      "logps/chosen": -172.0,
      "logps/rejected": -151.0,
      "loss": 0.652,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.09814453125,
      "rewards/margins": 0.0849609375,
      "rewards/rejected": 0.0130615234375,
      "step": 290
    },
    {
      "epoch": 0.03834110805802288,
      "grad_norm": 2.4911841810813926,
      "learning_rate": 1.9157088122605365e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.765625,
      "logps/chosen": -183.0,
      "logps/rejected": -176.0,
      "loss": 0.6468,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.068359375,
      "rewards/margins": 0.1015625,
      "rewards/rejected": -0.033447265625,
      "step": 300
    },
    {
      "epoch": 0.039619144993290306,
      "grad_norm": 3.0360943409746497,
      "learning_rate": 1.9795657726692208e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -0.734375,
      "logps/chosen": -171.0,
      "logps/rejected": -176.0,
      "loss": 0.6361,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.06591796875,
      "rewards/margins": 0.1357421875,
      "rewards/rejected": -0.06982421875,
      "step": 310
    },
    {
      "epoch": 0.040897181928557735,
      "grad_norm": 2.74095164364408,
      "learning_rate": 2.0434227330779055e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.87890625,
      "logps/chosen": -173.0,
      "logps/rejected": -180.0,
      "loss": 0.6288,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.045166015625,
      "rewards/margins": 0.142578125,
      "rewards/rejected": -0.09765625,
      "step": 320
    },
    {
      "epoch": 0.042175218863825165,
      "grad_norm": 2.5066071600629694,
      "learning_rate": 2.1072796934865898e-07,
      "logits/chosen": -1.46875,
      "logits/rejected": -0.90234375,
      "logps/chosen": -187.0,
      "logps/rejected": -190.0,
      "loss": 0.6217,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.0030364990234375,
      "rewards/margins": 0.1748046875,
      "rewards/rejected": -0.171875,
      "step": 330
    },
    {
      "epoch": 0.043453255799092594,
      "grad_norm": 2.8333916762075635,
      "learning_rate": 2.1711366538952745e-07,
      "logits/chosen": -1.5,
      "logits/rejected": -0.96875,
      "logps/chosen": -195.0,
      "logps/rejected": -182.0,
      "loss": 0.6103,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0145263671875,
      "rewards/margins": 0.22265625,
      "rewards/rejected": -0.2373046875,
      "step": 340
    },
    {
      "epoch": 0.04473129273436002,
      "grad_norm": 2.794637084189911,
      "learning_rate": 2.234993614303959e-07,
      "logits/chosen": -1.4921875,
      "logits/rejected": -1.0625,
      "logps/chosen": -177.0,
      "logps/rejected": -201.0,
      "loss": 0.5998,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.076171875,
      "rewards/margins": 0.255859375,
      "rewards/rejected": -0.33203125,
      "step": 350
    },
    {
      "epoch": 0.04600932966962745,
      "grad_norm": 2.863954505219251,
      "learning_rate": 2.2988505747126435e-07,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.046875,
      "logps/chosen": -202.0,
      "logps/rejected": -187.0,
      "loss": 0.5929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0986328125,
      "rewards/margins": 0.2353515625,
      "rewards/rejected": -0.333984375,
      "step": 360
    },
    {
      "epoch": 0.04728736660489488,
      "grad_norm": 2.532485664652631,
      "learning_rate": 2.3627075351213284e-07,
      "logits/chosen": -1.578125,
      "logits/rejected": -0.98828125,
      "logps/chosen": -199.0,
      "logps/rejected": -215.0,
      "loss": 0.5768,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.126953125,
      "rewards/margins": 0.369140625,
      "rewards/rejected": -0.49609375,
      "step": 370
    },
    {
      "epoch": 0.04856540354016231,
      "grad_norm": 2.7563880302070953,
      "learning_rate": 2.4265644955300125e-07,
      "logits/chosen": -1.59375,
      "logits/rejected": -1.1640625,
      "logps/chosen": -185.0,
      "logps/rejected": -220.0,
      "loss": 0.5592,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.185546875,
      "rewards/margins": 0.40234375,
      "rewards/rejected": -0.5859375,
      "step": 380
    },
    {
      "epoch": 0.04984344047542974,
      "grad_norm": 3.031584972796619,
      "learning_rate": 2.490421455938697e-07,
      "logits/chosen": -1.703125,
      "logits/rejected": -1.15625,
      "logps/chosen": -221.0,
      "logps/rejected": -209.0,
      "loss": 0.5698,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.296875,
      "rewards/margins": 0.310546875,
      "rewards/rejected": -0.609375,
      "step": 390
    },
    {
      "epoch": 0.05112147741069717,
      "grad_norm": 2.810407572102546,
      "learning_rate": 2.554278416347382e-07,
      "logits/chosen": -1.6640625,
      "logits/rejected": -1.375,
      "logps/chosen": -223.0,
      "logps/rejected": -258.0,
      "loss": 0.5661,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.353515625,
      "rewards/margins": 0.40234375,
      "rewards/rejected": -0.75390625,
      "step": 400
    },
    {
      "epoch": 0.0523995143459646,
      "grad_norm": 3.6478005495173487,
      "learning_rate": 2.6181353767560667e-07,
      "logits/chosen": -1.7265625,
      "logits/rejected": -1.2890625,
      "logps/chosen": -216.0,
      "logps/rejected": -251.0,
      "loss": 0.5509,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.2734375,
      "rewards/margins": 0.5078125,
      "rewards/rejected": -0.78125,
      "step": 410
    },
    {
      "epoch": 0.05367755128123203,
      "grad_norm": 3.1969297194305044,
      "learning_rate": 2.6819923371647505e-07,
      "logits/chosen": -1.6953125,
      "logits/rejected": -1.296875,
      "logps/chosen": -212.0,
      "logps/rejected": -258.0,
      "loss": 0.5408,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3046875,
      "rewards/margins": 0.60546875,
      "rewards/rejected": -0.91015625,
      "step": 420
    },
    {
      "epoch": 0.05495558821649946,
      "grad_norm": 3.2668307666716236,
      "learning_rate": 2.7458492975734354e-07,
      "logits/chosen": -1.828125,
      "logits/rejected": -1.4296875,
      "logps/chosen": -214.0,
      "logps/rejected": -229.0,
      "loss": 0.5458,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.34765625,
      "rewards/margins": 0.4609375,
      "rewards/rejected": -0.80859375,
      "step": 430
    },
    {
      "epoch": 0.056233625151766886,
      "grad_norm": 3.827576625046517,
      "learning_rate": 2.8097062579821203e-07,
      "logits/chosen": -1.8046875,
      "logits/rejected": -1.3984375,
      "logps/chosen": -225.0,
      "logps/rejected": -282.0,
      "loss": 0.5354,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.380859375,
      "rewards/margins": 0.6796875,
      "rewards/rejected": -1.0625,
      "step": 440
    },
    {
      "epoch": 0.057511662087034315,
      "grad_norm": 3.4047823485736974,
      "learning_rate": 2.873563218390804e-07,
      "logits/chosen": -1.7265625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -222.0,
      "logps/rejected": -270.0,
      "loss": 0.5254,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.421875,
      "rewards/margins": 0.66796875,
      "rewards/rejected": -1.0859375,
      "step": 450
    },
    {
      "epoch": 0.058789699022301745,
      "grad_norm": 3.5621152707110406,
      "learning_rate": 2.937420178799489e-07,
      "logits/chosen": -1.6640625,
      "logits/rejected": -1.40625,
      "logps/chosen": -206.0,
      "logps/rejected": -276.0,
      "loss": 0.5081,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.41015625,
      "rewards/margins": 0.765625,
      "rewards/rejected": -1.171875,
      "step": 460
    },
    {
      "epoch": 0.060067735957569174,
      "grad_norm": 3.756210566682855,
      "learning_rate": 3.001277139208174e-07,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.296875,
      "logps/chosen": -219.0,
      "logps/rejected": -280.0,
      "loss": 0.519,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.470703125,
      "rewards/margins": 0.703125,
      "rewards/rejected": -1.171875,
      "step": 470
    },
    {
      "epoch": 0.0613457728928366,
      "grad_norm": 5.497277904897848,
      "learning_rate": 3.065134099616858e-07,
      "logits/chosen": -1.734375,
      "logits/rejected": -1.3515625,
      "logps/chosen": -238.0,
      "logps/rejected": -300.0,
      "loss": 0.4946,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.4296875,
      "rewards/margins": 0.8984375,
      "rewards/rejected": -1.328125,
      "step": 480
    },
    {
      "epoch": 0.06262380982810403,
      "grad_norm": 5.3555959619707805,
      "learning_rate": 3.1289910600255427e-07,
      "logits/chosen": -1.7578125,
      "logits/rejected": -1.4375,
      "logps/chosen": -233.0,
      "logps/rejected": -304.0,
      "loss": 0.4864,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.53125,
      "rewards/margins": 0.9140625,
      "rewards/rejected": -1.4453125,
      "step": 490
    },
    {
      "epoch": 0.06390184676337146,
      "grad_norm": 5.328718671067563,
      "learning_rate": 3.192848020434227e-07,
      "logits/chosen": -1.7265625,
      "logits/rejected": -1.3671875,
      "logps/chosen": -233.0,
      "logps/rejected": -296.0,
      "loss": 0.5157,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.60546875,
      "rewards/margins": 0.71484375,
      "rewards/rejected": -1.3203125,
      "step": 500
    },
    {
      "epoch": 0.06517988369863889,
      "grad_norm": 4.8119071508315105,
      "learning_rate": 3.2567049808429114e-07,
      "logits/chosen": -1.734375,
      "logits/rejected": -1.421875,
      "logps/chosen": -228.0,
      "logps/rejected": -284.0,
      "loss": 0.4833,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.462890625,
      "rewards/margins": 0.8359375,
      "rewards/rejected": -1.296875,
      "step": 510
    },
    {
      "epoch": 0.06645792063390632,
      "grad_norm": 3.819886861472629,
      "learning_rate": 3.3205619412515963e-07,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.390625,
      "logps/chosen": -218.0,
      "logps/rejected": -296.0,
      "loss": 0.4781,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.51953125,
      "rewards/margins": 0.79296875,
      "rewards/rejected": -1.3125,
      "step": 520
    },
    {
      "epoch": 0.06773595756917375,
      "grad_norm": 4.1791969534636895,
      "learning_rate": 3.3844189016602807e-07,
      "logits/chosen": -1.6875,
      "logits/rejected": -1.3671875,
      "logps/chosen": -254.0,
      "logps/rejected": -322.0,
      "loss": 0.476,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.65625,
      "rewards/margins": 0.90625,
      "rewards/rejected": -1.5625,
      "step": 530
    },
    {
      "epoch": 0.06901399450444118,
      "grad_norm": 5.090343790872815,
      "learning_rate": 3.4482758620689656e-07,
      "logits/chosen": -1.8125,
      "logits/rejected": -1.4375,
      "logps/chosen": -250.0,
      "logps/rejected": -346.0,
      "loss": 0.4533,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.59375,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -1.6953125,
      "step": 540
    },
    {
      "epoch": 0.07029203143970861,
      "grad_norm": 5.23934577009625,
      "learning_rate": 3.51213282247765e-07,
      "logits/chosen": -1.7109375,
      "logits/rejected": -1.390625,
      "logps/chosen": -225.0,
      "logps/rejected": -322.0,
      "loss": 0.464,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.54296875,
      "rewards/margins": 1.0,
      "rewards/rejected": -1.5390625,
      "step": 550
    },
    {
      "epoch": 0.07157006837497604,
      "grad_norm": 5.1268484646535795,
      "learning_rate": 3.5759897828863344e-07,
      "logits/chosen": -1.765625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -256.0,
      "logps/rejected": -348.0,
      "loss": 0.4548,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.62890625,
      "rewards/margins": 1.0625,
      "rewards/rejected": -1.6875,
      "step": 560
    },
    {
      "epoch": 0.07284810531024347,
      "grad_norm": 9.951061187527841,
      "learning_rate": 3.639846743295019e-07,
      "logits/chosen": -1.7890625,
      "logits/rejected": -1.46875,
      "logps/chosen": -250.0,
      "logps/rejected": -328.0,
      "loss": 0.4679,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.74609375,
      "rewards/margins": 1.0078125,
      "rewards/rejected": -1.7578125,
      "step": 570
    },
    {
      "epoch": 0.0741261422455109,
      "grad_norm": 6.878155067465852,
      "learning_rate": 3.703703703703703e-07,
      "logits/chosen": -1.625,
      "logits/rejected": -1.40625,
      "logps/chosen": -253.0,
      "logps/rejected": -316.0,
      "loss": 0.4498,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.71875,
      "rewards/margins": 0.86328125,
      "rewards/rejected": -1.5859375,
      "step": 580
    },
    {
      "epoch": 0.07540417918077832,
      "grad_norm": 5.963396257819761,
      "learning_rate": 3.767560664112388e-07,
      "logits/chosen": -1.8515625,
      "logits/rejected": -1.5390625,
      "logps/chosen": -268.0,
      "logps/rejected": -362.0,
      "loss": 0.4464,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.75390625,
      "rewards/margins": 1.078125,
      "rewards/rejected": -1.828125,
      "step": 590
    },
    {
      "epoch": 0.07668221611604575,
      "grad_norm": 5.306711685604817,
      "learning_rate": 3.831417624521073e-07,
      "logits/chosen": -1.8515625,
      "logits/rejected": -1.453125,
      "logps/chosen": -272.0,
      "logps/rejected": -408.0,
      "loss": 0.4319,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.7265625,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -2.234375,
      "step": 600
    },
    {
      "epoch": 0.07796025305131318,
      "grad_norm": 5.391194298338634,
      "learning_rate": 3.895274584929757e-07,
      "logits/chosen": -1.71875,
      "logits/rejected": -1.4765625,
      "logps/chosen": -266.0,
      "logps/rejected": -362.0,
      "loss": 0.4383,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.78515625,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -1.8203125,
      "step": 610
    },
    {
      "epoch": 0.07923828998658061,
      "grad_norm": 6.20647779035015,
      "learning_rate": 3.9591315453384417e-07,
      "logits/chosen": -1.7890625,
      "logits/rejected": -1.5703125,
      "logps/chosen": -278.0,
      "logps/rejected": -346.0,
      "loss": 0.4254,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.8046875,
      "rewards/margins": 0.98828125,
      "rewards/rejected": -1.796875,
      "step": 620
    },
    {
      "epoch": 0.08051632692184804,
      "grad_norm": 6.17515508580775,
      "learning_rate": 4.0229885057471266e-07,
      "logits/chosen": -1.703125,
      "logits/rejected": -1.2890625,
      "logps/chosen": -266.0,
      "logps/rejected": -356.0,
      "loss": 0.4237,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.8828125,
      "rewards/margins": 1.140625,
      "rewards/rejected": -2.03125,
      "step": 630
    },
    {
      "epoch": 0.08179436385711547,
      "grad_norm": 5.6867443102183195,
      "learning_rate": 4.086845466155811e-07,
      "logits/chosen": -1.8359375,
      "logits/rejected": -1.59375,
      "logps/chosen": -262.0,
      "logps/rejected": -374.0,
      "loss": 0.4235,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.72265625,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -1.9609375,
      "step": 640
    },
    {
      "epoch": 0.0830724007923829,
      "grad_norm": 4.991738448525276,
      "learning_rate": 4.1507024265644953e-07,
      "logits/chosen": -1.7265625,
      "logits/rejected": -1.375,
      "logps/chosen": -252.0,
      "logps/rejected": -386.0,
      "loss": 0.4119,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8125,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -2.109375,
      "step": 650
    },
    {
      "epoch": 0.08435043772765033,
      "grad_norm": 5.720731244641413,
      "learning_rate": 4.2145593869731797e-07,
      "logits/chosen": -1.75,
      "logits/rejected": -1.4609375,
      "logps/chosen": -264.0,
      "logps/rejected": -376.0,
      "loss": 0.4166,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.87890625,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -2.1875,
      "step": 660
    },
    {
      "epoch": 0.08562847466291776,
      "grad_norm": 12.168384297620667,
      "learning_rate": 4.2784163473818646e-07,
      "logits/chosen": -1.734375,
      "logits/rejected": -1.4296875,
      "logps/chosen": -260.0,
      "logps/rejected": -384.0,
      "loss": 0.4506,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.7890625,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -2.09375,
      "step": 670
    },
    {
      "epoch": 0.08690651159818519,
      "grad_norm": 6.313556291086186,
      "learning_rate": 4.342273307790549e-07,
      "logits/chosen": -1.8203125,
      "logits/rejected": -1.234375,
      "logps/chosen": -294.0,
      "logps/rejected": -398.0,
      "loss": 0.432,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.92578125,
      "rewards/margins": 1.25,
      "rewards/rejected": -2.171875,
      "step": 680
    },
    {
      "epoch": 0.08818454853345262,
      "grad_norm": 9.109779279221746,
      "learning_rate": 4.4061302681992333e-07,
      "logits/chosen": -1.7109375,
      "logits/rejected": -1.5859375,
      "logps/chosen": -274.0,
      "logps/rejected": -376.0,
      "loss": 0.4106,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.9140625,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -2.125,
      "step": 690
    },
    {
      "epoch": 0.08946258546872005,
      "grad_norm": 8.818625438436053,
      "learning_rate": 4.469987228607918e-07,
      "logits/chosen": -1.796875,
      "logits/rejected": -1.4765625,
      "logps/chosen": -268.0,
      "logps/rejected": -392.0,
      "loss": 0.4178,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.89453125,
      "rewards/margins": 1.375,
      "rewards/rejected": -2.265625,
      "step": 700
    },
    {
      "epoch": 0.09074062240398748,
      "grad_norm": 7.4984442879084705,
      "learning_rate": 4.5338441890166026e-07,
      "logits/chosen": -1.7578125,
      "logits/rejected": -1.5078125,
      "logps/chosen": -274.0,
      "logps/rejected": -386.0,
      "loss": 0.4037,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.94140625,
      "rewards/margins": 1.3125,
      "rewards/rejected": -2.25,
      "step": 710
    },
    {
      "epoch": 0.0920186593392549,
      "grad_norm": 7.76698374196518,
      "learning_rate": 4.597701149425287e-07,
      "logits/chosen": -1.71875,
      "logits/rejected": -1.546875,
      "logps/chosen": -290.0,
      "logps/rejected": -402.0,
      "loss": 0.3961,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.984375,
      "rewards/margins": 1.34375,
      "rewards/rejected": -2.328125,
      "step": 720
    },
    {
      "epoch": 0.09329669627452233,
      "grad_norm": 8.666693260652762,
      "learning_rate": 4.661558109833972e-07,
      "logits/chosen": -1.78125,
      "logits/rejected": -1.4375,
      "logps/chosen": -264.0,
      "logps/rejected": -394.0,
      "loss": 0.3783,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.8984375,
      "rewards/margins": 1.59375,
      "rewards/rejected": -2.5,
      "step": 730
    },
    {
      "epoch": 0.09457473320978976,
      "grad_norm": 7.737596262830641,
      "learning_rate": 4.725415070242657e-07,
      "logits/chosen": -1.7109375,
      "logits/rejected": -1.53125,
      "logps/chosen": -308.0,
      "logps/rejected": -432.0,
      "loss": 0.3929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.078125,
      "rewards/margins": 1.53125,
      "rewards/rejected": -2.609375,
      "step": 740
    },
    {
      "epoch": 0.09585277014505719,
      "grad_norm": 9.719880510937397,
      "learning_rate": 4.789272030651341e-07,
      "logits/chosen": -1.6796875,
      "logits/rejected": -1.4921875,
      "logps/chosen": -272.0,
      "logps/rejected": -400.0,
      "loss": 0.3883,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.0234375,
      "rewards/margins": 1.4375,
      "rewards/rejected": -2.453125,
      "step": 750
    },
    {
      "epoch": 0.09713080708032462,
      "grad_norm": 6.72460998332888,
      "learning_rate": 4.853128991060025e-07,
      "logits/chosen": -1.796875,
      "logits/rejected": -1.5703125,
      "logps/chosen": -294.0,
      "logps/rejected": -474.0,
      "loss": 0.404,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.98828125,
      "rewards/margins": 1.875,
      "rewards/rejected": -2.859375,
      "step": 760
    },
    {
      "epoch": 0.09840884401559205,
      "grad_norm": 7.5277791889315715,
      "learning_rate": 4.91698595146871e-07,
      "logits/chosen": -1.703125,
      "logits/rejected": -1.4921875,
      "logps/chosen": -292.0,
      "logps/rejected": -428.0,
      "loss": 0.4021,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.09375,
      "rewards/margins": 1.4375,
      "rewards/rejected": -2.53125,
      "step": 770
    },
    {
      "epoch": 0.09968688095085948,
      "grad_norm": 6.445741662773321,
      "learning_rate": 4.980842911877394e-07,
      "logits/chosen": -1.7890625,
      "logits/rejected": -1.546875,
      "logps/chosen": -288.0,
      "logps/rejected": -446.0,
      "loss": 0.3566,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.0390625,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -2.765625,
      "step": 780
    },
    {
      "epoch": 0.10096491788612691,
      "grad_norm": 7.23763808821445,
      "learning_rate": 4.999987806263758e-07,
      "logits/chosen": -1.765625,
      "logits/rejected": -1.53125,
      "logps/chosen": -296.0,
      "logps/rejected": -454.0,
      "loss": 0.3836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1484375,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -2.875,
      "step": 790
    },
    {
      "epoch": 0.10224295482139434,
      "grad_norm": 7.027499502219845,
      "learning_rate": 4.999928082127696e-07,
      "logits/chosen": -1.75,
      "logits/rejected": -1.4375,
      "logps/chosen": -294.0,
      "logps/rejected": -450.0,
      "loss": 0.3825,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.109375,
      "rewards/margins": 1.703125,
      "rewards/rejected": -2.8125,
      "step": 800
    },
    {
      "epoch": 0.10352099175666177,
      "grad_norm": 9.245631263118588,
      "learning_rate": 4.999818589113488e-07,
      "logits/chosen": -1.6875,
      "logits/rejected": -1.5234375,
      "logps/chosen": -290.0,
      "logps/rejected": -428.0,
      "loss": 0.3716,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.0546875,
      "rewards/margins": 1.65625,
      "rewards/rejected": -2.71875,
      "step": 810
    },
    {
      "epoch": 0.1047990286919292,
      "grad_norm": 9.413822502760253,
      "learning_rate": 4.99965932940093e-07,
      "logits/chosen": -1.65625,
      "logits/rejected": -1.3125,
      "logps/chosen": -284.0,
      "logps/rejected": -464.0,
      "loss": 0.382,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.109375,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -2.921875,
      "step": 820
    },
    {
      "epoch": 0.10607706562719663,
      "grad_norm": 8.379828387685025,
      "learning_rate": 4.999450306160585e-07,
      "logits/chosen": -1.78125,
      "logits/rejected": -1.4921875,
      "logps/chosen": -314.0,
      "logps/rejected": -488.0,
      "loss": 0.3844,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.328125,
      "rewards/margins": 1.9765625,
      "rewards/rejected": -3.3125,
      "step": 830
    },
    {
      "epoch": 0.10735510256246406,
      "grad_norm": 8.558859471551255,
      "learning_rate": 4.999191523553715e-07,
      "logits/chosen": -1.8046875,
      "logits/rejected": -1.515625,
      "logps/chosen": -290.0,
      "logps/rejected": -454.0,
      "loss": 0.3834,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.99609375,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -2.75,
      "step": 840
    },
    {
      "epoch": 0.10863313949773148,
      "grad_norm": 10.086891504492751,
      "learning_rate": 4.998882986732195e-07,
      "logits/chosen": -1.6640625,
      "logits/rejected": -1.5,
      "logps/chosen": -306.0,
      "logps/rejected": -486.0,
      "loss": 0.3849,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.203125,
      "rewards/margins": 1.71875,
      "rewards/rejected": -2.921875,
      "step": 850
    },
    {
      "epoch": 0.10991117643299891,
      "grad_norm": 11.273042189751745,
      "learning_rate": 4.998524701838414e-07,
      "logits/chosen": -1.6875,
      "logits/rejected": -1.4609375,
      "logps/chosen": -284.0,
      "logps/rejected": -460.0,
      "loss": 0.3631,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.046875,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -2.859375,
      "step": 860
    },
    {
      "epoch": 0.11118921336826634,
      "grad_norm": 7.366753318792554,
      "learning_rate": 4.998116676005154e-07,
      "logits/chosen": -1.75,
      "logits/rejected": -1.5625,
      "logps/chosen": -308.0,
      "logps/rejected": -452.0,
      "loss": 0.3829,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.3359375,
      "rewards/margins": 1.625,
      "rewards/rejected": -2.953125,
      "step": 870
    },
    {
      "epoch": 0.11246725030353377,
      "grad_norm": 9.01550332393051,
      "learning_rate": 4.997658917355441e-07,
      "logits/chosen": -1.84375,
      "logits/rejected": -1.609375,
      "logps/chosen": -312.0,
      "logps/rejected": -486.0,
      "loss": 0.3644,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.25,
      "rewards/margins": 1.8125,
      "rewards/rejected": -3.0625,
      "step": 880
    },
    {
      "epoch": 0.1137452872388012,
      "grad_norm": 14.967847853168832,
      "learning_rate": 4.997151435002394e-07,
      "logits/chosen": -1.8046875,
      "logits/rejected": -1.5625,
      "logps/chosen": -336.0,
      "logps/rejected": -486.0,
      "loss": 0.3605,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.28125,
      "rewards/margins": 1.734375,
      "rewards/rejected": -3.015625,
      "step": 890
    },
    {
      "epoch": 0.11502332417406863,
      "grad_norm": 8.1830855296527,
      "learning_rate": 4.996594239049032e-07,
      "logits/chosen": -1.796875,
      "logits/rejected": -1.390625,
      "logps/chosen": -300.0,
      "logps/rejected": -454.0,
      "loss": 0.3867,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.203125,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -2.8125,
      "step": 900
    },
    {
      "epoch": 0.11630136110933606,
      "grad_norm": 9.974450677301276,
      "learning_rate": 4.995987340588082e-07,
      "logits/chosen": -1.8203125,
      "logits/rejected": -1.5234375,
      "logps/chosen": -330.0,
      "logps/rejected": -504.0,
      "loss": 0.351,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.328125,
      "rewards/margins": 1.953125,
      "rewards/rejected": -3.28125,
      "step": 910
    },
    {
      "epoch": 0.11757939804460349,
      "grad_norm": 6.577679078150954,
      "learning_rate": 4.995330751701755e-07,
      "logits/chosen": -1.7734375,
      "logits/rejected": -1.4296875,
      "logps/chosen": -304.0,
      "logps/rejected": -436.0,
      "loss": 0.3895,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3359375,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -2.921875,
      "step": 920
    },
    {
      "epoch": 0.11885743497987092,
      "grad_norm": 6.8165923621662206,
      "learning_rate": 4.994624485461503e-07,
      "logits/chosen": -1.71875,
      "logits/rejected": -1.46875,
      "logps/chosen": -290.0,
      "logps/rejected": -460.0,
      "loss": 0.3676,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.140625,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -2.859375,
      "step": 930
    },
    {
      "epoch": 0.12013547191513835,
      "grad_norm": 7.3214147693167035,
      "learning_rate": 4.99386855592776e-07,
      "logits/chosen": -1.8046875,
      "logits/rejected": -1.53125,
      "logps/chosen": -314.0,
      "logps/rejected": -490.0,
      "loss": 0.3507,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.3046875,
      "rewards/margins": 1.8671875,
      "rewards/rejected": -3.171875,
      "step": 940
    },
    {
      "epoch": 0.12141350885040578,
      "grad_norm": 9.164802728331658,
      "learning_rate": 4.993062978149668e-07,
      "logits/chosen": -1.75,
      "logits/rejected": -1.4453125,
      "logps/chosen": -316.0,
      "logps/rejected": -508.0,
      "loss": 0.363,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.3203125,
      "rewards/margins": 1.9140625,
      "rewards/rejected": -3.234375,
      "step": 950
    },
    {
      "epoch": 0.1226915457856732,
      "grad_norm": 8.342972886477973,
      "learning_rate": 4.992207768164769e-07,
      "logits/chosen": -1.7265625,
      "logits/rejected": -1.5,
      "logps/chosen": -290.0,
      "logps/rejected": -422.0,
      "loss": 0.3964,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.2421875,
      "rewards/margins": 1.4296875,
      "rewards/rejected": -2.671875,
      "step": 960
    },
    {
      "epoch": 0.12396958272094064,
      "grad_norm": 14.223530525362122,
      "learning_rate": 4.991302942998686e-07,
      "logits/chosen": -1.7578125,
      "logits/rejected": -1.515625,
      "logps/chosen": -304.0,
      "logps/rejected": -478.0,
      "loss": 0.3334,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.2890625,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -3.109375,
      "step": 970
    },
    {
      "epoch": 0.12524761965620806,
      "grad_norm": 9.96717338528781,
      "learning_rate": 4.990348520664794e-07,
      "logits/chosen": -1.8203125,
      "logits/rejected": -1.5625,
      "logps/chosen": -348.0,
      "logps/rejected": -486.0,
      "loss": 0.3613,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.5234375,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -3.21875,
      "step": 980
    },
    {
      "epoch": 0.1265256565914755,
      "grad_norm": 7.263697477315377,
      "learning_rate": 4.989344520163849e-07,
      "logits/chosen": -1.6796875,
      "logits/rejected": -1.4375,
      "logps/chosen": -280.0,
      "logps/rejected": -424.0,
      "loss": 0.3521,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.28125,
      "rewards/margins": 1.390625,
      "rewards/rejected": -2.671875,
      "step": 990
    },
    {
      "epoch": 0.12780369352674292,
      "grad_norm": 25.32445739207142,
      "learning_rate": 4.98829096148362e-07,
      "logits/chosen": -1.7734375,
      "logits/rejected": -1.46875,
      "logps/chosen": -334.0,
      "logps/rejected": -528.0,
      "loss": 0.3525,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.5,
      "rewards/margins": 2.046875,
      "rewards/rejected": -3.546875,
      "step": 1000
    },
    {
      "epoch": 0.12908173046201035,
      "grad_norm": 9.318037197433556,
      "learning_rate": 4.987187865598482e-07,
      "logits/chosen": -1.7421875,
      "logits/rejected": -1.4765625,
      "logps/chosen": -314.0,
      "logps/rejected": -482.0,
      "loss": 0.3457,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.359375,
      "rewards/margins": 1.984375,
      "rewards/rejected": -3.34375,
      "step": 1010
    },
    {
      "epoch": 0.13035976739727778,
      "grad_norm": 10.035043211708267,
      "learning_rate": 4.986035254469005e-07,
      "logits/chosen": -1.7421875,
      "logits/rejected": -1.6015625,
      "logps/chosen": -310.0,
      "logps/rejected": -506.0,
      "loss": 0.3625,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.4296875,
      "rewards/margins": 2.03125,
      "rewards/rejected": -3.46875,
      "step": 1020
    },
    {
      "epoch": 0.1316378043325452,
      "grad_norm": 8.827841498590644,
      "learning_rate": 4.984833151041512e-07,
      "logits/chosen": -1.7109375,
      "logits/rejected": -1.546875,
      "logps/chosen": -310.0,
      "logps/rejected": -524.0,
      "loss": 0.3488,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.359375,
      "rewards/margins": 2.171875,
      "rewards/rejected": -3.53125,
      "step": 1030
    },
    {
      "epoch": 0.13291584126781264,
      "grad_norm": 9.299845190410423,
      "learning_rate": 4.98358157924763e-07,
      "logits/chosen": -1.828125,
      "logits/rejected": -1.671875,
      "logps/chosen": -338.0,
      "logps/rejected": -564.0,
      "loss": 0.3303,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 2.203125,
      "rewards/rejected": -3.84375,
      "step": 1040
    },
    {
      "epoch": 0.13419387820308007,
      "grad_norm": 26.87268364477717,
      "learning_rate": 4.982280564003806e-07,
      "logits/chosen": -1.90625,
      "logits/rejected": -1.65625,
      "logps/chosen": -360.0,
      "logps/rejected": -492.0,
      "loss": 0.3395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.59375,
      "rewards/margins": 1.609375,
      "rewards/rejected": -3.203125,
      "step": 1050
    },
    {
      "epoch": 0.1354719151383475,
      "grad_norm": 8.464636707470351,
      "learning_rate": 4.98093013121081e-07,
      "logits/chosen": -1.8125,
      "logits/rejected": -1.6484375,
      "logps/chosen": -326.0,
      "logps/rejected": -494.0,
      "loss": 0.3297,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.34375,
      "rewards/margins": 1.9453125,
      "rewards/rejected": -3.28125,
      "step": 1060
    },
    {
      "epoch": 0.13674995207361493,
      "grad_norm": 8.284952785634527,
      "learning_rate": 4.979530307753227e-07,
      "logits/chosen": -1.7734375,
      "logits/rejected": -1.625,
      "logps/chosen": -320.0,
      "logps/rejected": -482.0,
      "loss": 0.3299,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.3671875,
      "rewards/margins": 1.84375,
      "rewards/rejected": -3.203125,
      "step": 1070
    },
    {
      "epoch": 0.13802798900888236,
      "grad_norm": 9.235579707840643,
      "learning_rate": 4.978081121498916e-07,
      "logits/chosen": -1.875,
      "logits/rejected": -1.6015625,
      "logps/chosen": -318.0,
      "logps/rejected": -498.0,
      "loss": 0.3377,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.4375,
      "rewards/margins": 1.8828125,
      "rewards/rejected": -3.328125,
      "step": 1080
    },
    {
      "epoch": 0.13930602594414979,
      "grad_norm": 10.506581657904592,
      "learning_rate": 4.976582601298456e-07,
      "logits/chosen": -1.75,
      "logits/rejected": -1.453125,
      "logps/chosen": -334.0,
      "logps/rejected": -516.0,
      "loss": 0.321,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.484375,
      "rewards/margins": 2.0,
      "rewards/rejected": -3.484375,
      "step": 1090
    },
    {
      "epoch": 0.14058406287941722,
      "grad_norm": 9.271267307316478,
      "learning_rate": 4.975034776984573e-07,
      "logits/chosen": -1.8671875,
      "logits/rejected": -1.6640625,
      "logps/chosen": -332.0,
      "logps/rejected": -474.0,
      "loss": 0.3299,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.4765625,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -3.109375,
      "step": 1100
    },
    {
      "epoch": 0.14186209981468464,
      "grad_norm": 8.734325905187168,
      "learning_rate": 4.973437679371546e-07,
      "logits/chosen": -1.8125,
      "logits/rejected": -1.546875,
      "logps/chosen": -308.0,
      "logps/rejected": -498.0,
      "loss": 0.3386,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.390625,
      "rewards/margins": 2.09375,
      "rewards/rejected": -3.484375,
      "step": 1110
    },
    {
      "epoch": 0.14314013674995207,
      "grad_norm": 11.514104100521278,
      "learning_rate": 4.971791340254592e-07,
      "logits/chosen": -1.78125,
      "logits/rejected": -1.6171875,
      "logps/chosen": -316.0,
      "logps/rejected": -464.0,
      "loss": 0.3611,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.34375,
      "rewards/margins": 1.7109375,
      "rewards/rejected": -3.046875,
      "step": 1120
    },
    {
      "epoch": 0.1444181736852195,
      "grad_norm": 13.432472212370357,
      "learning_rate": 4.970095792409233e-07,
      "logits/chosen": -1.8125,
      "logits/rejected": -1.53125,
      "logps/chosen": -328.0,
      "logps/rejected": -466.0,
      "loss": 0.328,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.3515625,
      "rewards/margins": 1.734375,
      "rewards/rejected": -3.09375,
      "step": 1130
    },
    {
      "epoch": 0.14569621062048693,
      "grad_norm": 9.984842677250851,
      "learning_rate": 4.968351069590646e-07,
      "logits/chosen": -1.8828125,
      "logits/rejected": -1.515625,
      "logps/chosen": -368.0,
      "logps/rejected": -572.0,
      "loss": 0.3425,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.796875,
      "rewards/margins": 2.296875,
      "rewards/rejected": -4.09375,
      "step": 1140
    },
    {
      "epoch": 0.14697424755575436,
      "grad_norm": 8.698875634428825,
      "learning_rate": 4.966557206532992e-07,
      "logits/chosen": -1.7890625,
      "logits/rejected": -1.5703125,
      "logps/chosen": -340.0,
      "logps/rejected": -492.0,
      "loss": 0.3702,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.46875,
      "rewards/margins": 1.75,
      "rewards/rejected": -3.21875,
      "step": 1150
    },
    {
      "epoch": 0.1482522844910218,
      "grad_norm": 10.15701341755144,
      "learning_rate": 4.964714238948715e-07,
      "logits/chosen": -1.796875,
      "logits/rejected": -1.5703125,
      "logps/chosen": -346.0,
      "logps/rejected": -528.0,
      "loss": 0.3403,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 1.8828125,
      "rewards/rejected": -3.515625,
      "step": 1160
    },
    {
      "epoch": 0.14953032142628922,
      "grad_norm": 7.990135589889346,
      "learning_rate": 4.962822203527845e-07,
      "logits/chosen": -1.8125,
      "logits/rejected": -1.5390625,
      "logps/chosen": -350.0,
      "logps/rejected": -528.0,
      "loss": 0.3174,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 1.953125,
      "rewards/rejected": -3.671875,
      "step": 1170
    },
    {
      "epoch": 0.15080835836155665,
      "grad_norm": 10.739318059363967,
      "learning_rate": 4.960881137937256e-07,
      "logits/chosen": -1.8515625,
      "logits/rejected": -1.640625,
      "logps/chosen": -330.0,
      "logps/rejected": -524.0,
      "loss": 0.3269,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.4765625,
      "rewards/margins": 2.0625,
      "rewards/rejected": -3.546875,
      "step": 1180
    },
    {
      "epoch": 0.15208639529682408,
      "grad_norm": 8.790608340603956,
      "learning_rate": 4.958891080819923e-07,
      "logits/chosen": -1.890625,
      "logits/rejected": -1.7890625,
      "logps/chosen": -352.0,
      "logps/rejected": -520.0,
      "loss": 0.3416,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.828125,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -3.578125,
      "step": 1190
    },
    {
      "epoch": 0.1533644322320915,
      "grad_norm": 10.917026729040735,
      "learning_rate": 4.956852071794151e-07,
      "logits/chosen": -1.8125,
      "logits/rejected": -1.5390625,
      "logps/chosen": -332.0,
      "logps/rejected": -484.0,
      "loss": 0.3488,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.3828125,
      "rewards/margins": 1.8125,
      "rewards/rejected": -3.203125,
      "step": 1200
    },
    {
      "epoch": 0.15464246916735894,
      "grad_norm": 7.987752032175459,
      "learning_rate": 4.954764151452782e-07,
      "logits/chosen": -1.828125,
      "logits/rejected": -1.6640625,
      "logps/chosen": -334.0,
      "logps/rejected": -544.0,
      "loss": 0.3166,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 2.15625,
      "rewards/rejected": -3.734375,
      "step": 1210
    },
    {
      "epoch": 0.15592050610262637,
      "grad_norm": 8.995958841495346,
      "learning_rate": 4.952627361362395e-07,
      "logits/chosen": -1.875,
      "logits/rejected": -1.6640625,
      "logps/chosen": -362.0,
      "logps/rejected": -560.0,
      "loss": 0.339,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 1.96875,
      "rewards/rejected": -3.921875,
      "step": 1220
    },
    {
      "epoch": 0.1571985430378938,
      "grad_norm": 9.55825678258976,
      "learning_rate": 4.950441744062471e-07,
      "logits/chosen": -1.8984375,
      "logits/rejected": -1.703125,
      "logps/chosen": -338.0,
      "logps/rejected": -536.0,
      "loss": 0.3372,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.546875,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -3.453125,
      "step": 1230
    },
    {
      "epoch": 0.15847657997316122,
      "grad_norm": 8.13681539565613,
      "learning_rate": 4.948207343064551e-07,
      "logits/chosen": -1.84375,
      "logits/rejected": -1.703125,
      "logps/chosen": -348.0,
      "logps/rejected": -568.0,
      "loss": 0.3241,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.703125,
      "rewards/margins": 2.359375,
      "rewards/rejected": -4.0625,
      "step": 1240
    },
    {
      "epoch": 0.15975461690842865,
      "grad_norm": 12.00249925049519,
      "learning_rate": 4.945924202851366e-07,
      "logits/chosen": -1.8828125,
      "logits/rejected": -1.625,
      "logps/chosen": -358.0,
      "logps/rejected": -580.0,
      "loss": 0.3255,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 2.4375,
      "rewards/rejected": -4.09375,
      "step": 1250
    },
    {
      "epoch": 0.16103265384369608,
      "grad_norm": 9.072313429586368,
      "learning_rate": 4.943592368875955e-07,
      "logits/chosen": -1.8828125,
      "logits/rejected": -1.609375,
      "logps/chosen": -348.0,
      "logps/rejected": -524.0,
      "loss": 0.349,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -3.59375,
      "step": 1260
    },
    {
      "epoch": 0.1623106907789635,
      "grad_norm": 10.89915653387236,
      "learning_rate": 4.941211887560757e-07,
      "logits/chosen": -1.8828125,
      "logits/rejected": -1.671875,
      "logps/chosen": -380.0,
      "logps/rejected": -580.0,
      "loss": 0.2906,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 2.125,
      "rewards/rejected": -3.984375,
      "step": 1270
    },
    {
      "epoch": 0.16358872771423094,
      "grad_norm": 8.773512793257732,
      "learning_rate": 4.938782806296691e-07,
      "logits/chosen": -1.78125,
      "logits/rejected": -1.5859375,
      "logps/chosen": -376.0,
      "logps/rejected": -532.0,
      "loss": 0.3032,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 1.8046875,
      "rewards/rejected": -3.75,
      "step": 1280
    },
    {
      "epoch": 0.16486676464949837,
      "grad_norm": 11.705696163482397,
      "learning_rate": 4.936305173442206e-07,
      "logits/chosen": -1.8359375,
      "logits/rejected": -1.7109375,
      "logps/chosen": -354.0,
      "logps/rejected": -552.0,
      "loss": 0.3113,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.640625,
      "rewards/margins": 2.265625,
      "rewards/rejected": -3.90625,
      "step": 1290
    },
    {
      "epoch": 0.1661448015847658,
      "grad_norm": 8.1878727745444,
      "learning_rate": 4.933779038322324e-07,
      "logits/chosen": -1.75,
      "logits/rejected": -1.578125,
      "logps/chosen": -350.0,
      "logps/rejected": -600.0,
      "loss": 0.3141,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 2.59375,
      "rewards/rejected": -4.25,
      "step": 1300
    },
    {
      "epoch": 0.16742283852003323,
      "grad_norm": 9.336486450909815,
      "learning_rate": 4.931204451227658e-07,
      "logits/chosen": -1.8125,
      "logits/rejected": -1.703125,
      "logps/chosen": -352.0,
      "logps/rejected": -532.0,
      "loss": 0.3216,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.65625,
      "rewards/margins": 1.9296875,
      "rewards/rejected": -3.59375,
      "step": 1310
    },
    {
      "epoch": 0.16870087545530066,
      "grad_norm": 10.074589538360476,
      "learning_rate": 4.928581463413405e-07,
      "logits/chosen": -1.859375,
      "logits/rejected": -1.625,
      "logps/chosen": -336.0,
      "logps/rejected": -532.0,
      "loss": 0.332,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.6875,
      "rewards/margins": 2.0,
      "rewards/rejected": -3.6875,
      "step": 1320
    },
    {
      "epoch": 0.1699789123905681,
      "grad_norm": 8.799236105738936,
      "learning_rate": 4.925910127098333e-07,
      "logits/chosen": -1.9453125,
      "logits/rejected": -1.625,
      "logps/chosen": -348.0,
      "logps/rejected": -552.0,
      "loss": 0.3154,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 2.296875,
      "rewards/rejected": -4.0,
      "step": 1330
    },
    {
      "epoch": 0.17125694932583552,
      "grad_norm": 8.871356132954851,
      "learning_rate": 4.923190495463736e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.7578125,
      "logps/chosen": -380.0,
      "logps/rejected": -588.0,
      "loss": 0.2999,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 2.203125,
      "rewards/rejected": -4.125,
      "step": 1340
    },
    {
      "epoch": 0.17253498626110295,
      "grad_norm": 10.447819183633786,
      "learning_rate": 4.920422622652377e-07,
      "logits/chosen": -1.890625,
      "logits/rejected": -1.6796875,
      "logps/chosen": -396.0,
      "logps/rejected": -580.0,
      "loss": 0.3352,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.03125,
      "rewards/margins": 2.03125,
      "rewards/rejected": -4.0625,
      "step": 1350
    },
    {
      "epoch": 0.17381302319637038,
      "grad_norm": 9.670031173791987,
      "learning_rate": 4.917606563767411e-07,
      "logits/chosen": -1.84375,
      "logits/rejected": -1.6953125,
      "logps/chosen": -386.0,
      "logps/rejected": -620.0,
      "loss": 0.3193,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.765625,
      "rewards/margins": 2.65625,
      "rewards/rejected": -4.40625,
      "step": 1360
    },
    {
      "epoch": 0.1750910601316378,
      "grad_norm": 11.221875701350609,
      "learning_rate": 4.914742374871289e-07,
      "logits/chosen": -1.890625,
      "logits/rejected": -1.6171875,
      "logps/chosen": -364.0,
      "logps/rejected": -540.0,
      "loss": 0.3055,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.890625,
      "rewards/margins": 1.9453125,
      "rewards/rejected": -3.828125,
      "step": 1370
    },
    {
      "epoch": 0.17636909706690523,
      "grad_norm": 11.042366265928507,
      "learning_rate": 4.911830112984638e-07,
      "logits/chosen": -1.9296875,
      "logits/rejected": -1.8203125,
      "logps/chosen": -362.0,
      "logps/rejected": -524.0,
      "loss": 0.3161,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.828125,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -3.5625,
      "step": 1380
    },
    {
      "epoch": 0.17764713400217266,
      "grad_norm": 7.999382609881203,
      "learning_rate": 4.908869836085127e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.8046875,
      "logps/chosen": -374.0,
      "logps/rejected": -584.0,
      "loss": 0.2983,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 2.28125,
      "rewards/rejected": -4.21875,
      "step": 1390
    },
    {
      "epoch": 0.1789251709374401,
      "grad_norm": 10.762813175491655,
      "learning_rate": 4.905861603106318e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.7734375,
      "logps/chosen": -406.0,
      "logps/rejected": -636.0,
      "loss": 0.3259,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.046875,
      "rewards/margins": 2.515625,
      "rewards/rejected": -4.5625,
      "step": 1400
    },
    {
      "epoch": 0.18020320787270752,
      "grad_norm": 15.057860048577368,
      "learning_rate": 4.902805473936483e-07,
      "logits/chosen": -1.984375,
      "logits/rejected": -1.6875,
      "logps/chosen": -368.0,
      "logps/rejected": -600.0,
      "loss": 0.2914,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 2.34375,
      "rewards/rejected": -4.15625,
      "step": 1410
    },
    {
      "epoch": 0.18148124480797495,
      "grad_norm": 9.344505682460305,
      "learning_rate": 4.899701509417423e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.796875,
      "logps/chosen": -366.0,
      "logps/rejected": -616.0,
      "loss": 0.3094,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 2.625,
      "rewards/rejected": -4.46875,
      "step": 1420
    },
    {
      "epoch": 0.18275928174324238,
      "grad_norm": 10.902136842245536,
      "learning_rate": 4.896549771343247e-07,
      "logits/chosen": -1.984375,
      "logits/rejected": -1.765625,
      "logps/chosen": -372.0,
      "logps/rejected": -600.0,
      "loss": 0.306,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.015625,
      "rewards/margins": 2.40625,
      "rewards/rejected": -4.40625,
      "step": 1430
    },
    {
      "epoch": 0.1840373186785098,
      "grad_norm": 8.87306010554798,
      "learning_rate": 4.893350322459149e-07,
      "logits/chosen": -1.9765625,
      "logits/rejected": -1.8515625,
      "logps/chosen": -372.0,
      "logps/rejected": -552.0,
      "loss": 0.3145,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.7578125,
      "rewards/margins": 2.015625,
      "rewards/rejected": -3.78125,
      "step": 1440
    },
    {
      "epoch": 0.18531535561377724,
      "grad_norm": 7.591625004900557,
      "learning_rate": 4.890103226460152e-07,
      "logits/chosen": -1.9140625,
      "logits/rejected": -1.8203125,
      "logps/chosen": -364.0,
      "logps/rejected": -564.0,
      "loss": 0.3036,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.7734375,
      "rewards/margins": 2.109375,
      "rewards/rejected": -3.890625,
      "step": 1450
    },
    {
      "epoch": 0.18659339254904467,
      "grad_norm": 8.618147277122679,
      "learning_rate": 4.886808547989846e-07,
      "logits/chosen": -1.984375,
      "logits/rejected": -1.796875,
      "logps/chosen": -374.0,
      "logps/rejected": -564.0,
      "loss": 0.3217,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.0,
      "rewards/margins": 2.078125,
      "rewards/rejected": -4.0625,
      "step": 1460
    },
    {
      "epoch": 0.1878714294843121,
      "grad_norm": 9.07004217164032,
      "learning_rate": 4.883466352639099e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -1.8125,
      "logps/chosen": -398.0,
      "logps/rejected": -604.0,
      "loss": 0.317,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 2.28125,
      "rewards/rejected": -4.21875,
      "step": 1470
    },
    {
      "epoch": 0.18914946641957953,
      "grad_norm": 11.480663082502499,
      "learning_rate": 4.88007670694475e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.8046875,
      "logps/chosen": -370.0,
      "logps/rejected": -600.0,
      "loss": 0.3004,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 2.53125,
      "rewards/rejected": -4.28125,
      "step": 1480
    },
    {
      "epoch": 0.19042750335484696,
      "grad_norm": 10.635609676655566,
      "learning_rate": 4.876639678388285e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.7265625,
      "logps/chosen": -380.0,
      "logps/rejected": -636.0,
      "loss": 0.3186,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.875,
      "rewards/margins": 2.828125,
      "rewards/rejected": -4.6875,
      "step": 1490
    },
    {
      "epoch": 0.19170554029011438,
      "grad_norm": 8.418336047015984,
      "learning_rate": 4.873155335394497e-07,
      "logits/chosen": -1.96875,
      "logits/rejected": -1.71875,
      "logps/chosen": -352.0,
      "logps/rejected": -560.0,
      "loss": 0.3058,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.7578125,
      "rewards/margins": 2.265625,
      "rewards/rejected": -4.03125,
      "step": 1500
    },
    {
      "epoch": 0.1929835772253818,
      "grad_norm": 9.127049388471088,
      "learning_rate": 4.869623747330116e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.765625,
      "logps/chosen": -386.0,
      "logps/rejected": -580.0,
      "loss": 0.3131,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.015625,
      "rewards/margins": 2.34375,
      "rewards/rejected": -4.34375,
      "step": 1510
    },
    {
      "epoch": 0.19426161416064924,
      "grad_norm": 10.559771971672896,
      "learning_rate": 4.866044984502436e-07,
      "logits/chosen": -1.96875,
      "logits/rejected": -1.8125,
      "logps/chosen": -358.0,
      "logps/rejected": -612.0,
      "loss": 0.2916,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.75,
      "rewards/margins": 2.515625,
      "rewards/rejected": -4.28125,
      "step": 1520
    },
    {
      "epoch": 0.19553965109591667,
      "grad_norm": 10.474797534750943,
      "learning_rate": 4.862419118157909e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.8359375,
      "logps/chosen": -356.0,
      "logps/rejected": -556.0,
      "loss": 0.3003,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 2.078125,
      "rewards/rejected": -3.90625,
      "step": 1530
    },
    {
      "epoch": 0.1968176880311841,
      "grad_norm": 11.943955819492983,
      "learning_rate": 4.858746220480734e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.7734375,
      "logps/chosen": -372.0,
      "logps/rejected": -580.0,
      "loss": 0.3209,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.84375,
      "rewards/margins": 2.296875,
      "rewards/rejected": -4.125,
      "step": 1540
    },
    {
      "epoch": 0.19809572496645153,
      "grad_norm": 18.259331826096556,
      "learning_rate": 4.855026364591413e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.875,
      "logps/chosen": -392.0,
      "logps/rejected": -600.0,
      "loss": 0.2976,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 2.234375,
      "rewards/rejected": -4.15625,
      "step": 1550
    },
    {
      "epoch": 0.19937376190171896,
      "grad_norm": 10.14281851505766,
      "learning_rate": 4.851259624545297e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.828125,
      "logps/chosen": -378.0,
      "logps/rejected": -612.0,
      "loss": 0.3065,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.859375,
      "rewards/margins": 2.5,
      "rewards/rejected": -4.375,
      "step": 1560
    },
    {
      "epoch": 0.2006517988369864,
      "grad_norm": 9.90745447776379,
      "learning_rate": 4.847446075331114e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.78125,
      "logps/chosen": -350.0,
      "logps/rejected": -604.0,
      "loss": 0.3097,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.6875,
      "rewards/margins": 2.671875,
      "rewards/rejected": -4.375,
      "step": 1570
    },
    {
      "epoch": 0.20192983577225382,
      "grad_norm": 14.171024556856327,
      "learning_rate": 4.843585792869476e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -1.875,
      "logps/chosen": -366.0,
      "logps/rejected": -588.0,
      "loss": 0.3156,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 2.390625,
      "rewards/rejected": -4.3125,
      "step": 1580
    },
    {
      "epoch": 0.20320787270752125,
      "grad_norm": 9.896906083587025,
      "learning_rate": 4.839678854011362e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -2.015625,
      "logps/chosen": -368.0,
      "logps/rejected": -556.0,
      "loss": 0.3182,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 2.0625,
      "rewards/rejected": -3.875,
      "step": 1590
    },
    {
      "epoch": 0.20448590964278868,
      "grad_norm": 9.24381584130229,
      "learning_rate": 4.835725336536598e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.890625,
      "logps/chosen": -356.0,
      "logps/rejected": -564.0,
      "loss": 0.3155,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.765625,
      "rewards/margins": 2.28125,
      "rewards/rejected": -4.0625,
      "step": 1600
    },
    {
      "epoch": 0.2057639465780561,
      "grad_norm": 17.14761646357549,
      "learning_rate": 4.831725319152298e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.9375,
      "logps/chosen": -366.0,
      "logps/rejected": -604.0,
      "loss": 0.299,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.90625,
      "rewards/margins": 2.46875,
      "rewards/rejected": -4.375,
      "step": 1610
    },
    {
      "epoch": 0.20704198351332354,
      "grad_norm": 12.244087069129968,
      "learning_rate": 4.827678881491305e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.859375,
      "logps/chosen": -408.0,
      "logps/rejected": -676.0,
      "loss": 0.2768,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.234375,
      "rewards/margins": 2.5,
      "rewards/rejected": -4.75,
      "step": 1620
    },
    {
      "epoch": 0.20832002044859096,
      "grad_norm": 7.95172276398267,
      "learning_rate": 4.8235861041106e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -1.921875,
      "logps/chosen": -372.0,
      "logps/rejected": -584.0,
      "loss": 0.2805,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 2.28125,
      "rewards/rejected": -4.21875,
      "step": 1630
    },
    {
      "epoch": 0.2095980573838584,
      "grad_norm": 8.196845897150656,
      "learning_rate": 4.819447068489705e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -2.015625,
      "logps/chosen": -404.0,
      "logps/rejected": -632.0,
      "loss": 0.2867,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.078125,
      "rewards/margins": 2.40625,
      "rewards/rejected": -4.5,
      "step": 1640
    },
    {
      "epoch": 0.21087609431912582,
      "grad_norm": 12.502557510515002,
      "learning_rate": 4.815261857029052e-07,
      "logits/chosen": -2.046875,
      "logits/rejected": -1.9375,
      "logps/chosen": -370.0,
      "logps/rejected": -596.0,
      "loss": 0.291,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.890625,
      "rewards/margins": 2.5625,
      "rewards/rejected": -4.4375,
      "step": 1650
    },
    {
      "epoch": 0.21215413125439325,
      "grad_norm": 11.65576722487078,
      "learning_rate": 4.811030553048351e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -2.015625,
      "logps/chosen": -398.0,
      "logps/rejected": -616.0,
      "loss": 0.2922,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.0625,
      "rewards/margins": 2.359375,
      "rewards/rejected": -4.40625,
      "step": 1660
    },
    {
      "epoch": 0.21343216818966068,
      "grad_norm": 11.819684637346922,
      "learning_rate": 4.806753240784924e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.0625,
      "logps/chosen": -408.0,
      "logps/rejected": -676.0,
      "loss": 0.2954,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.140625,
      "rewards/margins": 2.75,
      "rewards/rejected": -4.875,
      "step": 1670
    },
    {
      "epoch": 0.2147102051249281,
      "grad_norm": 13.173632872807703,
      "learning_rate": 4.802430005392037e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.9453125,
      "logps/chosen": -372.0,
      "logps/rejected": -568.0,
      "loss": 0.3239,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.078125,
      "rewards/margins": 2.1875,
      "rewards/rejected": -4.25,
      "step": 1680
    },
    {
      "epoch": 0.21598824206019554,
      "grad_norm": 11.433487771208389,
      "learning_rate": 4.798060932937194e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -2.0625,
      "logps/chosen": -390.0,
      "logps/rejected": -572.0,
      "loss": 0.2969,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.03125,
      "rewards/margins": 2.078125,
      "rewards/rejected": -4.125,
      "step": 1690
    },
    {
      "epoch": 0.21726627899546297,
      "grad_norm": 13.195411332493167,
      "learning_rate": 4.79364611040043e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.953125,
      "logps/chosen": -384.0,
      "logps/rejected": -636.0,
      "loss": 0.2901,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.0625,
      "rewards/margins": 2.671875,
      "rewards/rejected": -4.75,
      "step": 1700
    },
    {
      "epoch": 0.2185443159307304,
      "grad_norm": 7.598136416164986,
      "learning_rate": 4.789185625672583e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -1.9609375,
      "logps/chosen": -428.0,
      "logps/rejected": -660.0,
      "loss": 0.2952,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.390625,
      "rewards/margins": 2.484375,
      "rewards/rejected": -4.875,
      "step": 1710
    },
    {
      "epoch": 0.21982235286599783,
      "grad_norm": 11.396308764545081,
      "learning_rate": 4.784679567553531e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -1.9296875,
      "logps/chosen": -382.0,
      "logps/rejected": -628.0,
      "loss": 0.2893,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 2.625,
      "rewards/rejected": -4.59375,
      "step": 1720
    },
    {
      "epoch": 0.22110038980126526,
      "grad_norm": 9.36417642806668,
      "learning_rate": 4.780128025750442e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.9921875,
      "logps/chosen": -396.0,
      "logps/rejected": -612.0,
      "loss": 0.2951,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.203125,
      "rewards/margins": 2.25,
      "rewards/rejected": -4.46875,
      "step": 1730
    },
    {
      "epoch": 0.22237842673653269,
      "grad_norm": 13.929973464920243,
      "learning_rate": 4.775531090875971e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.015625,
      "logps/chosen": -416.0,
      "logps/rejected": -636.0,
      "loss": 0.3067,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.234375,
      "rewards/margins": 2.421875,
      "rewards/rejected": -4.65625,
      "step": 1740
    },
    {
      "epoch": 0.22365646367180012,
      "grad_norm": 13.077656829978261,
      "learning_rate": 4.770888854446471e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -2.046875,
      "logps/chosen": -386.0,
      "logps/rejected": -616.0,
      "loss": 0.2971,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.078125,
      "rewards/margins": 2.609375,
      "rewards/rejected": -4.6875,
      "step": 1750
    },
    {
      "epoch": 0.22493450060706754,
      "grad_norm": 8.870431434815218,
      "learning_rate": 4.766201408880156e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.125,
      "logps/chosen": -408.0,
      "logps/rejected": -596.0,
      "loss": 0.2993,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.15625,
      "rewards/margins": 2.140625,
      "rewards/rejected": -4.3125,
      "step": 1760
    },
    {
      "epoch": 0.22621253754233497,
      "grad_norm": 10.21525053940827,
      "learning_rate": 4.761468847495277e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -1.875,
      "logps/chosen": -422.0,
      "logps/rejected": -648.0,
      "loss": 0.2816,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.28125,
      "rewards/margins": 2.40625,
      "rewards/rejected": -4.6875,
      "step": 1770
    },
    {
      "epoch": 0.2274905744776024,
      "grad_norm": 7.511297370302717,
      "learning_rate": 4.756691264508251e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.9296875,
      "logps/chosen": -402.0,
      "logps/rejected": -632.0,
      "loss": 0.2936,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.203125,
      "rewards/margins": 2.328125,
      "rewards/rejected": -4.53125,
      "step": 1780
    },
    {
      "epoch": 0.22876861141286983,
      "grad_norm": 8.958901047884922,
      "learning_rate": 4.751868755031793e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.9453125,
      "logps/chosen": -378.0,
      "logps/rejected": -644.0,
      "loss": 0.3017,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.046875,
      "rewards/margins": 2.6875,
      "rewards/rejected": -4.71875,
      "step": 1790
    },
    {
      "epoch": 0.23004664834813726,
      "grad_norm": 9.752191035059079,
      "learning_rate": 4.747001415073018e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -2.015625,
      "logps/chosen": -402.0,
      "logps/rejected": -600.0,
      "loss": 0.2987,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.015625,
      "rewards/margins": 2.328125,
      "rewards/rejected": -4.34375,
      "step": 1800
    },
    {
      "epoch": 0.2313246852834047,
      "grad_norm": 9.845826365642544,
      "learning_rate": 4.742089341531535e-07,
      "logits/chosen": -2.046875,
      "logits/rejected": -1.9453125,
      "logps/chosen": -384.0,
      "logps/rejected": -600.0,
      "loss": 0.2864,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.03125,
      "rewards/margins": 2.40625,
      "rewards/rejected": -4.4375,
      "step": 1810
    },
    {
      "epoch": 0.23260272221867212,
      "grad_norm": 10.951837846829397,
      "learning_rate": 4.7371326321975127e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.9765625,
      "logps/chosen": -404.0,
      "logps/rejected": -688.0,
      "loss": 0.2969,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.296875,
      "rewards/margins": 2.828125,
      "rewards/rejected": -5.125,
      "step": 1820
    },
    {
      "epoch": 0.23388075915393955,
      "grad_norm": 14.506581360915282,
      "learning_rate": 4.7321313857497336e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.03125,
      "logps/chosen": -400.0,
      "logps/rejected": -664.0,
      "loss": 0.2983,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.203125,
      "rewards/margins": 2.625,
      "rewards/rejected": -4.84375,
      "step": 1830
    },
    {
      "epoch": 0.23515879608920698,
      "grad_norm": 9.98163017378731,
      "learning_rate": 4.7270857017536335e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -1.953125,
      "logps/chosen": -420.0,
      "logps/rejected": -668.0,
      "loss": 0.2991,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.375,
      "rewards/margins": 2.65625,
      "rewards/rejected": -5.03125,
      "step": 1840
    },
    {
      "epoch": 0.2364368330244744,
      "grad_norm": 11.796643964407641,
      "learning_rate": 4.7219956806593143e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.015625,
      "logps/chosen": -420.0,
      "logps/rejected": -636.0,
      "loss": 0.3037,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.3125,
      "rewards/margins": 2.421875,
      "rewards/rejected": -4.71875,
      "step": 1850
    },
    {
      "epoch": 0.23771486995974184,
      "grad_norm": 12.07357485760213,
      "learning_rate": 4.716861423799546e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -2.0,
      "logps/chosen": -438.0,
      "logps/rejected": -656.0,
      "loss": 0.283,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.390625,
      "rewards/margins": 2.546875,
      "rewards/rejected": -4.9375,
      "step": 1860
    },
    {
      "epoch": 0.23899290689500927,
      "grad_norm": 12.583585685951734,
      "learning_rate": 4.711683033387752e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -1.9765625,
      "logps/chosen": -404.0,
      "logps/rejected": -684.0,
      "loss": 0.2827,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.21875,
      "rewards/margins": 2.703125,
      "rewards/rejected": -4.9375,
      "step": 1870
    },
    {
      "epoch": 0.2402709438302767,
      "grad_norm": 10.432291296377514,
      "learning_rate": 4.70646061251597e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -1.9921875,
      "logps/chosen": -416.0,
      "logps/rejected": -684.0,
      "loss": 0.2821,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.265625,
      "rewards/margins": 2.875,
      "rewards/rejected": -5.15625,
      "step": 1880
    },
    {
      "epoch": 0.24154898076554412,
      "grad_norm": 9.853074532263726,
      "learning_rate": 4.701194265152802e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.8359375,
      "logps/chosen": -356.0,
      "logps/rejected": -572.0,
      "loss": 0.2881,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 2.421875,
      "rewards/rejected": -4.21875,
      "step": 1890
    },
    {
      "epoch": 0.24282701770081155,
      "grad_norm": 16.26864960200766,
      "learning_rate": 4.6958840961413447e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.015625,
      "logps/chosen": -400.0,
      "logps/rejected": -604.0,
      "loss": 0.2977,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.203125,
      "rewards/margins": 2.34375,
      "rewards/rejected": -4.53125,
      "step": 1900
    },
    {
      "epoch": 0.24410505463607898,
      "grad_norm": 12.094311331286951,
      "learning_rate": 4.6905302111971004e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.96875,
      "logps/chosen": -416.0,
      "logps/rejected": -656.0,
      "loss": 0.2932,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.203125,
      "rewards/margins": 2.640625,
      "rewards/rejected": -4.84375,
      "step": 1910
    },
    {
      "epoch": 0.2453830915713464,
      "grad_norm": 16.712376845423194,
      "learning_rate": 4.6851327169058735e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -2.0,
      "logps/chosen": -410.0,
      "logps/rejected": -608.0,
      "loss": 0.3077,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.09375,
      "rewards/margins": 2.3125,
      "rewards/rejected": -4.40625,
      "step": 1920
    },
    {
      "epoch": 0.24666112850661384,
      "grad_norm": 8.968098811619914,
      "learning_rate": 4.679691720721651e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -1.9921875,
      "logps/chosen": -446.0,
      "logps/rejected": -644.0,
      "loss": 0.285,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.53125,
      "rewards/margins": 2.28125,
      "rewards/rejected": -4.8125,
      "step": 1930
    },
    {
      "epoch": 0.24793916544188127,
      "grad_norm": 10.303974330567755,
      "learning_rate": 4.674207330964458e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -2.015625,
      "logps/chosen": -408.0,
      "logps/rejected": -648.0,
      "loss": 0.2943,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.296875,
      "rewards/margins": 2.484375,
      "rewards/rejected": -4.78125,
      "step": 1940
    },
    {
      "epoch": 0.2492172023771487,
      "grad_norm": 9.116375291122598,
      "learning_rate": 4.668679656818207e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -2.125,
      "logps/chosen": -422.0,
      "logps/rejected": -672.0,
      "loss": 0.2822,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.390625,
      "rewards/margins": 2.546875,
      "rewards/rejected": -4.9375,
      "step": 1950
    },
    {
      "epoch": 0.25049523931241613,
      "grad_norm": 10.154366578606442,
      "learning_rate": 4.663108808328519e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.125,
      "logps/chosen": -428.0,
      "logps/rejected": -616.0,
      "loss": 0.3024,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.5,
      "rewards/margins": 2.125,
      "rewards/rejected": -4.625,
      "step": 1960
    },
    {
      "epoch": 0.25177327624768353,
      "grad_norm": 9.714032273252545,
      "learning_rate": 4.6574948964005367e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -1.9453125,
      "logps/chosen": -404.0,
      "logps/rejected": -600.0,
      "loss": 0.3077,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.296875,
      "rewards/margins": 2.21875,
      "rewards/rejected": -4.5,
      "step": 1970
    },
    {
      "epoch": 0.253051313182951,
      "grad_norm": 7.276857074842524,
      "learning_rate": 4.6518380327967145e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.21875,
      "logps/chosen": -396.0,
      "logps/rejected": -640.0,
      "loss": 0.2545,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.265625,
      "rewards/margins": 2.40625,
      "rewards/rejected": -4.65625,
      "step": 1980
    },
    {
      "epoch": 0.2543293501182184,
      "grad_norm": 10.43923738252122,
      "learning_rate": 4.6461383301345947e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.15625,
      "logps/chosen": -430.0,
      "logps/rejected": -672.0,
      "loss": 0.2749,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.5,
      "rewards/margins": 2.4375,
      "rewards/rejected": -4.9375,
      "step": 1990
    },
    {
      "epoch": 0.25560738705348585,
      "grad_norm": 12.056287834554103,
      "learning_rate": 4.6403959018845637e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.125,
      "logps/chosen": -400.0,
      "logps/rejected": -660.0,
      "loss": 0.2729,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.15625,
      "rewards/margins": 2.6875,
      "rewards/rejected": -4.84375,
      "step": 2000
    },
    {
      "epoch": 0.25688542398875325,
      "grad_norm": 11.852707026412647,
      "learning_rate": 4.6346108623675954e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.015625,
      "logps/chosen": -406.0,
      "logps/rejected": -648.0,
      "loss": 0.2987,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.1875,
      "rewards/margins": 2.609375,
      "rewards/rejected": -4.8125,
      "step": 2010
    },
    {
      "epoch": 0.2581634609240207,
      "grad_norm": 10.513655947330705,
      "learning_rate": 4.628783326752974e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.125,
      "logps/chosen": -398.0,
      "logps/rejected": -620.0,
      "loss": 0.2844,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.0625,
      "rewards/margins": 2.453125,
      "rewards/rejected": -4.5,
      "step": 2020
    },
    {
      "epoch": 0.2594414978592881,
      "grad_norm": 12.288711339432558,
      "learning_rate": 4.622913411056e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.9921875,
      "logps/chosen": -392.0,
      "logps/rejected": -672.0,
      "loss": 0.2724,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.203125,
      "rewards/margins": 2.84375,
      "rewards/rejected": -5.03125,
      "step": 2030
    },
    {
      "epoch": 0.26071953479455556,
      "grad_norm": 10.32299990977189,
      "learning_rate": 4.617001232135684e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -2.125,
      "logps/chosen": -412.0,
      "logps/rejected": -656.0,
      "loss": 0.294,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.40625,
      "rewards/margins": 2.515625,
      "rewards/rejected": -4.9375,
      "step": 2040
    },
    {
      "epoch": 0.26199757172982296,
      "grad_norm": 12.088122060546333,
      "learning_rate": 4.6110469076924153e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.109375,
      "logps/chosen": -470.0,
      "logps/rejected": -732.0,
      "loss": 0.27,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.671875,
      "rewards/margins": 2.921875,
      "rewards/rejected": -5.59375,
      "step": 2050
    },
    {
      "epoch": 0.2632756086650904,
      "grad_norm": 9.976628438368381,
      "learning_rate": 4.605050556265624e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.140625,
      "logps/chosen": -466.0,
      "logps/rejected": -688.0,
      "loss": 0.2865,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.71875,
      "rewards/margins": 2.453125,
      "rewards/rejected": -5.15625,
      "step": 2060
    },
    {
      "epoch": 0.2645536456003578,
      "grad_norm": 10.44034404911423,
      "learning_rate": 4.599012297231417e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.0625,
      "logps/chosen": -398.0,
      "logps/rejected": -688.0,
      "loss": 0.2695,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.140625,
      "rewards/margins": 2.859375,
      "rewards/rejected": -5.0,
      "step": 2070
    },
    {
      "epoch": 0.2658316825356253,
      "grad_norm": 9.534673448913239,
      "learning_rate": 4.5929322508002045e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -1.9921875,
      "logps/chosen": -432.0,
      "logps/rejected": -632.0,
      "loss": 0.289,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.359375,
      "rewards/margins": 2.46875,
      "rewards/rejected": -4.8125,
      "step": 2080
    },
    {
      "epoch": 0.2671097194708927,
      "grad_norm": 9.35051832470006,
      "learning_rate": 4.586810538014304e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.25,
      "logps/chosen": -420.0,
      "logps/rejected": -652.0,
      "loss": 0.2789,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.40625,
      "rewards/margins": 2.59375,
      "rewards/rejected": -5.0,
      "step": 2090
    },
    {
      "epoch": 0.26838775640616014,
      "grad_norm": 12.046777575913504,
      "learning_rate": 4.580647280745532e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.171875,
      "logps/chosen": -414.0,
      "logps/rejected": -644.0,
      "loss": 0.2857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.296875,
      "rewards/margins": 2.546875,
      "rewards/rejected": -4.84375,
      "step": 2100
    },
    {
      "epoch": 0.26966579334142754,
      "grad_norm": 10.443186997291225,
      "learning_rate": 4.5744426016927783e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.171875,
      "logps/chosen": -446.0,
      "logps/rejected": -656.0,
      "loss": 0.2812,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.609375,
      "rewards/margins": 2.359375,
      "rewards/rejected": -4.96875,
      "step": 2110
    },
    {
      "epoch": 0.270943830276695,
      "grad_norm": 10.189176574528263,
      "learning_rate": 4.5681966243795645e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.09375,
      "logps/chosen": -394.0,
      "logps/rejected": -688.0,
      "loss": 0.2784,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.171875,
      "rewards/margins": 2.890625,
      "rewards/rejected": -5.0625,
      "step": 2120
    },
    {
      "epoch": 0.2722218672119624,
      "grad_norm": 12.102070272805506,
      "learning_rate": 4.5619094731515783e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.25,
      "logps/chosen": -438.0,
      "logps/rejected": -708.0,
      "loss": 0.2647,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.453125,
      "rewards/margins": 3.0,
      "rewards/rejected": -5.4375,
      "step": 2130
    },
    {
      "epoch": 0.27349990414722986,
      "grad_norm": 9.930898269179549,
      "learning_rate": 4.5555812731742085e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.21875,
      "logps/chosen": -414.0,
      "logps/rejected": -640.0,
      "loss": 0.26,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.453125,
      "rewards/margins": 2.296875,
      "rewards/rejected": -4.75,
      "step": 2140
    },
    {
      "epoch": 0.27477794108249726,
      "grad_norm": 12.075161047304881,
      "learning_rate": 4.549212150430042e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.21875,
      "logps/chosen": -432.0,
      "logps/rejected": -692.0,
      "loss": 0.2651,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.4375,
      "rewards/margins": 2.734375,
      "rewards/rejected": -5.15625,
      "step": 2150
    },
    {
      "epoch": 0.2760559780177647,
      "grad_norm": 13.438155990023017,
      "learning_rate": 4.5428022317163654e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.25,
      "logps/chosen": -432.0,
      "logps/rejected": -688.0,
      "loss": 0.2887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.40625,
      "rewards/margins": 2.609375,
      "rewards/rejected": -5.0,
      "step": 2160
    },
    {
      "epoch": 0.2773340149530321,
      "grad_norm": 12.585155614271198,
      "learning_rate": 4.5363516446426353e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.203125,
      "logps/chosen": -400.0,
      "logps/rejected": -604.0,
      "loss": 0.2746,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.203125,
      "rewards/margins": 2.265625,
      "rewards/rejected": -4.46875,
      "step": 2170
    },
    {
      "epoch": 0.27861205188829957,
      "grad_norm": 16.417942479762438,
      "learning_rate": 4.5298605176279383e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.203125,
      "logps/chosen": -440.0,
      "logps/rejected": -740.0,
      "loss": 0.2912,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.484375,
      "rewards/margins": 3.15625,
      "rewards/rejected": -5.65625,
      "step": 2180
    },
    {
      "epoch": 0.279890088823567,
      "grad_norm": 12.868817634047597,
      "learning_rate": 4.5233289798984355e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.03125,
      "logps/chosen": -412.0,
      "logps/rejected": -652.0,
      "loss": 0.2769,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.296875,
      "rewards/margins": 2.53125,
      "rewards/rejected": -4.8125,
      "step": 2190
    },
    {
      "epoch": 0.28116812575883443,
      "grad_norm": 11.40166415349329,
      "learning_rate": 4.51675716148479e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -1.9453125,
      "logps/chosen": -396.0,
      "logps/rejected": -632.0,
      "loss": 0.2852,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.078125,
      "rewards/margins": 2.5625,
      "rewards/rejected": -4.625,
      "step": 2200
    },
    {
      "epoch": 0.28244616269410183,
      "grad_norm": 9.054658893187522,
      "learning_rate": 4.510145193219577e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.328125,
      "logps/chosen": -446.0,
      "logps/rejected": -712.0,
      "loss": 0.2891,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.5625,
      "rewards/margins": 2.84375,
      "rewards/rejected": -5.40625,
      "step": 2210
    },
    {
      "epoch": 0.2837241996293693,
      "grad_norm": 9.888688064685908,
      "learning_rate": 4.503493206734681e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.21875,
      "logps/chosen": -420.0,
      "logps/rejected": -660.0,
      "loss": 0.2989,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.359375,
      "rewards/margins": 2.5625,
      "rewards/rejected": -4.90625,
      "step": 2220
    },
    {
      "epoch": 0.2850022365646367,
      "grad_norm": 9.761654815230207,
      "learning_rate": 4.4968013344586723e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.15625,
      "logps/chosen": -418.0,
      "logps/rejected": -628.0,
      "loss": 0.2712,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.265625,
      "rewards/margins": 2.375,
      "rewards/rejected": -4.65625,
      "step": 2230
    },
    {
      "epoch": 0.28628027349990415,
      "grad_norm": 12.16374136199616,
      "learning_rate": 4.4900697096141754e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.28125,
      "logps/chosen": -410.0,
      "logps/rejected": -664.0,
      "loss": 0.2904,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.25,
      "rewards/margins": 2.546875,
      "rewards/rejected": -4.8125,
      "step": 2240
    },
    {
      "epoch": 0.28755831043517155,
      "grad_norm": 11.145755318967831,
      "learning_rate": 4.483298466215211e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.1875,
      "logps/chosen": -424.0,
      "logps/rejected": -692.0,
      "loss": 0.271,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.359375,
      "rewards/margins": 2.9375,
      "rewards/rejected": -5.28125,
      "step": 2250
    },
    {
      "epoch": 0.288836347370439,
      "grad_norm": 11.709681178155686,
      "learning_rate": 4.4764877390645317e-07,
      "logits/chosen": -2.25,
      "logits/rejected": -2.21875,
      "logps/chosen": -424.0,
      "logps/rejected": -660.0,
      "loss": 0.2593,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.546875,
      "rewards/margins": 2.4375,
      "rewards/rejected": -5.0,
      "step": 2260
    },
    {
      "epoch": 0.2901143843057064,
      "grad_norm": 9.70204184024738,
      "learning_rate": 4.469637663750939e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.25,
      "logps/chosen": -434.0,
      "logps/rejected": -632.0,
      "loss": 0.2767,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.390625,
      "rewards/margins": 2.421875,
      "rewards/rejected": -4.8125,
      "step": 2270
    },
    {
      "epoch": 0.29139242124097386,
      "grad_norm": 10.169093906005957,
      "learning_rate": 4.4627483766465813e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -2.171875,
      "logps/chosen": -414.0,
      "logps/rejected": -640.0,
      "loss": 0.2716,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.28125,
      "rewards/margins": 2.53125,
      "rewards/rejected": -4.8125,
      "step": 2280
    },
    {
      "epoch": 0.29267045817624127,
      "grad_norm": 12.81018729190582,
      "learning_rate": 4.4558200149042393e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.28125,
      "logps/chosen": -432.0,
      "logps/rejected": -668.0,
      "loss": 0.2733,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.296875,
      "rewards/margins": 2.484375,
      "rewards/rejected": -4.78125,
      "step": 2290
    },
    {
      "epoch": 0.2939484951115087,
      "grad_norm": 10.779020618470144,
      "learning_rate": 4.4488527164545976e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.234375,
      "logps/chosen": -420.0,
      "logps/rejected": -688.0,
      "loss": 0.2406,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.3125,
      "rewards/margins": 2.828125,
      "rewards/rejected": -5.125,
      "step": 2300
    },
    {
      "epoch": 0.2952265320467761,
      "grad_norm": 9.668977441880676,
      "learning_rate": 4.4418466200034974e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.21875,
      "logps/chosen": -412.0,
      "logps/rejected": -680.0,
      "loss": 0.2741,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.4375,
      "rewards/margins": 2.890625,
      "rewards/rejected": -5.3125,
      "step": 2310
    },
    {
      "epoch": 0.2965045689820436,
      "grad_norm": 15.335060457090837,
      "learning_rate": 4.4348018650291764e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.21875,
      "logps/chosen": -400.0,
      "logps/rejected": -640.0,
      "loss": 0.2932,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.046875,
      "rewards/margins": 2.75,
      "rewards/rejected": -4.8125,
      "step": 2320
    },
    {
      "epoch": 0.297782605917311,
      "grad_norm": 10.624992257766362,
      "learning_rate": 4.427718591779489e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.15625,
      "logps/chosen": -396.0,
      "logps/rejected": -676.0,
      "loss": 0.263,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.25,
      "rewards/margins": 2.765625,
      "rewards/rejected": -5.03125,
      "step": 2330
    },
    {
      "epoch": 0.29906064285257844,
      "grad_norm": 10.438098677547671,
      "learning_rate": 4.4205969412691167e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.234375,
      "logps/chosen": -446.0,
      "logps/rejected": -664.0,
      "loss": 0.2664,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.53125,
      "rewards/margins": 2.453125,
      "rewards/rejected": -5.0,
      "step": 2340
    },
    {
      "epoch": 0.30033867978784584,
      "grad_norm": 10.26905874756198,
      "learning_rate": 4.4134370552767617e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.4375,
      "logps/chosen": -470.0,
      "logps/rejected": -748.0,
      "loss": 0.2716,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.71875,
      "rewards/margins": 2.984375,
      "rewards/rejected": -5.71875,
      "step": 2350
    },
    {
      "epoch": 0.3016167167231133,
      "grad_norm": 12.970902436822943,
      "learning_rate": 4.406239076342322e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -2.21875,
      "logps/chosen": -436.0,
      "logps/rejected": -676.0,
      "loss": 0.2767,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.5,
      "rewards/margins": 2.640625,
      "rewards/rejected": -5.125,
      "step": 2360
    },
    {
      "epoch": 0.3028947536583807,
      "grad_norm": 7.357272482933819,
      "learning_rate": 4.399003147764053e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.0625,
      "logps/chosen": -422.0,
      "logps/rejected": -656.0,
      "loss": 0.2851,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.3125,
      "rewards/margins": 2.359375,
      "rewards/rejected": -4.6875,
      "step": 2370
    },
    {
      "epoch": 0.30417279059364816,
      "grad_norm": 9.997024713754136,
      "learning_rate": 4.3917294135957185e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.265625,
      "logps/chosen": -402.0,
      "logps/rejected": -604.0,
      "loss": 0.2756,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.234375,
      "rewards/margins": 2.3125,
      "rewards/rejected": -4.5625,
      "step": 2380
    },
    {
      "epoch": 0.30545082752891556,
      "grad_norm": 7.555159686312881,
      "learning_rate": 4.3844180186437205e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.171875,
      "logps/chosen": -438.0,
      "logps/rejected": -668.0,
      "loss": 0.2735,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.546875,
      "rewards/margins": 2.53125,
      "rewards/rejected": -5.09375,
      "step": 2390
    },
    {
      "epoch": 0.306728864464183,
      "grad_norm": 10.025080434032118,
      "learning_rate": 4.3770691084642153e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.015625,
      "logps/chosen": -416.0,
      "logps/rejected": -640.0,
      "loss": 0.271,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.4375,
      "rewards/margins": 2.421875,
      "rewards/rejected": -4.875,
      "step": 2400
    },
    {
      "epoch": 0.3080069013994504,
      "grad_norm": 9.844634098430475,
      "learning_rate": 4.3696828293602185e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.375,
      "logps/chosen": -436.0,
      "logps/rejected": -644.0,
      "loss": 0.2628,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.375,
      "rewards/margins": 2.578125,
      "rewards/rejected": -4.9375,
      "step": 2410
    },
    {
      "epoch": 0.3092849383347179,
      "grad_norm": 11.836817377307739,
      "learning_rate": 4.3622593283786895e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.40625,
      "logps/chosen": -426.0,
      "logps/rejected": -708.0,
      "loss": 0.2507,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.4375,
      "rewards/margins": 3.078125,
      "rewards/rejected": -5.5,
      "step": 2420
    },
    {
      "epoch": 0.3105629752699853,
      "grad_norm": 11.550693147766143,
      "learning_rate": 4.354798753307606e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.34375,
      "logps/chosen": -422.0,
      "logps/rejected": -704.0,
      "loss": 0.2492,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.34375,
      "rewards/margins": 3.109375,
      "rewards/rejected": -5.4375,
      "step": 2430
    },
    {
      "epoch": 0.31184101220525273,
      "grad_norm": 11.466982332907172,
      "learning_rate": 4.3473012526730216e-07,
      "logits/chosen": -2.25,
      "logits/rejected": -2.3125,
      "logps/chosen": -410.0,
      "logps/rejected": -656.0,
      "loss": 0.2655,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.1875,
      "rewards/margins": 2.59375,
      "rewards/rejected": -4.78125,
      "step": 2440
    },
    {
      "epoch": 0.31311904914052013,
      "grad_norm": 9.045674296055328,
      "learning_rate": 4.339766975736109e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.296875,
      "logps/chosen": -416.0,
      "logps/rejected": -736.0,
      "loss": 0.2633,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.453125,
      "rewards/margins": 2.984375,
      "rewards/rejected": -5.4375,
      "step": 2450
    },
    {
      "epoch": 0.3143970860757876,
      "grad_norm": 11.89984472978286,
      "learning_rate": 4.332196072490185e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.28125,
      "logps/chosen": -420.0,
      "logps/rejected": -696.0,
      "loss": 0.2721,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.625,
      "rewards/margins": 2.859375,
      "rewards/rejected": -5.46875,
      "step": 2460
    },
    {
      "epoch": 0.315675123011055,
      "grad_norm": 11.646871169229096,
      "learning_rate": 4.324588693657733e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.0625,
      "logps/chosen": -444.0,
      "logps/rejected": -740.0,
      "loss": 0.2718,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.53125,
      "rewards/margins": 3.140625,
      "rewards/rejected": -5.6875,
      "step": 2470
    },
    {
      "epoch": 0.31695315994632245,
      "grad_norm": 9.706434117562274,
      "learning_rate": 4.3169449906873925e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.28125,
      "logps/chosen": -444.0,
      "logps/rejected": -728.0,
      "loss": 0.2589,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.453125,
      "rewards/margins": 2.96875,
      "rewards/rejected": -5.4375,
      "step": 2480
    },
    {
      "epoch": 0.31823119688158985,
      "grad_norm": 8.034872428293301,
      "learning_rate": 4.309265115750949e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -2.296875,
      "logps/chosen": -434.0,
      "logps/rejected": -676.0,
      "loss": 0.2607,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.515625,
      "rewards/margins": 2.6875,
      "rewards/rejected": -5.21875,
      "step": 2490
    },
    {
      "epoch": 0.3195092338168573,
      "grad_norm": 11.156572212234126,
      "learning_rate": 4.301549221740305e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.203125,
      "logps/chosen": -442.0,
      "logps/rejected": -724.0,
      "loss": 0.2845,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.53125,
      "rewards/margins": 2.90625,
      "rewards/rejected": -5.4375,
      "step": 2500
    },
    {
      "epoch": 0.3207872707521247,
      "grad_norm": 10.814779153303135,
      "learning_rate": 4.293797462264436e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.3125,
      "logps/chosen": -434.0,
      "logps/rejected": -712.0,
      "loss": 0.2589,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.484375,
      "rewards/margins": 2.765625,
      "rewards/rejected": -5.25,
      "step": 2510
    },
    {
      "epoch": 0.32206530768739217,
      "grad_norm": 11.659094925830505,
      "learning_rate": 4.286009991646329e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.234375,
      "logps/chosen": -438.0,
      "logps/rejected": -676.0,
      "loss": 0.2706,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.421875,
      "rewards/margins": 2.796875,
      "rewards/rejected": -5.21875,
      "step": 2520
    },
    {
      "epoch": 0.32334334462265957,
      "grad_norm": 9.22941070189285,
      "learning_rate": 4.2781869649199153e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.140625,
      "logps/chosen": -416.0,
      "logps/rejected": -656.0,
      "loss": 0.263,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.359375,
      "rewards/margins": 2.5625,
      "rewards/rejected": -4.90625,
      "step": 2530
    },
    {
      "epoch": 0.324621381557927,
      "grad_norm": 10.628278849266108,
      "learning_rate": 4.2703285378269815e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.234375,
      "logps/chosen": -426.0,
      "logps/rejected": -660.0,
      "loss": 0.268,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.34375,
      "rewards/margins": 2.703125,
      "rewards/rejected": -5.0625,
      "step": 2540
    },
    {
      "epoch": 0.3258994184931944,
      "grad_norm": 12.957030194128356,
      "learning_rate": 4.262434866814067e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.28125,
      "logps/chosen": -430.0,
      "logps/rejected": -724.0,
      "loss": 0.2382,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.53125,
      "rewards/margins": 3.046875,
      "rewards/rejected": -5.5625,
      "step": 2550
    },
    {
      "epoch": 0.3271774554284619,
      "grad_norm": 10.47574828518788,
      "learning_rate": 4.254506109029353e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.15625,
      "logps/chosen": -436.0,
      "logps/rejected": -672.0,
      "loss": 0.2743,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.328125,
      "rewards/margins": 2.796875,
      "rewards/rejected": -5.125,
      "step": 2560
    },
    {
      "epoch": 0.3284554923637293,
      "grad_norm": 10.581737496101809,
      "learning_rate": 4.2465424223195327e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.171875,
      "logps/chosen": -402.0,
      "logps/rejected": -636.0,
      "loss": 0.2575,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.1875,
      "rewards/margins": 2.546875,
      "rewards/rejected": -4.75,
      "step": 2570
    },
    {
      "epoch": 0.32973352929899674,
      "grad_norm": 12.120190411903161,
      "learning_rate": 4.238543965226668e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.203125,
      "logps/chosen": -408.0,
      "logps/rejected": -660.0,
      "loss": 0.2716,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.234375,
      "rewards/margins": 2.78125,
      "rewards/rejected": -5.0,
      "step": 2580
    },
    {
      "epoch": 0.33101156623426414,
      "grad_norm": 10.973781797711162,
      "learning_rate": 4.230510896985035e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.375,
      "logps/chosen": -466.0,
      "logps/rejected": -704.0,
      "loss": 0.2505,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.6875,
      "rewards/margins": 2.71875,
      "rewards/rejected": -5.40625,
      "step": 2590
    },
    {
      "epoch": 0.3322896031695316,
      "grad_norm": 12.034715318189681,
      "learning_rate": 4.2224433775179506e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.265625,
      "logps/chosen": -464.0,
      "logps/rejected": -704.0,
      "loss": 0.2932,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.765625,
      "rewards/margins": 2.640625,
      "rewards/rejected": -5.40625,
      "step": 2600
    },
    {
      "epoch": 0.333567640104799,
      "grad_norm": 11.11881259976669,
      "learning_rate": 4.2143415674345937e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.265625,
      "logps/chosen": -424.0,
      "logps/rejected": -660.0,
      "loss": 0.2533,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.375,
      "rewards/margins": 2.4375,
      "rewards/rejected": -4.8125,
      "step": 2610
    },
    {
      "epoch": 0.33484567704006646,
      "grad_norm": 9.606926010914945,
      "learning_rate": 4.2062056280268033e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.203125,
      "logps/chosen": -426.0,
      "logps/rejected": -708.0,
      "loss": 0.2423,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.640625,
      "rewards/margins": 2.859375,
      "rewards/rejected": -5.5,
      "step": 2620
    },
    {
      "epoch": 0.33612371397533386,
      "grad_norm": 15.828477485682688,
      "learning_rate": 4.198035721265869e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.328125,
      "logps/chosen": -406.0,
      "logps/rejected": -712.0,
      "loss": 0.267,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.3125,
      "rewards/margins": 3.09375,
      "rewards/rejected": -5.40625,
      "step": 2630
    },
    {
      "epoch": 0.3374017509106013,
      "grad_norm": 10.929725980005715,
      "learning_rate": 4.1898320097993085e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.3125,
      "logps/chosen": -426.0,
      "logps/rejected": -648.0,
      "loss": 0.2883,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.296875,
      "rewards/margins": 2.46875,
      "rewards/rejected": -4.75,
      "step": 2640
    },
    {
      "epoch": 0.3386797878458687,
      "grad_norm": 11.244428417350711,
      "learning_rate": 4.181594656947625e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.375,
      "logps/chosen": -402.0,
      "logps/rejected": -648.0,
      "loss": 0.2349,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.25,
      "rewards/margins": 2.578125,
      "rewards/rejected": -4.8125,
      "step": 2650
    },
    {
      "epoch": 0.3399578247811362,
      "grad_norm": 11.613826187753885,
      "learning_rate": 4.173323826701062e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.3125,
      "logps/chosen": -472.0,
      "logps/rejected": -732.0,
      "loss": 0.2619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.84375,
      "rewards/margins": 2.71875,
      "rewards/rejected": -5.5625,
      "step": 2660
    },
    {
      "epoch": 0.3412358617164036,
      "grad_norm": 13.389232781869358,
      "learning_rate": 4.165019683716332e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.21875,
      "logps/chosen": -434.0,
      "logps/rejected": -672.0,
      "loss": 0.2559,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.515625,
      "rewards/margins": 2.515625,
      "rewards/rejected": -5.03125,
      "step": 2670
    },
    {
      "epoch": 0.34251389865167103,
      "grad_norm": 11.782073620423972,
      "learning_rate": 4.1566823933133444e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.375,
      "logps/chosen": -412.0,
      "logps/rejected": -688.0,
      "loss": 0.2788,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.296875,
      "rewards/margins": 2.953125,
      "rewards/rejected": -5.25,
      "step": 2680
    },
    {
      "epoch": 0.34379193558693844,
      "grad_norm": 12.908332707882192,
      "learning_rate": 4.148312121471908e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.34375,
      "logps/chosen": -446.0,
      "logps/rejected": -736.0,
      "loss": 0.2511,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.140625,
      "rewards/rejected": -5.75,
      "step": 2690
    },
    {
      "epoch": 0.3450699725222059,
      "grad_norm": 12.249392495707431,
      "learning_rate": 4.1399090348284336e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.3125,
      "logps/chosen": -430.0,
      "logps/rejected": -692.0,
      "loss": 0.2585,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.484375,
      "rewards/margins": 2.8125,
      "rewards/rejected": -5.28125,
      "step": 2700
    },
    {
      "epoch": 0.3463480094574733,
      "grad_norm": 11.169978502508878,
      "learning_rate": 4.1314733006726116e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.34375,
      "logps/chosen": -414.0,
      "logps/rejected": -648.0,
      "loss": 0.2726,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.296875,
      "rewards/margins": 2.640625,
      "rewards/rejected": -4.9375,
      "step": 2710
    },
    {
      "epoch": 0.34762604639274075,
      "grad_norm": 10.707577279722418,
      "learning_rate": 4.1230050869440825e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.375,
      "logps/chosen": -444.0,
      "logps/rejected": -680.0,
      "loss": 0.2592,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.34375,
      "rewards/margins": 2.734375,
      "rewards/rejected": -5.09375,
      "step": 2720
    },
    {
      "epoch": 0.34890408332800815,
      "grad_norm": 8.827949466883961,
      "learning_rate": 4.114504562229096e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.46875,
      "logps/chosen": -422.0,
      "logps/rejected": -688.0,
      "loss": 0.2701,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.453125,
      "rewards/margins": 2.859375,
      "rewards/rejected": -5.3125,
      "step": 2730
    },
    {
      "epoch": 0.3501821202632756,
      "grad_norm": 13.154347846349706,
      "learning_rate": 4.105971895757151e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.390625,
      "logps/chosen": -392.0,
      "logps/rejected": -648.0,
      "loss": 0.2772,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.28125,
      "rewards/margins": 2.703125,
      "rewards/rejected": -5.0,
      "step": 2740
    },
    {
      "epoch": 0.351460157198543,
      "grad_norm": 10.399248642028768,
      "learning_rate": 4.0974072573976295e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.296875,
      "logps/chosen": -394.0,
      "logps/rejected": -680.0,
      "loss": 0.2527,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.203125,
      "rewards/margins": 2.890625,
      "rewards/rejected": -5.09375,
      "step": 2750
    },
    {
      "epoch": 0.35273819413381047,
      "grad_norm": 9.82428120317813,
      "learning_rate": 4.088810817656414e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.3125,
      "logps/chosen": -474.0,
      "logps/rejected": -716.0,
      "loss": 0.2593,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.78125,
      "rewards/margins": 2.75,
      "rewards/rejected": -5.53125,
      "step": 2760
    },
    {
      "epoch": 0.35401623106907787,
      "grad_norm": 11.50941018900212,
      "learning_rate": 4.0801827476724923e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.46875,
      "logps/chosen": -446.0,
      "logps/rejected": -700.0,
      "loss": 0.2431,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.578125,
      "rewards/margins": 2.71875,
      "rewards/rejected": -5.3125,
      "step": 2770
    },
    {
      "epoch": 0.3552942680043453,
      "grad_norm": 11.10969911670547,
      "learning_rate": 4.071523219214551e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.390625,
      "logps/chosen": -448.0,
      "logps/rejected": -792.0,
      "loss": 0.2445,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.65625,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.0,
      "step": 2780
    },
    {
      "epoch": 0.3565723049396127,
      "grad_norm": 8.104569743473352,
      "learning_rate": 4.062832404677556e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.4375,
      "logps/chosen": -442.0,
      "logps/rejected": -684.0,
      "loss": 0.2475,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.5,
      "rewards/margins": 2.703125,
      "rewards/rejected": -5.1875,
      "step": 2790
    },
    {
      "epoch": 0.3578503418748802,
      "grad_norm": 11.673687594651634,
      "learning_rate": 4.054110477079321e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.21875,
      "logps/chosen": -468.0,
      "logps/rejected": -728.0,
      "loss": 0.265,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.640625,
      "rewards/margins": 2.875,
      "rewards/rejected": -5.5,
      "step": 2800
    },
    {
      "epoch": 0.3591283788101476,
      "grad_norm": 8.970696498042463,
      "learning_rate": 4.0453576100570606e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.28125,
      "logps/chosen": -396.0,
      "logps/rejected": -636.0,
      "loss": 0.272,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.234375,
      "rewards/margins": 2.46875,
      "rewards/rejected": -4.6875,
      "step": 2810
    },
    {
      "epoch": 0.36040641574541504,
      "grad_norm": 14.739843608173656,
      "learning_rate": 4.036573977863938e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.3125,
      "logps/chosen": -448.0,
      "logps/rejected": -712.0,
      "loss": 0.2604,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.453125,
      "rewards/margins": 2.984375,
      "rewards/rejected": -5.4375,
      "step": 2820
    },
    {
      "epoch": 0.36168445268068244,
      "grad_norm": 14.568226478232424,
      "learning_rate": 4.027759755365591e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.375,
      "logps/chosen": -440.0,
      "logps/rejected": -760.0,
      "loss": 0.25,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.578125,
      "rewards/margins": 3.359375,
      "rewards/rejected": -5.9375,
      "step": 2830
    },
    {
      "epoch": 0.3629624896159499,
      "grad_norm": 8.837111129892946,
      "learning_rate": 4.018915118036653e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.125,
      "logps/chosen": -416.0,
      "logps/rejected": -728.0,
      "loss": 0.2558,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.21875,
      "rewards/margins": 3.34375,
      "rewards/rejected": -5.5625,
      "step": 2840
    },
    {
      "epoch": 0.3642405265512173,
      "grad_norm": 7.138930837493291,
      "learning_rate": 4.0100402419572617e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.21875,
      "logps/chosen": -406.0,
      "logps/rejected": -668.0,
      "loss": 0.2541,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.015625,
      "rewards/margins": 2.703125,
      "rewards/rejected": -4.71875,
      "step": 2850
    },
    {
      "epoch": 0.36551856348648476,
      "grad_norm": 7.983961276336517,
      "learning_rate": 4.0011353038095497e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.34375,
      "logps/chosen": -436.0,
      "logps/rejected": -724.0,
      "loss": 0.2458,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.5,
      "rewards/margins": 2.9375,
      "rewards/rejected": -5.4375,
      "step": 2860
    },
    {
      "epoch": 0.36679660042175216,
      "grad_norm": 11.52040604673254,
      "learning_rate": 3.992200480874128e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.265625,
      "logps/chosen": -416.0,
      "logps/rejected": -760.0,
      "loss": 0.2365,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.625,
      "rewards/margins": 3.296875,
      "rewards/rejected": -5.90625,
      "step": 2870
    },
    {
      "epoch": 0.3680746373570196,
      "grad_norm": 14.756150497011767,
      "learning_rate": 3.983235951026562e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.390625,
      "logps/chosen": -438.0,
      "logps/rejected": -724.0,
      "loss": 0.2553,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.65625,
      "rewards/margins": 2.953125,
      "rewards/rejected": -5.625,
      "step": 2880
    },
    {
      "epoch": 0.369352674292287,
      "grad_norm": 7.2203575753797375,
      "learning_rate": 3.974241892733821e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.296875,
      "logps/chosen": -404.0,
      "logps/rejected": -676.0,
      "loss": 0.2259,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.34375,
      "rewards/margins": 2.734375,
      "rewards/rejected": -5.0625,
      "step": 2890
    },
    {
      "epoch": 0.3706307112275545,
      "grad_norm": 10.718843520834543,
      "learning_rate": 3.965218485050733e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.328125,
      "logps/chosen": -416.0,
      "logps/rejected": -712.0,
      "loss": 0.2456,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.40625,
      "rewards/margins": 2.984375,
      "rewards/rejected": -5.375,
      "step": 2900
    },
    {
      "epoch": 0.3719087481628219,
      "grad_norm": 10.073390021857904,
      "learning_rate": 3.9561659076164166e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.328125,
      "logps/chosen": -422.0,
      "logps/rejected": -732.0,
      "loss": 0.2553,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.234375,
      "rewards/margins": 3.40625,
      "rewards/rejected": -5.625,
      "step": 2910
    },
    {
      "epoch": 0.37318678509808934,
      "grad_norm": 10.414959052127895,
      "learning_rate": 3.947084340650706e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.296875,
      "logps/chosen": -402.0,
      "logps/rejected": -760.0,
      "loss": 0.249,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.296875,
      "rewards/margins": 3.53125,
      "rewards/rejected": -5.8125,
      "step": 2920
    },
    {
      "epoch": 0.37446482203335674,
      "grad_norm": 10.49482397981431,
      "learning_rate": 3.937973964950561e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.28125,
      "logps/chosen": -438.0,
      "logps/rejected": -720.0,
      "loss": 0.2474,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.578125,
      "rewards/margins": 3.0,
      "rewards/rejected": -5.5625,
      "step": 2930
    },
    {
      "epoch": 0.3757428589686242,
      "grad_norm": 14.707267984695216,
      "learning_rate": 3.928834961886472e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.375,
      "logps/chosen": -448.0,
      "logps/rejected": -720.0,
      "loss": 0.245,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.53125,
      "rewards/margins": 3.0,
      "rewards/rejected": -5.53125,
      "step": 2940
    },
    {
      "epoch": 0.3770208959038916,
      "grad_norm": 9.485294946885139,
      "learning_rate": 3.919667513398843e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.28125,
      "logps/chosen": -440.0,
      "logps/rejected": -752.0,
      "loss": 0.2284,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.5625,
      "rewards/margins": 3.1875,
      "rewards/rejected": -5.75,
      "step": 2950
    },
    {
      "epoch": 0.37829893283915905,
      "grad_norm": 9.921180142338146,
      "learning_rate": 3.910471801994377e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.140625,
      "logps/chosen": -450.0,
      "logps/rejected": -720.0,
      "loss": 0.2545,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.5,
      "rewards/margins": 3.09375,
      "rewards/rejected": -5.59375,
      "step": 2960
    },
    {
      "epoch": 0.37957696977442645,
      "grad_norm": 13.308396614966219,
      "learning_rate": 3.901248010742435e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.3125,
      "logps/chosen": -420.0,
      "logps/rejected": -708.0,
      "loss": 0.2507,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.515625,
      "rewards/margins": 2.921875,
      "rewards/rejected": -5.4375,
      "step": 2970
    },
    {
      "epoch": 0.3808550067096939,
      "grad_norm": 11.217873677324668,
      "learning_rate": 3.891996323271396e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.28125,
      "logps/chosen": -456.0,
      "logps/rejected": -744.0,
      "loss": 0.2709,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.0,
      "rewards/rejected": -5.71875,
      "step": 2980
    },
    {
      "epoch": 0.3821330436449613,
      "grad_norm": 7.6617539343495595,
      "learning_rate": 3.8827169237650023e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.40625,
      "logps/chosen": -420.0,
      "logps/rejected": -680.0,
      "loss": 0.2367,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.359375,
      "rewards/margins": 2.75,
      "rewards/rejected": -5.125,
      "step": 2990
    },
    {
      "epoch": 0.38341108058022877,
      "grad_norm": 9.292715824169168,
      "learning_rate": 3.87340999695869e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.3125,
      "logps/chosen": -432.0,
      "logps/rejected": -712.0,
      "loss": 0.2492,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.625,
      "rewards/margins": 2.796875,
      "rewards/rejected": -5.40625,
      "step": 3000
    },
    {
      "epoch": 0.38468911751549617,
      "grad_norm": 11.507105558718663,
      "learning_rate": 3.8640757281359104e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.328125,
      "logps/chosen": -426.0,
      "logps/rejected": -680.0,
      "loss": 0.2645,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.390625,
      "rewards/margins": 2.71875,
      "rewards/rejected": -5.125,
      "step": 3010
    },
    {
      "epoch": 0.3859671544507636,
      "grad_norm": 11.691045860260699,
      "learning_rate": 3.8547143031244454e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.234375,
      "logps/chosen": -434.0,
      "logps/rejected": -760.0,
      "loss": 0.2466,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.46875,
      "rewards/margins": 3.359375,
      "rewards/rejected": -5.8125,
      "step": 3020
    },
    {
      "epoch": 0.38724519138603103,
      "grad_norm": 12.239360872004156,
      "learning_rate": 3.845325908292704e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.40625,
      "logps/chosen": -462.0,
      "logps/rejected": -784.0,
      "loss": 0.2414,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.703125,
      "rewards/margins": 3.296875,
      "rewards/rejected": -6.0,
      "step": 3030
    },
    {
      "epoch": 0.3885232283212985,
      "grad_norm": 9.983638981877096,
      "learning_rate": 3.835910730546013e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.28125,
      "logps/chosen": -428.0,
      "logps/rejected": -620.0,
      "loss": 0.2598,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.390625,
      "rewards/margins": 2.1875,
      "rewards/rejected": -4.59375,
      "step": 3040
    },
    {
      "epoch": 0.3898012652565659,
      "grad_norm": 11.279243125715178,
      "learning_rate": 3.8264689573229e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.375,
      "logps/chosen": -440.0,
      "logps/rejected": -728.0,
      "loss": 0.2442,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.453125,
      "rewards/margins": 3.0625,
      "rewards/rejected": -5.5,
      "step": 3050
    },
    {
      "epoch": 0.39107930219183334,
      "grad_norm": 11.518430645477059,
      "learning_rate": 3.8170007765913563e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.421875,
      "logps/chosen": -484.0,
      "logps/rejected": -776.0,
      "loss": 0.2157,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.15625,
      "step": 3060
    },
    {
      "epoch": 0.39235733912710075,
      "grad_norm": 7.819947934011583,
      "learning_rate": 3.8075063768450977e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.375,
      "logps/chosen": -460.0,
      "logps/rejected": -776.0,
      "loss": 0.2456,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.71875,
      "rewards/margins": 3.265625,
      "rewards/rejected": -5.96875,
      "step": 3070
    },
    {
      "epoch": 0.3936353760623682,
      "grad_norm": 11.35118723009356,
      "learning_rate": 3.79798594709981e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.390625,
      "logps/chosen": -440.0,
      "logps/rejected": -724.0,
      "loss": 0.2573,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.015625,
      "rewards/rejected": -5.625,
      "step": 3080
    },
    {
      "epoch": 0.3949134129976356,
      "grad_norm": 12.047891934813237,
      "learning_rate": 3.78843967688939e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.234375,
      "logps/chosen": -430.0,
      "logps/rejected": -712.0,
      "loss": 0.2382,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.484375,
      "rewards/margins": 3.0,
      "rewards/rejected": -5.5,
      "step": 3090
    },
    {
      "epoch": 0.39619144993290306,
      "grad_norm": 9.123822686209294,
      "learning_rate": 3.7788677562621676e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.390625,
      "logps/chosen": -428.0,
      "logps/rejected": -692.0,
      "loss": 0.2489,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.484375,
      "rewards/margins": 2.828125,
      "rewards/rejected": -5.3125,
      "step": 3100
    },
    {
      "epoch": 0.39746948686817046,
      "grad_norm": 10.184155916234085,
      "learning_rate": 3.769270375777126e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.28125,
      "logps/chosen": -432.0,
      "logps/rejected": -740.0,
      "loss": 0.2624,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.578125,
      "rewards/margins": 3.015625,
      "rewards/rejected": -5.59375,
      "step": 3110
    },
    {
      "epoch": 0.3987475238034379,
      "grad_norm": 10.02436576862269,
      "learning_rate": 3.7596477265001053e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.421875,
      "logps/chosen": -418.0,
      "logps/rejected": -736.0,
      "loss": 0.2413,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.421875,
      "rewards/margins": 3.28125,
      "rewards/rejected": -5.6875,
      "step": 3120
    },
    {
      "epoch": 0.4000255607387053,
      "grad_norm": 11.236376923081936,
      "learning_rate": 3.75e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.359375,
      "logps/chosen": -446.0,
      "logps/rejected": -716.0,
      "loss": 0.2477,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.515625,
      "rewards/margins": 3.0,
      "rewards/rejected": -5.53125,
      "step": 3130
    },
    {
      "epoch": 0.4013035976739728,
      "grad_norm": 11.648021212875348,
      "learning_rate": 3.740327388344945e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.3125,
      "logps/chosen": -444.0,
      "logps/rejected": -764.0,
      "loss": 0.2268,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.625,
      "rewards/margins": 3.28125,
      "rewards/rejected": -5.90625,
      "step": 3140
    },
    {
      "epoch": 0.4025816346092402,
      "grad_norm": 8.06922286095696,
      "learning_rate": 3.7306300840984927e-07,
      "logits/chosen": -2.25,
      "logits/rejected": -2.3125,
      "logps/chosen": -468.0,
      "logps/rejected": -832.0,
      "loss": 0.2369,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.640625,
      "rewards/margins": 3.75,
      "rewards/rejected": -6.40625,
      "step": 3150
    },
    {
      "epoch": 0.40385967154450764,
      "grad_norm": 10.108638710167176,
      "learning_rate": 3.720908280315777e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.234375,
      "logps/chosen": -432.0,
      "logps/rejected": -716.0,
      "loss": 0.249,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.453125,
      "rewards/margins": 3.03125,
      "rewards/rejected": -5.46875,
      "step": 3160
    },
    {
      "epoch": 0.40513770847977504,
      "grad_norm": 11.581551550646601,
      "learning_rate": 3.711162170539673e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.265625,
      "logps/chosen": -434.0,
      "logps/rejected": -704.0,
      "loss": 0.2369,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.5,
      "rewards/margins": 2.90625,
      "rewards/rejected": -5.40625,
      "step": 3170
    },
    {
      "epoch": 0.4064157454150425,
      "grad_norm": 10.587867209773176,
      "learning_rate": 3.701391948796945e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.328125,
      "logps/chosen": -452.0,
      "logps/rejected": -704.0,
      "loss": 0.2363,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.59375,
      "rewards/margins": 2.828125,
      "rewards/rejected": -5.4375,
      "step": 3180
    },
    {
      "epoch": 0.4076937823503099,
      "grad_norm": 9.278469364982655,
      "learning_rate": 3.6915978095943745e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.359375,
      "logps/chosen": -420.0,
      "logps/rejected": -696.0,
      "loss": 0.2426,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.46875,
      "rewards/margins": 2.921875,
      "rewards/rejected": -5.375,
      "step": 3190
    },
    {
      "epoch": 0.40897181928557735,
      "grad_norm": 13.38740635025647,
      "learning_rate": 3.6817799479149007e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.328125,
      "logps/chosen": -414.0,
      "logps/rejected": -672.0,
      "loss": 0.2779,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.390625,
      "rewards/margins": 2.765625,
      "rewards/rejected": -5.15625,
      "step": 3200
    },
    {
      "epoch": 0.41024985622084476,
      "grad_norm": 13.40562304866693,
      "learning_rate": 3.6719385592137306e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.28125,
      "logps/chosen": -404.0,
      "logps/rejected": -680.0,
      "loss": 0.2609,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.265625,
      "rewards/margins": 2.9375,
      "rewards/rejected": -5.21875,
      "step": 3210
    },
    {
      "epoch": 0.4115278931561122,
      "grad_norm": 8.35310091377115,
      "learning_rate": 3.662073839414452e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.265625,
      "logps/chosen": -432.0,
      "logps/rejected": -692.0,
      "loss": 0.2513,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.484375,
      "rewards/margins": 2.75,
      "rewards/rejected": -5.21875,
      "step": 3220
    },
    {
      "epoch": 0.4128059300913796,
      "grad_norm": 9.930894543844001,
      "learning_rate": 3.6521859849051274e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.3125,
      "logps/chosen": -442.0,
      "logps/rejected": -700.0,
      "loss": 0.2556,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.609375,
      "rewards/margins": 2.75,
      "rewards/rejected": -5.375,
      "step": 3230
    },
    {
      "epoch": 0.41408396702664707,
      "grad_norm": 11.62064783882735,
      "learning_rate": 3.6422751925343906e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.421875,
      "logps/chosen": -466.0,
      "logps/rejected": -784.0,
      "loss": 0.2272,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.09375,
      "step": 3240
    },
    {
      "epoch": 0.41536200396191447,
      "grad_norm": 11.078641311322919,
      "learning_rate": 3.6323416596075244e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.40625,
      "logps/chosen": -492.0,
      "logps/rejected": -772.0,
      "loss": 0.2508,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.9375,
      "rewards/margins": 2.9375,
      "rewards/rejected": -5.875,
      "step": 3250
    },
    {
      "epoch": 0.41664004089718193,
      "grad_norm": 11.187991009603497,
      "learning_rate": 3.622385583882535e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.234375,
      "logps/chosen": -426.0,
      "logps/rejected": -680.0,
      "loss": 0.2524,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.578125,
      "rewards/margins": 2.734375,
      "rewards/rejected": -5.3125,
      "step": 3260
    },
    {
      "epoch": 0.41791807783244933,
      "grad_norm": 7.600502661996142,
      "learning_rate": 3.6124071635662107e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.484375,
      "logps/chosen": -452.0,
      "logps/rejected": -724.0,
      "loss": 0.2462,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.625,
      "rewards/margins": 3.015625,
      "rewards/rejected": -5.65625,
      "step": 3270
    },
    {
      "epoch": 0.4191961147677168,
      "grad_norm": 14.062309806947976,
      "learning_rate": 3.6024065973101803e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.484375,
      "logps/chosen": -446.0,
      "logps/rejected": -732.0,
      "loss": 0.2419,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.65625,
      "rewards/margins": 3.125,
      "rewards/rejected": -5.78125,
      "step": 3280
    },
    {
      "epoch": 0.4204741517029842,
      "grad_norm": 10.249470870134443,
      "learning_rate": 3.5923840842069583e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.484375,
      "logps/chosen": -434.0,
      "logps/rejected": -724.0,
      "loss": 0.2398,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.484375,
      "rewards/margins": 3.03125,
      "rewards/rejected": -5.53125,
      "step": 3290
    },
    {
      "epoch": 0.42175218863825165,
      "grad_norm": 11.420849078361023,
      "learning_rate": 3.5823398237859746e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.484375,
      "logps/chosen": -442.0,
      "logps/rejected": -720.0,
      "loss": 0.2438,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.71875,
      "rewards/margins": 2.8125,
      "rewards/rejected": -5.53125,
      "step": 3300
    },
    {
      "epoch": 0.42303022557351905,
      "grad_norm": 13.524707674748731,
      "learning_rate": 3.572274016009613e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.390625,
      "logps/chosen": -460.0,
      "logps/rejected": -764.0,
      "loss": 0.2495,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.703125,
      "rewards/margins": 2.90625,
      "rewards/rejected": -5.625,
      "step": 3310
    },
    {
      "epoch": 0.4243082625087865,
      "grad_norm": 9.429943944446737,
      "learning_rate": 3.562186861269223e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.53125,
      "logps/chosen": -434.0,
      "logps/rejected": -708.0,
      "loss": 0.2271,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.453125,
      "rewards/margins": 2.984375,
      "rewards/rejected": -5.4375,
      "step": 3320
    },
    {
      "epoch": 0.4255862994440539,
      "grad_norm": 14.833668893179714,
      "learning_rate": 3.5520785603811296e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.40625,
      "logps/chosen": -434.0,
      "logps/rejected": -704.0,
      "loss": 0.2642,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.65625,
      "rewards/margins": 2.734375,
      "rewards/rejected": -5.375,
      "step": 3330
    },
    {
      "epoch": 0.42686433637932136,
      "grad_norm": 10.7026360381983,
      "learning_rate": 3.541949314582641e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.421875,
      "logps/chosen": -456.0,
      "logps/rejected": -776.0,
      "loss": 0.235,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.03125,
      "step": 3340
    },
    {
      "epoch": 0.42814237331458876,
      "grad_norm": 8.283214307967622,
      "learning_rate": 3.5317993255280383e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.46875,
      "logps/chosen": -486.0,
      "logps/rejected": -764.0,
      "loss": 0.2302,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.15625,
      "rewards/rejected": -6.03125,
      "step": 3350
    },
    {
      "epoch": 0.4294204102498562,
      "grad_norm": 11.09056073170585,
      "learning_rate": 3.5216287952845614e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.359375,
      "logps/chosen": -448.0,
      "logps/rejected": -800.0,
      "loss": 0.2316,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.625,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.21875,
      "step": 3360
    },
    {
      "epoch": 0.4306984471851236,
      "grad_norm": 10.197944605053964,
      "learning_rate": 3.511437926328387e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.28125,
      "logps/chosen": -456.0,
      "logps/rejected": -800.0,
      "loss": 0.2403,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.796875,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.125,
      "step": 3370
    },
    {
      "epoch": 0.4319764841203911,
      "grad_norm": 9.172107195653277,
      "learning_rate": 3.501226921540598e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.421875,
      "logps/chosen": -424.0,
      "logps/rejected": -776.0,
      "loss": 0.243,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.53125,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.09375,
      "step": 3380
    },
    {
      "epoch": 0.4332545210556585,
      "grad_norm": 11.369002183707178,
      "learning_rate": 3.4909959842031436e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.4375,
      "logps/chosen": -428.0,
      "logps/rejected": -696.0,
      "loss": 0.2254,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.546875,
      "rewards/margins": 2.84375,
      "rewards/rejected": -5.40625,
      "step": 3390
    },
    {
      "epoch": 0.43453255799092594,
      "grad_norm": 11.761323305437706,
      "learning_rate": 3.480745317994793e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.515625,
      "logps/chosen": -464.0,
      "logps/rejected": -788.0,
      "loss": 0.2269,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.15625,
      "step": 3400
    },
    {
      "epoch": 0.43581059492619334,
      "grad_norm": 11.404614335086178,
      "learning_rate": 3.4704751269870774e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.46875,
      "logps/chosen": -438.0,
      "logps/rejected": -692.0,
      "loss": 0.2183,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.640625,
      "rewards/margins": 2.78125,
      "rewards/rejected": -5.4375,
      "step": 3410
    },
    {
      "epoch": 0.4370886318614608,
      "grad_norm": 12.807394152897395,
      "learning_rate": 3.460185615640234e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.5,
      "logps/chosen": -420.0,
      "logps/rejected": -752.0,
      "loss": 0.2331,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.46875,
      "rewards/margins": 3.40625,
      "rewards/rejected": -5.875,
      "step": 3420
    },
    {
      "epoch": 0.4383666687967282,
      "grad_norm": 14.365989057088319,
      "learning_rate": 3.4498769887991316e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.46875,
      "logps/chosen": -424.0,
      "logps/rejected": -736.0,
      "loss": 0.2503,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.4375,
      "rewards/margins": 3.0625,
      "rewards/rejected": -5.5,
      "step": 3430
    },
    {
      "epoch": 0.43964470573199566,
      "grad_norm": 11.400666849834195,
      "learning_rate": 3.4395494516891885e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.515625,
      "logps/chosen": -454.0,
      "logps/rejected": -692.0,
      "loss": 0.2483,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.8125,
      "rewards/margins": 2.59375,
      "rewards/rejected": -5.40625,
      "step": 3440
    },
    {
      "epoch": 0.44092274266726306,
      "grad_norm": 13.306311012116545,
      "learning_rate": 3.4292032099122956e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5,
      "logps/chosen": -428.0,
      "logps/rejected": -744.0,
      "loss": 0.2242,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.203125,
      "rewards/rejected": -5.8125,
      "step": 3450
    },
    {
      "epoch": 0.4422007796025305,
      "grad_norm": 8.42377616346876,
      "learning_rate": 3.4188384694427166e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.46875,
      "logps/chosen": -426.0,
      "logps/rejected": -660.0,
      "loss": 0.2457,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.59375,
      "rewards/margins": 2.515625,
      "rewards/rejected": -5.09375,
      "step": 3460
    },
    {
      "epoch": 0.4434788165377979,
      "grad_norm": 16.064077555775576,
      "learning_rate": 3.4084554366229884e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.59375,
      "logps/chosen": -470.0,
      "logps/rejected": -720.0,
      "loss": 0.2335,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.109375,
      "rewards/rejected": -5.71875,
      "step": 3470
    },
    {
      "epoch": 0.44475685347306537,
      "grad_norm": 8.997167184239114,
      "learning_rate": 3.398054318159816e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.546875,
      "logps/chosen": -474.0,
      "logps/rejected": -780.0,
      "loss": 0.2281,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.1875,
      "step": 3480
    },
    {
      "epoch": 0.4460348904083328,
      "grad_norm": 11.082449521354196,
      "learning_rate": 3.387635321119955e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.296875,
      "logps/chosen": -432.0,
      "logps/rejected": -712.0,
      "loss": 0.2556,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.578125,
      "rewards/margins": 2.859375,
      "rewards/rejected": -5.4375,
      "step": 3490
    },
    {
      "epoch": 0.44731292734360023,
      "grad_norm": 8.987478762126443,
      "learning_rate": 3.3771986529260914e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.265625,
      "logps/chosen": -424.0,
      "logps/rejected": -744.0,
      "loss": 0.2414,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.15625,
      "rewards/rejected": -5.75,
      "step": 3500
    },
    {
      "epoch": 0.44859096427886763,
      "grad_norm": 14.213029240058788,
      "learning_rate": 3.3667445213527075e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.265625,
      "logps/chosen": -428.0,
      "logps/rejected": -728.0,
      "loss": 0.237,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.546875,
      "rewards/margins": 3.046875,
      "rewards/rejected": -5.59375,
      "step": 3510
    },
    {
      "epoch": 0.4498690012141351,
      "grad_norm": 14.912496967850114,
      "learning_rate": 3.356273134521951e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.53125,
      "logps/chosen": -458.0,
      "logps/rejected": -736.0,
      "loss": 0.2265,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.109375,
      "rewards/rejected": -5.71875,
      "step": 3520
    },
    {
      "epoch": 0.4511470381494025,
      "grad_norm": 11.108904952166707,
      "learning_rate": 3.3457847008994895e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.4375,
      "logps/chosen": -462.0,
      "logps/rejected": -752.0,
      "loss": 0.2532,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.015625,
      "rewards/rejected": -5.875,
      "step": 3530
    },
    {
      "epoch": 0.45242507508466995,
      "grad_norm": 11.480012612605357,
      "learning_rate": 3.3352794292903584e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.359375,
      "logps/chosen": -456.0,
      "logps/rejected": -740.0,
      "loss": 0.2556,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.78125,
      "rewards/margins": 2.890625,
      "rewards/rejected": -5.65625,
      "step": 3540
    },
    {
      "epoch": 0.45370311201993735,
      "grad_norm": 10.185788541305119,
      "learning_rate": 3.324757528834809e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.421875,
      "logps/chosen": -466.0,
      "logps/rejected": -728.0,
      "loss": 0.2402,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.6875,
      "rewards/margins": 3.0,
      "rewards/rejected": -5.6875,
      "step": 3550
    },
    {
      "epoch": 0.4549811489552048,
      "grad_norm": 11.346501222449417,
      "learning_rate": 3.314219209004139e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.390625,
      "logps/chosen": -446.0,
      "logps/rejected": -716.0,
      "loss": 0.2324,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.75,
      "rewards/margins": 2.9375,
      "rewards/rejected": -5.6875,
      "step": 3560
    },
    {
      "epoch": 0.4562591858904722,
      "grad_norm": 10.494438109194048,
      "learning_rate": 3.303664679596526e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.546875,
      "logps/chosen": -472.0,
      "logps/rejected": -804.0,
      "loss": 0.2147,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.375,
      "step": 3570
    },
    {
      "epoch": 0.45753722282573966,
      "grad_norm": 10.35667324122109,
      "learning_rate": 3.293094150732849e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.34375,
      "logps/chosen": -478.0,
      "logps/rejected": -804.0,
      "loss": 0.2097,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.03125,
      "rewards/margins": 3.171875,
      "rewards/rejected": -6.1875,
      "step": 3580
    },
    {
      "epoch": 0.45881525976100707,
      "grad_norm": 12.165116238726197,
      "learning_rate": 3.2825078328525076e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.546875,
      "logps/chosen": -460.0,
      "logps/rejected": -716.0,
      "loss": 0.2467,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.671875,
      "rewards/margins": 2.90625,
      "rewards/rejected": -5.5625,
      "step": 3590
    },
    {
      "epoch": 0.4600932966962745,
      "grad_norm": 12.03512787069905,
      "learning_rate": 3.271905936709231e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.34375,
      "logps/chosen": -474.0,
      "logps/rejected": -768.0,
      "loss": 0.2062,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.921875,
      "rewards/margins": 2.96875,
      "rewards/rejected": -5.90625,
      "step": 3600
    },
    {
      "epoch": 0.4613713336315419,
      "grad_norm": 13.435705559361411,
      "learning_rate": 3.261288673366881e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.515625,
      "logps/chosen": -458.0,
      "logps/rejected": -744.0,
      "loss": 0.2378,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.65625,
      "rewards/margins": 3.1875,
      "rewards/rejected": -5.84375,
      "step": 3610
    },
    {
      "epoch": 0.4626493705668094,
      "grad_norm": 15.608394381025834,
      "learning_rate": 3.250656254195252e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.5,
      "logps/chosen": -440.0,
      "logps/rejected": -728.0,
      "loss": 0.2477,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.0,
      "rewards/rejected": -5.59375,
      "step": 3620
    },
    {
      "epoch": 0.4639274075020768,
      "grad_norm": 10.231033902768853,
      "learning_rate": 3.240008890865864e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.578125,
      "logps/chosen": -466.0,
      "logps/rejected": -760.0,
      "loss": 0.2245,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.140625,
      "rewards/rejected": -6.03125,
      "step": 3630
    },
    {
      "epoch": 0.46520544443734424,
      "grad_norm": 9.590780712884744,
      "learning_rate": 3.2293467953477454e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.4375,
      "logps/chosen": -492.0,
      "logps/rejected": -744.0,
      "loss": 0.2329,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 2.625,
      "rewards/rejected": -5.5625,
      "step": 3640
    },
    {
      "epoch": 0.46648348137261164,
      "grad_norm": 12.021179707291619,
      "learning_rate": 3.218670179903216e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.359375,
      "logps/chosen": -464.0,
      "logps/rejected": -732.0,
      "loss": 0.2346,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.921875,
      "rewards/margins": 2.984375,
      "rewards/rejected": -5.90625,
      "step": 3650
    },
    {
      "epoch": 0.4677615183078791,
      "grad_norm": 14.352184608817506,
      "learning_rate": 3.207979257083658e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.46875,
      "logps/chosen": -460.0,
      "logps/rejected": -816.0,
      "loss": 0.2427,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.640625,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.1875,
      "step": 3660
    },
    {
      "epoch": 0.4690395552431465,
      "grad_norm": 8.063775078522017,
      "learning_rate": 3.19727423972529e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.46875,
      "logps/chosen": -448.0,
      "logps/rejected": -764.0,
      "loss": 0.2145,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.6875,
      "rewards/margins": 3.171875,
      "rewards/rejected": -5.84375,
      "step": 3670
    },
    {
      "epoch": 0.47031759217841396,
      "grad_norm": 9.86097760822928,
      "learning_rate": 3.186555340944923e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5625,
      "logps/chosen": -418.0,
      "logps/rejected": -704.0,
      "loss": 0.2456,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.53125,
      "rewards/margins": 3.109375,
      "rewards/rejected": -5.65625,
      "step": 3680
    },
    {
      "epoch": 0.47159562911368136,
      "grad_norm": 8.183712188723796,
      "learning_rate": 3.175822774135724e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.46875,
      "logps/chosen": -474.0,
      "logps/rejected": -760.0,
      "loss": 0.2329,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.140625,
      "rewards/rejected": -5.875,
      "step": 3690
    },
    {
      "epoch": 0.4728736660489488,
      "grad_norm": 9.075647075009591,
      "learning_rate": 3.165076752962962e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.53125,
      "logps/chosen": -446.0,
      "logps/rejected": -780.0,
      "loss": 0.2084,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.625,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.1875,
      "step": 3700
    },
    {
      "epoch": 0.4741517029842162,
      "grad_norm": 11.39183309609158,
      "learning_rate": 3.15431749135976e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.546875,
      "logps/chosen": -456.0,
      "logps/rejected": -752.0,
      "loss": 0.2162,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.0625,
      "rewards/rejected": -5.875,
      "step": 3710
    },
    {
      "epoch": 0.4754297399194837,
      "grad_norm": 10.391323276920465,
      "learning_rate": 3.1435452035228306e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.703125,
      "logps/chosen": -448.0,
      "logps/rejected": -720.0,
      "loss": 0.2436,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.640625,
      "rewards/margins": 3.046875,
      "rewards/rejected": -5.6875,
      "step": 3720
    },
    {
      "epoch": 0.4767077768547511,
      "grad_norm": 9.937002697261795,
      "learning_rate": 3.132760103908216e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.53125,
      "logps/chosen": -472.0,
      "logps/rejected": -744.0,
      "loss": 0.2229,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.125,
      "rewards/rejected": -5.875,
      "step": 3730
    },
    {
      "epoch": 0.47798581379001853,
      "grad_norm": 14.246129484598066,
      "learning_rate": 3.1219624072270166e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.59375,
      "logps/chosen": -460.0,
      "logps/rejected": -804.0,
      "loss": 0.214,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.65625,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.21875,
      "step": 3740
    },
    {
      "epoch": 0.47926385072528593,
      "grad_norm": 11.93620246674566,
      "learning_rate": 3.111152328441115e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -462.0,
      "logps/rejected": -824.0,
      "loss": 0.256,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.671875,
      "rewards/rejected": -6.5,
      "step": 3750
    },
    {
      "epoch": 0.4805418876605534,
      "grad_norm": 9.513704287172576,
      "learning_rate": 3.100330082758901e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.53125,
      "logps/chosen": -438.0,
      "logps/rejected": -740.0,
      "loss": 0.2079,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.53125,
      "rewards/margins": 3.03125,
      "rewards/rejected": -5.5625,
      "step": 3760
    },
    {
      "epoch": 0.4818199245958208,
      "grad_norm": 10.987399099040324,
      "learning_rate": 3.089495885630983e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.296875,
      "logps/chosen": -440.0,
      "logps/rejected": -720.0,
      "loss": 0.2381,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.421875,
      "rewards/margins": 3.015625,
      "rewards/rejected": -5.4375,
      "step": 3770
    },
    {
      "epoch": 0.48309796153108825,
      "grad_norm": 12.268444866575676,
      "learning_rate": 3.0786499527459005e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.515625,
      "logps/chosen": -444.0,
      "logps/rejected": -688.0,
      "loss": 0.2307,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.515625,
      "rewards/margins": 2.6875,
      "rewards/rejected": -5.1875,
      "step": 3780
    },
    {
      "epoch": 0.48437599846635565,
      "grad_norm": 11.376187701907822,
      "learning_rate": 3.0677925000258285e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.359375,
      "logps/chosen": -438.0,
      "logps/rejected": -728.0,
      "loss": 0.2332,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.578125,
      "rewards/margins": 2.859375,
      "rewards/rejected": -5.4375,
      "step": 3790
    },
    {
      "epoch": 0.4856540354016231,
      "grad_norm": 13.097241496252588,
      "learning_rate": 3.056923743622283e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.4375,
      "logps/chosen": -416.0,
      "logps/rejected": -716.0,
      "loss": 0.2315,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.484375,
      "rewards/margins": 3.078125,
      "rewards/rejected": -5.5625,
      "step": 3800
    },
    {
      "epoch": 0.4869320723368905,
      "grad_norm": 10.624986161727016,
      "learning_rate": 3.0460438999118144e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -474.0,
      "logps/rejected": -756.0,
      "loss": 0.2386,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.1875,
      "rewards/rejected": -6.03125,
      "step": 3810
    },
    {
      "epoch": 0.48821010927215797,
      "grad_norm": 14.012685156301236,
      "learning_rate": 3.035153185491698e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.640625,
      "logps/chosen": -452.0,
      "logps/rejected": -764.0,
      "loss": 0.2357,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.03125,
      "step": 3820
    },
    {
      "epoch": 0.48948814620742537,
      "grad_norm": 11.823646849431594,
      "learning_rate": 3.024251817175628e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.609375,
      "logps/chosen": -460.0,
      "logps/rejected": -760.0,
      "loss": 0.217,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.671875,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.0,
      "step": 3830
    },
    {
      "epoch": 0.4907661831426928,
      "grad_norm": 9.448379629814621,
      "learning_rate": 3.0133400119893947e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.640625,
      "logps/chosen": -416.0,
      "logps/rejected": -756.0,
      "loss": 0.2295,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.40625,
      "rewards/margins": 3.6875,
      "rewards/rejected": -6.09375,
      "step": 3840
    },
    {
      "epoch": 0.4920442200779602,
      "grad_norm": 12.186758531174249,
      "learning_rate": 3.0024179871665706e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.609375,
      "logps/chosen": -468.0,
      "logps/rejected": -760.0,
      "loss": 0.2211,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.15625,
      "rewards/rejected": -6.03125,
      "step": 3850
    },
    {
      "epoch": 0.4933222570132277,
      "grad_norm": 13.8757269089055,
      "learning_rate": 2.9914859601441774e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5,
      "logps/chosen": -456.0,
      "logps/rejected": -756.0,
      "loss": 0.24,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.6875,
      "rewards/margins": 3.296875,
      "rewards/rejected": -5.96875,
      "step": 3860
    },
    {
      "epoch": 0.4946002939484951,
      "grad_norm": 12.669138275619774,
      "learning_rate": 2.980544148558365e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.4375,
      "logps/chosen": -456.0,
      "logps/rejected": -800.0,
      "loss": 0.2407,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.53125,
      "rewards/rejected": -6.34375,
      "step": 3870
    },
    {
      "epoch": 0.49587833088376254,
      "grad_norm": 10.342940879368044,
      "learning_rate": 2.9695927702400767e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.359375,
      "logps/chosen": -452.0,
      "logps/rejected": -756.0,
      "loss": 0.2426,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.65625,
      "rewards/margins": 3.140625,
      "rewards/rejected": -5.78125,
      "step": 3880
    },
    {
      "epoch": 0.49715636781902994,
      "grad_norm": 10.678619672559408,
      "learning_rate": 2.958632043210706e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.296875,
      "logps/chosen": -436.0,
      "logps/rejected": -716.0,
      "loss": 0.2407,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.46875,
      "rewards/margins": 2.9375,
      "rewards/rejected": -5.40625,
      "step": 3890
    },
    {
      "epoch": 0.4984344047542974,
      "grad_norm": 11.983363596064175,
      "learning_rate": 2.9476621856777686e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.46875,
      "logps/chosen": -424.0,
      "logps/rejected": -716.0,
      "loss": 0.2354,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.421875,
      "rewards/margins": 3.109375,
      "rewards/rejected": -5.53125,
      "step": 3900
    },
    {
      "epoch": 0.4997124416895648,
      "grad_norm": 9.692128472306477,
      "learning_rate": 2.9366834160305466e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.421875,
      "logps/chosen": -458.0,
      "logps/rejected": -780.0,
      "loss": 0.2227,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.671875,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.09375,
      "step": 3910
    },
    {
      "epoch": 0.5009904786248323,
      "grad_norm": 8.568902554595155,
      "learning_rate": 2.925695952835746e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.390625,
      "logps/chosen": -460.0,
      "logps/rejected": -796.0,
      "loss": 0.199,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.484375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.09375,
      "step": 3920
    },
    {
      "epoch": 0.5022685155600997,
      "grad_norm": 15.658832510192545,
      "learning_rate": 2.9147000148331483e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.453125,
      "logps/chosen": -468.0,
      "logps/rejected": -776.0,
      "loss": 0.2344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.171875,
      "rewards/rejected": -5.96875,
      "step": 3930
    },
    {
      "epoch": 0.5035465524953671,
      "grad_norm": 10.603410107542684,
      "learning_rate": 2.903695820931249e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.59375,
      "logps/chosen": -470.0,
      "logps/rejected": -744.0,
      "loss": 0.2251,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.875,
      "rewards/margins": 2.921875,
      "rewards/rejected": -5.8125,
      "step": 3940
    },
    {
      "epoch": 0.5048245894306346,
      "grad_norm": 13.023213696995848,
      "learning_rate": 2.8926835902029043e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.390625,
      "logps/chosen": -448.0,
      "logps/rejected": -744.0,
      "loss": 0.2187,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.703125,
      "rewards/margins": 3.09375,
      "rewards/rejected": -5.78125,
      "step": 3950
    },
    {
      "epoch": 0.506102626365902,
      "grad_norm": 12.803594935909652,
      "learning_rate": 2.8816635418809693e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -470.0,
      "logps/rejected": -828.0,
      "loss": 0.2324,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.65625,
      "rewards/rejected": -6.59375,
      "step": 3960
    },
    {
      "epoch": 0.5073806633011694,
      "grad_norm": 14.214826224329563,
      "learning_rate": 2.870635895353931e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.484375,
      "logps/chosen": -458.0,
      "logps/rejected": -760.0,
      "loss": 0.2135,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.796875,
      "rewards/margins": 3.265625,
      "rewards/rejected": -6.0625,
      "step": 3970
    },
    {
      "epoch": 0.5086587002364368,
      "grad_norm": 11.899840196425894,
      "learning_rate": 2.8596008701615456e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.375,
      "logps/chosen": -462.0,
      "logps/rejected": -748.0,
      "loss": 0.2428,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.90625,
      "rewards/margins": 3.03125,
      "rewards/rejected": -5.9375,
      "step": 3980
    },
    {
      "epoch": 0.5099367371717043,
      "grad_norm": 11.761767305501182,
      "learning_rate": 2.8485586859904596e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.453125,
      "logps/chosen": -438.0,
      "logps/rejected": -756.0,
      "loss": 0.258,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.6875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -5.875,
      "step": 3990
    },
    {
      "epoch": 0.5112147741069717,
      "grad_norm": 9.947351908928635,
      "learning_rate": 2.8375095626698464e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.46875,
      "logps/chosen": -446.0,
      "logps/rejected": -704.0,
      "loss": 0.2293,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.6875,
      "rewards/margins": 2.90625,
      "rewards/rejected": -5.59375,
      "step": 4000
    },
    {
      "epoch": 0.5124928110422391,
      "grad_norm": 14.571366188667085,
      "learning_rate": 2.826453720167021e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.390625,
      "logps/chosen": -468.0,
      "logps/rejected": -736.0,
      "loss": 0.2393,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.828125,
      "rewards/margins": 2.765625,
      "rewards/rejected": -5.59375,
      "step": 4010
    },
    {
      "epoch": 0.5137708479775065,
      "grad_norm": 13.755279172005668,
      "learning_rate": 2.815391378583069e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.34375,
      "logps/chosen": -488.0,
      "logps/rejected": -768.0,
      "loss": 0.2395,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.09375,
      "rewards/margins": 3.0,
      "rewards/rejected": -6.09375,
      "step": 4020
    },
    {
      "epoch": 0.515048884912774,
      "grad_norm": 9.089891730196479,
      "learning_rate": 2.8043227581484556e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.546875,
      "logps/chosen": -484.0,
      "logps/rejected": -776.0,
      "loss": 0.217,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.234375,
      "rewards/rejected": -6.0625,
      "step": 4030
    },
    {
      "epoch": 0.5163269218480414,
      "grad_norm": 11.534907277669905,
      "learning_rate": 2.79324807921865e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -446.0,
      "logps/rejected": -704.0,
      "loss": 0.2205,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.625,
      "rewards/margins": 2.734375,
      "rewards/rejected": -5.34375,
      "step": 4040
    },
    {
      "epoch": 0.5176049587833088,
      "grad_norm": 11.275336019189043,
      "learning_rate": 2.7821675622697317e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -478.0,
      "logps/rejected": -836.0,
      "loss": 0.2286,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.78125,
      "rewards/margins": 3.734375,
      "rewards/rejected": -6.53125,
      "step": 4050
    },
    {
      "epoch": 0.5188829957185762,
      "grad_norm": 9.774796668721777,
      "learning_rate": 2.7710814278940057e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.640625,
      "logps/chosen": -504.0,
      "logps/rejected": -780.0,
      "loss": 0.2466,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.015625,
      "rewards/rejected": -6.09375,
      "step": 4060
    },
    {
      "epoch": 0.5201610326538437,
      "grad_norm": 11.278458709304754,
      "learning_rate": 2.759989896795611e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -458.0,
      "logps/rejected": -752.0,
      "loss": 0.2374,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.78125,
      "rewards/margins": 3.046875,
      "rewards/rejected": -5.84375,
      "step": 4070
    },
    {
      "epoch": 0.5214390695891111,
      "grad_norm": 9.46467691902563,
      "learning_rate": 2.748893189786122e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -454.0,
      "logps/rejected": -732.0,
      "loss": 0.2014,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.6875,
      "rewards/margins": 2.921875,
      "rewards/rejected": -5.59375,
      "step": 4080
    },
    {
      "epoch": 0.5227171065243785,
      "grad_norm": 10.686133960054068,
      "learning_rate": 2.7377915277801586e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.4375,
      "logps/chosen": -482.0,
      "logps/rejected": -816.0,
      "loss": 0.2264,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.78125,
      "rewards/margins": 3.6875,
      "rewards/rejected": -6.46875,
      "step": 4090
    },
    {
      "epoch": 0.5239951434596459,
      "grad_norm": 12.438258218326403,
      "learning_rate": 2.726685131790983e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.34375,
      "logps/chosen": -480.0,
      "logps/rejected": -796.0,
      "loss": 0.2237,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.375,
      "step": 4100
    },
    {
      "epoch": 0.5252731803949134,
      "grad_norm": 13.32806206461564,
      "learning_rate": 2.715574222926105e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.375,
      "logps/chosen": -460.0,
      "logps/rejected": -848.0,
      "loss": 0.2251,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.6875,
      "rewards/margins": 3.953125,
      "rewards/rejected": -6.625,
      "step": 4110
    },
    {
      "epoch": 0.5265512173301808,
      "grad_norm": 7.955318664473031,
      "learning_rate": 2.704459022382874e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.4375,
      "logps/chosen": -444.0,
      "logps/rejected": -768.0,
      "loss": 0.2224,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.09375,
      "step": 4120
    },
    {
      "epoch": 0.5278292542654482,
      "grad_norm": 7.561304541551742,
      "learning_rate": 2.6933397514440796e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.4375,
      "logps/chosen": -474.0,
      "logps/rejected": -816.0,
      "loss": 0.226,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.671875,
      "rewards/margins": 3.65625,
      "rewards/rejected": -6.34375,
      "step": 4130
    },
    {
      "epoch": 0.5291072912007156,
      "grad_norm": 11.822994656517746,
      "learning_rate": 2.6822166314735483e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.328125,
      "logps/chosen": -462.0,
      "logps/rejected": -800.0,
      "loss": 0.2296,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.1875,
      "step": 4140
    },
    {
      "epoch": 0.5303853281359832,
      "grad_norm": 9.26408652002619,
      "learning_rate": 2.67108988391173e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.484375,
      "logps/chosen": -468.0,
      "logps/rejected": -748.0,
      "loss": 0.2339,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.03125,
      "rewards/margins": 2.859375,
      "rewards/rejected": -5.875,
      "step": 4150
    },
    {
      "epoch": 0.5316633650712506,
      "grad_norm": 11.338192875381297,
      "learning_rate": 2.659959730271295e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.375,
      "logps/chosen": -446.0,
      "logps/rejected": -776.0,
      "loss": 0.2198,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.515625,
      "rewards/rejected": -6.25,
      "step": 4160
    },
    {
      "epoch": 0.532941402006518,
      "grad_norm": 10.531614626996827,
      "learning_rate": 2.64882639213272e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.3125,
      "logps/chosen": -454.0,
      "logps/rejected": -744.0,
      "loss": 0.2199,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.171875,
      "rewards/rejected": -6.0,
      "step": 4170
    },
    {
      "epoch": 0.5342194389417854,
      "grad_norm": 12.2938774108133,
      "learning_rate": 2.6376900911398823e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.4375,
      "logps/chosen": -476.0,
      "logps/rejected": -800.0,
      "loss": 0.2108,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.90625,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.3125,
      "step": 4180
    },
    {
      "epoch": 0.5354974758770529,
      "grad_norm": 11.290550818971846,
      "learning_rate": 2.626551048995643e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.453125,
      "logps/chosen": -478.0,
      "logps/rejected": -740.0,
      "loss": 0.234,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.03125,
      "rewards/margins": 2.890625,
      "rewards/rejected": -5.9375,
      "step": 4190
    },
    {
      "epoch": 0.5367755128123203,
      "grad_norm": 14.918668730498784,
      "learning_rate": 2.6154094874574326e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.484375,
      "logps/chosen": -470.0,
      "logps/rejected": -792.0,
      "loss": 0.2221,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.21875,
      "rewards/rejected": -6.21875,
      "step": 4200
    },
    {
      "epoch": 0.5380535497475877,
      "grad_norm": 12.798957161618977,
      "learning_rate": 2.60426562833284e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.421875,
      "logps/chosen": -444.0,
      "logps/rejected": -752.0,
      "loss": 0.2218,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.578125,
      "rewards/margins": 3.25,
      "rewards/rejected": -5.84375,
      "step": 4210
    },
    {
      "epoch": 0.5393315866828551,
      "grad_norm": 9.154045605997945,
      "learning_rate": 2.593119693475193e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.515625,
      "logps/chosen": -438.0,
      "logps/rejected": -792.0,
      "loss": 0.2086,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.65625,
      "rewards/margins": 3.71875,
      "rewards/rejected": -6.375,
      "step": 4220
    },
    {
      "epoch": 0.5406096236181226,
      "grad_norm": 11.87671068724257,
      "learning_rate": 2.5819719047791467e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.5,
      "logps/chosen": -462.0,
      "logps/rejected": -812.0,
      "loss": 0.2106,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.28125,
      "step": 4230
    },
    {
      "epoch": 0.54188766055339,
      "grad_norm": 12.147746837997632,
      "learning_rate": 2.570822484176257e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.40625,
      "logps/chosen": -440.0,
      "logps/rejected": -764.0,
      "loss": 0.2107,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.28125,
      "rewards/rejected": -6.03125,
      "step": 4240
    },
    {
      "epoch": 0.5431656974886574,
      "grad_norm": 16.717365052466594,
      "learning_rate": 2.559671653630574e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.53125,
      "logps/chosen": -448.0,
      "logps/rejected": -772.0,
      "loss": 0.2287,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.796875,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.125,
      "step": 4250
    },
    {
      "epoch": 0.5444437344239248,
      "grad_norm": 12.790890636411676,
      "learning_rate": 2.5485196351342146e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.34375,
      "logps/chosen": -466.0,
      "logps/rejected": -784.0,
      "loss": 0.2285,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.15625,
      "step": 4260
    },
    {
      "epoch": 0.5457217713591923,
      "grad_norm": 8.819117954977331,
      "learning_rate": 2.537366650702944e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5,
      "logps/chosen": -466.0,
      "logps/rejected": -776.0,
      "loss": 0.2138,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.671875,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.0,
      "step": 4270
    },
    {
      "epoch": 0.5469998082944597,
      "grad_norm": 10.88244342857703,
      "learning_rate": 2.526212922371758e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.453125,
      "logps/chosen": -454.0,
      "logps/rejected": -744.0,
      "loss": 0.2044,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.0,
      "rewards/rejected": -5.75,
      "step": 4280
    },
    {
      "epoch": 0.5482778452297271,
      "grad_norm": 10.567218017494401,
      "learning_rate": 2.5150586721904653e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.375,
      "logps/chosen": -468.0,
      "logps/rejected": -832.0,
      "loss": 0.2137,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.828125,
      "rewards/rejected": -6.59375,
      "step": 4290
    },
    {
      "epoch": 0.5495558821649945,
      "grad_norm": 12.494049541409721,
      "learning_rate": 2.50390412221926e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.34375,
      "logps/chosen": -444.0,
      "logps/rejected": -764.0,
      "loss": 0.2322,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.09375,
      "step": 4300
    },
    {
      "epoch": 0.550833919100262,
      "grad_norm": 12.329447814303593,
      "learning_rate": 2.492749494524305e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.296875,
      "logps/chosen": -454.0,
      "logps/rejected": -784.0,
      "loss": 0.2082,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.0625,
      "step": 4310
    },
    {
      "epoch": 0.5521119560355294,
      "grad_norm": 9.645139066018423,
      "learning_rate": 2.481595011173312e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.390625,
      "logps/chosen": -430.0,
      "logps/rejected": -748.0,
      "loss": 0.2085,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.15625,
      "rewards/rejected": -5.75,
      "step": 4320
    },
    {
      "epoch": 0.5533899929707968,
      "grad_norm": 14.063134642966283,
      "learning_rate": 2.470440894231118e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.28125,
      "logps/chosen": -472.0,
      "logps/rejected": -744.0,
      "loss": 0.2468,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.875,
      "rewards/margins": 2.921875,
      "rewards/rejected": -5.8125,
      "step": 4330
    },
    {
      "epoch": 0.5546680299060642,
      "grad_norm": 10.487408576383878,
      "learning_rate": 2.4592873657552665e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.34375,
      "logps/chosen": -464.0,
      "logps/rejected": -752.0,
      "loss": 0.2086,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.0625,
      "rewards/rejected": -5.8125,
      "step": 4340
    },
    {
      "epoch": 0.5559460668413317,
      "grad_norm": 12.746146062764499,
      "learning_rate": 2.4481346477915834e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.375,
      "logps/chosen": -452.0,
      "logps/rejected": -784.0,
      "loss": 0.2067,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.1875,
      "step": 4350
    },
    {
      "epoch": 0.5572241037765991,
      "grad_norm": 11.431708194090662,
      "learning_rate": 2.436982962369761e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.5,
      "logps/chosen": -460.0,
      "logps/rejected": -756.0,
      "loss": 0.2139,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.640625,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.0625,
      "step": 4360
    },
    {
      "epoch": 0.5585021407118665,
      "grad_norm": 10.501725020270035,
      "learning_rate": 2.4258325314989364e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.125,
      "logps/chosen": -456.0,
      "logps/rejected": -784.0,
      "loss": 0.2319,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.296875,
      "rewards/rejected": -6.0625,
      "step": 4370
    },
    {
      "epoch": 0.559780177647134,
      "grad_norm": 13.5478767226387,
      "learning_rate": 2.4146835771632674e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.34375,
      "logps/chosen": -460.0,
      "logps/rejected": -700.0,
      "loss": 0.2271,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.796875,
      "rewards/margins": 2.671875,
      "rewards/rejected": -5.46875,
      "step": 4380
    },
    {
      "epoch": 0.5610582145824015,
      "grad_norm": 13.90247289121908,
      "learning_rate": 2.4035363213175206e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.34375,
      "logps/chosen": -454.0,
      "logps/rejected": -716.0,
      "loss": 0.2507,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.65625,
      "rewards/margins": 3.015625,
      "rewards/rejected": -5.6875,
      "step": 4390
    },
    {
      "epoch": 0.5623362515176689,
      "grad_norm": 10.867855904423637,
      "learning_rate": 2.3923909858826473e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.46875,
      "logps/chosen": -458.0,
      "logps/rejected": -728.0,
      "loss": 0.2178,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.8125,
      "rewards/margins": 2.96875,
      "rewards/rejected": -5.78125,
      "step": 4400
    },
    {
      "epoch": 0.5636142884529363,
      "grad_norm": 11.778950333833563,
      "learning_rate": 2.3812477927413648e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.5625,
      "logps/chosen": -468.0,
      "logps/rejected": -744.0,
      "loss": 0.2109,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.109375,
      "rewards/rejected": -6.03125,
      "step": 4410
    },
    {
      "epoch": 0.5648923253882037,
      "grad_norm": 10.711921258615417,
      "learning_rate": 2.370106963733746e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.390625,
      "logps/chosen": -490.0,
      "logps/rejected": -780.0,
      "loss": 0.2248,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.953125,
      "rewards/margins": 2.921875,
      "rewards/rejected": -5.875,
      "step": 4420
    },
    {
      "epoch": 0.5661703623234712,
      "grad_norm": 10.316356615017968,
      "learning_rate": 2.358968720652794e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.4375,
      "logps/chosen": -446.0,
      "logps/rejected": -784.0,
      "loss": 0.2244,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.1875,
      "step": 4430
    },
    {
      "epoch": 0.5674483992587386,
      "grad_norm": 12.351321977632542,
      "learning_rate": 2.3478332852400337e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -420.0,
      "logps/rejected": -760.0,
      "loss": 0.2055,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.546875,
      "rewards/margins": 3.265625,
      "rewards/rejected": -5.8125,
      "step": 4440
    },
    {
      "epoch": 0.568726436194006,
      "grad_norm": 16.201056496064517,
      "learning_rate": 2.3367008791810936e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.625,
      "logps/chosen": -490.0,
      "logps/rejected": -768.0,
      "loss": 0.2744,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.15625,
      "rewards/rejected": -6.0625,
      "step": 4450
    },
    {
      "epoch": 0.5700044731292734,
      "grad_norm": 8.290839287892952,
      "learning_rate": 2.3255717241012922e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.484375,
      "logps/chosen": -438.0,
      "logps/rejected": -772.0,
      "loss": 0.2185,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.390625,
      "rewards/margins": 3.546875,
      "rewards/rejected": -5.9375,
      "step": 4460
    },
    {
      "epoch": 0.5712825100645409,
      "grad_norm": 10.867257748860492,
      "learning_rate": 2.314446041561231e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.46875,
      "logps/chosen": -424.0,
      "logps/rejected": -712.0,
      "loss": 0.2252,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.515625,
      "rewards/margins": 2.953125,
      "rewards/rejected": -5.46875,
      "step": 4470
    },
    {
      "epoch": 0.5725605469998083,
      "grad_norm": 9.841042960420591,
      "learning_rate": 2.303324053052374e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.453125,
      "logps/chosen": -452.0,
      "logps/rejected": -756.0,
      "loss": 0.22,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.703125,
      "rewards/margins": 3.265625,
      "rewards/rejected": -5.96875,
      "step": 4480
    },
    {
      "epoch": 0.5738385839350757,
      "grad_norm": 10.142713716427117,
      "learning_rate": 2.29220597999265e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.453125,
      "logps/chosen": -464.0,
      "logps/rejected": -768.0,
      "loss": 0.2204,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.71875,
      "rewards/margins": 3.140625,
      "rewards/rejected": -5.875,
      "step": 4490
    },
    {
      "epoch": 0.5751166208703431,
      "grad_norm": 10.717642410067702,
      "learning_rate": 2.281092043722034e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5,
      "logps/chosen": -464.0,
      "logps/rejected": -780.0,
      "loss": 0.2052,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.6875,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.03125,
      "step": 4500
    },
    {
      "epoch": 0.5763946578056106,
      "grad_norm": 11.864816987705757,
      "learning_rate": 2.269982465498147e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.34375,
      "logps/chosen": -432.0,
      "logps/rejected": -820.0,
      "loss": 0.2016,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.828125,
      "rewards/rejected": -6.4375,
      "step": 4510
    },
    {
      "epoch": 0.577672694740878,
      "grad_norm": 16.184751073505005,
      "learning_rate": 2.2588774664918503e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.46875,
      "logps/chosen": -464.0,
      "logps/rejected": -772.0,
      "loss": 0.249,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.21875,
      "step": 4520
    },
    {
      "epoch": 0.5789507316761454,
      "grad_norm": 12.654794340813421,
      "learning_rate": 2.2477772677828396e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.421875,
      "logps/chosen": -456.0,
      "logps/rejected": -772.0,
      "loss": 0.2331,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.71875,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.03125,
      "step": 4530
    },
    {
      "epoch": 0.5802287686114128,
      "grad_norm": 14.146249599908726,
      "learning_rate": 2.2366820903552478e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.375,
      "logps/chosen": -442.0,
      "logps/rejected": -716.0,
      "loss": 0.2064,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.609375,
      "rewards/margins": 2.875,
      "rewards/rejected": -5.46875,
      "step": 4540
    },
    {
      "epoch": 0.5815068055466803,
      "grad_norm": 15.440005421062992,
      "learning_rate": 2.2255921550932418e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.484375,
      "logps/chosen": -464.0,
      "logps/rejected": -816.0,
      "loss": 0.2141,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.78125,
      "rewards/margins": 3.609375,
      "rewards/rejected": -6.375,
      "step": 4550
    },
    {
      "epoch": 0.5827848424819477,
      "grad_norm": 11.275659833724442,
      "learning_rate": 2.214507682776627e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.4375,
      "logps/chosen": -480.0,
      "logps/rejected": -840.0,
      "loss": 0.2294,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.765625,
      "rewards/rejected": -6.75,
      "step": 4560
    },
    {
      "epoch": 0.5840628794172151,
      "grad_norm": 13.372791094869948,
      "learning_rate": 2.203428894076454e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.421875,
      "logps/chosen": -454.0,
      "logps/rejected": -736.0,
      "loss": 0.2157,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.671875,
      "rewards/margins": 3.015625,
      "rewards/rejected": -5.6875,
      "step": 4570
    },
    {
      "epoch": 0.5853409163524825,
      "grad_norm": 14.829935890228384,
      "learning_rate": 2.1923560095506196e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.578125,
      "logps/chosen": -444.0,
      "logps/rejected": -764.0,
      "loss": 0.2405,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.671875,
      "rewards/margins": 3.375,
      "rewards/rejected": -6.03125,
      "step": 4580
    },
    {
      "epoch": 0.58661895328775,
      "grad_norm": 9.117818086967732,
      "learning_rate": 2.1812892496394835e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5,
      "logps/chosen": -452.0,
      "logps/rejected": -720.0,
      "loss": 0.2132,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.625,
      "rewards/margins": 3.0,
      "rewards/rejected": -5.625,
      "step": 4590
    },
    {
      "epoch": 0.5878969902230174,
      "grad_norm": 10.980188677753167,
      "learning_rate": 2.170228834661472e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.515625,
      "logps/chosen": -456.0,
      "logps/rejected": -816.0,
      "loss": 0.2167,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.703125,
      "rewards/margins": 3.78125,
      "rewards/rejected": -6.5,
      "step": 4600
    },
    {
      "epoch": 0.5891750271582848,
      "grad_norm": 9.830056678791607,
      "learning_rate": 2.1591749848086992e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.578125,
      "logps/chosen": -464.0,
      "logps/rejected": -784.0,
      "loss": 0.2071,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.34375,
      "step": 4610
    },
    {
      "epoch": 0.5904530640935522,
      "grad_norm": 11.420269806509154,
      "learning_rate": 2.148127920142577e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.46875,
      "logps/chosen": -464.0,
      "logps/rejected": -784.0,
      "loss": 0.2137,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.375,
      "rewards/rejected": -6.125,
      "step": 4620
    },
    {
      "epoch": 0.5917311010288198,
      "grad_norm": 11.590980692318727,
      "learning_rate": 2.1370878605894377e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.375,
      "logps/chosen": -478.0,
      "logps/rejected": -840.0,
      "loss": 0.1981,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -6.59375,
      "step": 4630
    },
    {
      "epoch": 0.5930091379640872,
      "grad_norm": 12.626025998959374,
      "learning_rate": 2.1260550259361576e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.421875,
      "logps/chosen": -506.0,
      "logps/rejected": -784.0,
      "loss": 0.2086,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.1875,
      "rewards/margins": 2.90625,
      "rewards/rejected": -6.09375,
      "step": 4640
    },
    {
      "epoch": 0.5942871748993546,
      "grad_norm": 9.87902474458853,
      "learning_rate": 2.115029635825774e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.640625,
      "logps/chosen": -446.0,
      "logps/rejected": -732.0,
      "loss": 0.2312,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.03125,
      "rewards/rejected": -5.78125,
      "step": 4650
    },
    {
      "epoch": 0.595565211834622,
      "grad_norm": 9.52202380715003,
      "learning_rate": 2.104011909753122e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -474.0,
      "logps/rejected": -776.0,
      "loss": 0.2332,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.15625,
      "step": 4660
    },
    {
      "epoch": 0.5968432487698895,
      "grad_norm": 11.259448764085388,
      "learning_rate": 2.093002067060457e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.46875,
      "logps/chosen": -434.0,
      "logps/rejected": -752.0,
      "loss": 0.2152,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.546875,
      "rewards/margins": 3.390625,
      "rewards/rejected": -5.9375,
      "step": 4670
    },
    {
      "epoch": 0.5981212857051569,
      "grad_norm": 10.93113837973278,
      "learning_rate": 2.082000326933092e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.640625,
      "logps/chosen": -434.0,
      "logps/rejected": -748.0,
      "loss": 0.216,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.640625,
      "rewards/margins": 3.328125,
      "rewards/rejected": -5.96875,
      "step": 4680
    },
    {
      "epoch": 0.5993993226404243,
      "grad_norm": 11.597413005040126,
      "learning_rate": 2.0710069083950342e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.4375,
      "logps/chosen": -498.0,
      "logps/rejected": -760.0,
      "loss": 0.2307,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.171875,
      "rewards/margins": 2.9375,
      "rewards/rejected": -6.09375,
      "step": 4690
    },
    {
      "epoch": 0.6006773595756917,
      "grad_norm": 9.387000730862303,
      "learning_rate": 2.060022030304621e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.453125,
      "logps/chosen": -464.0,
      "logps/rejected": -764.0,
      "loss": 0.227,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.296875,
      "rewards/rejected": -6.125,
      "step": 4700
    },
    {
      "epoch": 0.6019553965109592,
      "grad_norm": 11.024294110280408,
      "learning_rate": 2.0490459113501686e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.546875,
      "logps/chosen": -460.0,
      "logps/rejected": -772.0,
      "loss": 0.2006,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.09375,
      "step": 4710
    },
    {
      "epoch": 0.6032334334462266,
      "grad_norm": 14.989948246725117,
      "learning_rate": 2.0380787700456132e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.4375,
      "logps/chosen": -440.0,
      "logps/rejected": -748.0,
      "loss": 0.196,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.578125,
      "rewards/margins": 3.328125,
      "rewards/rejected": -5.90625,
      "step": 4720
    },
    {
      "epoch": 0.604511470381494,
      "grad_norm": 7.941678551938295,
      "learning_rate": 2.027120824726162e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.609375,
      "logps/chosen": -436.0,
      "logps/rejected": -756.0,
      "loss": 0.2172,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.703125,
      "rewards/margins": 3.1875,
      "rewards/rejected": -5.875,
      "step": 4730
    },
    {
      "epoch": 0.6057895073167614,
      "grad_norm": 10.636934183665666,
      "learning_rate": 2.0161722935439513e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.515625,
      "logps/chosen": -458.0,
      "logps/rejected": -760.0,
      "loss": 0.1965,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.265625,
      "rewards/rejected": -6.125,
      "step": 4740
    },
    {
      "epoch": 0.6070675442520289,
      "grad_norm": 10.124879101099717,
      "learning_rate": 2.0052333944636954e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.65625,
      "logps/chosen": -440.0,
      "logps/rejected": -704.0,
      "loss": 0.2241,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.703125,
      "rewards/margins": 2.890625,
      "rewards/rejected": -5.59375,
      "step": 4750
    },
    {
      "epoch": 0.6083455811872963,
      "grad_norm": 13.926666758312287,
      "learning_rate": 1.994304345258354e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.65625,
      "logps/chosen": -448.0,
      "logps/rejected": -740.0,
      "loss": 0.2055,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.3125,
      "rewards/rejected": -5.9375,
      "step": 4760
    },
    {
      "epoch": 0.6096236181225637,
      "grad_norm": 11.312364250036937,
      "learning_rate": 1.983385363504793e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.40625,
      "logps/chosen": -436.0,
      "logps/rejected": -784.0,
      "loss": 0.1977,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.578125,
      "rewards/margins": 3.640625,
      "rewards/rejected": -6.21875,
      "step": 4770
    },
    {
      "epoch": 0.6109016550578311,
      "grad_norm": 8.459733457183932,
      "learning_rate": 1.9724766665794541e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.703125,
      "logps/chosen": -464.0,
      "logps/rejected": -736.0,
      "loss": 0.2133,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.0,
      "rewards/rejected": -5.8125,
      "step": 4780
    },
    {
      "epoch": 0.6121796919930986,
      "grad_norm": 12.021910640946274,
      "learning_rate": 1.9615784716540293e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.46875,
      "logps/chosen": -470.0,
      "logps/rejected": -820.0,
      "loss": 0.2208,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.65625,
      "rewards/rejected": -6.53125,
      "step": 4790
    },
    {
      "epoch": 0.613457728928366,
      "grad_norm": 17.61328439200812,
      "learning_rate": 1.950690995691132e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.625,
      "logps/chosen": -476.0,
      "logps/rejected": -816.0,
      "loss": 0.2094,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.5,
      "step": 4800
    },
    {
      "epoch": 0.6147357658636334,
      "grad_norm": 12.667668804723492,
      "learning_rate": 1.939814455439986e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5625,
      "logps/chosen": -448.0,
      "logps/rejected": -784.0,
      "loss": 0.2263,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.25,
      "step": 4810
    },
    {
      "epoch": 0.6160138027989008,
      "grad_norm": 12.535166792943917,
      "learning_rate": 1.9289490674321017e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.453125,
      "logps/chosen": -454.0,
      "logps/rejected": -852.0,
      "loss": 0.1937,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.734375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -6.96875,
      "step": 4820
    },
    {
      "epoch": 0.6172918397341683,
      "grad_norm": 15.228931796851096,
      "learning_rate": 1.9180950479769687e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.453125,
      "logps/chosen": -464.0,
      "logps/rejected": -784.0,
      "loss": 0.2291,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.703125,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.15625,
      "step": 4830
    },
    {
      "epoch": 0.6185698766694357,
      "grad_norm": 11.584355932570276,
      "learning_rate": 1.9072526131577542e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -450.0,
      "logps/rejected": -812.0,
      "loss": 0.2171,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.578125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.28125,
      "step": 4840
    },
    {
      "epoch": 0.6198479136047031,
      "grad_norm": 13.195880889004616,
      "learning_rate": 1.8964219788269936e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.546875,
      "logps/chosen": -454.0,
      "logps/rejected": -788.0,
      "loss": 0.2116,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.703125,
      "rewards/margins": 3.625,
      "rewards/rejected": -6.3125,
      "step": 4850
    },
    {
      "epoch": 0.6211259505399706,
      "grad_norm": 13.006155593904623,
      "learning_rate": 1.885603360602298e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.421875,
      "logps/chosen": -438.0,
      "logps/rejected": -756.0,
      "loss": 0.2048,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.640625,
      "rewards/margins": 3.296875,
      "rewards/rejected": -5.9375,
      "step": 4860
    },
    {
      "epoch": 0.6224039874752381,
      "grad_norm": 13.021955078923298,
      "learning_rate": 1.8747969738620588e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.4375,
      "logps/chosen": -408.0,
      "logps/rejected": -736.0,
      "loss": 0.2318,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.328125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -5.65625,
      "step": 4870
    },
    {
      "epoch": 0.6236820244105055,
      "grad_norm": 13.800296744151176,
      "learning_rate": 1.8640030337411655e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.359375,
      "logps/chosen": -464.0,
      "logps/rejected": -776.0,
      "loss": 0.2171,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.1875,
      "rewards/rejected": -6.0625,
      "step": 4880
    },
    {
      "epoch": 0.6249600613457729,
      "grad_norm": 13.34045146874188,
      "learning_rate": 1.8532217551267143e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.40625,
      "logps/chosen": -472.0,
      "logps/rejected": -820.0,
      "loss": 0.2177,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.78125,
      "rewards/margins": 3.609375,
      "rewards/rejected": -6.375,
      "step": 4890
    },
    {
      "epoch": 0.6262380982810403,
      "grad_norm": 10.13525400450503,
      "learning_rate": 1.842453352653735e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.609375,
      "logps/chosen": -488.0,
      "logps/rejected": -804.0,
      "loss": 0.1945,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.4375,
      "step": 4900
    },
    {
      "epoch": 0.6275161352163078,
      "grad_norm": 12.573223381994719,
      "learning_rate": 1.8316980407009213e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.390625,
      "logps/chosen": -442.0,
      "logps/rejected": -800.0,
      "loss": 0.2168,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.6875,
      "rewards/margins": 3.671875,
      "rewards/rejected": -6.375,
      "step": 4910
    },
    {
      "epoch": 0.6287941721515752,
      "grad_norm": 11.04324555461357,
      "learning_rate": 1.820956033386355e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.453125,
      "logps/chosen": -470.0,
      "logps/rejected": -796.0,
      "loss": 0.2164,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.25,
      "step": 4920
    },
    {
      "epoch": 0.6300722090868426,
      "grad_norm": 12.19464084446454,
      "learning_rate": 1.8102275445632508e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.25,
      "logps/chosen": -434.0,
      "logps/rejected": -748.0,
      "loss": 0.219,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.5,
      "rewards/margins": 3.3125,
      "rewards/rejected": -5.8125,
      "step": 4930
    },
    {
      "epoch": 0.63135024602211,
      "grad_norm": 13.993585375132424,
      "learning_rate": 1.7995127878156934e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.546875,
      "logps/chosen": -474.0,
      "logps/rejected": -736.0,
      "loss": 0.2177,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.75,
      "rewards/margins": 2.984375,
      "rewards/rejected": -5.75,
      "step": 4940
    },
    {
      "epoch": 0.6326282829573775,
      "grad_norm": 10.522245839263013,
      "learning_rate": 1.788811976454388e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.46875,
      "logps/chosen": -474.0,
      "logps/rejected": -776.0,
      "loss": 0.2124,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.21875,
      "rewards/rejected": -6.125,
      "step": 4950
    },
    {
      "epoch": 0.6339063198926449,
      "grad_norm": 8.4842854925969,
      "learning_rate": 1.7781253235124143e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -502.0,
      "logps/rejected": -784.0,
      "loss": 0.23,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.046875,
      "rewards/rejected": -6.1875,
      "step": 4960
    },
    {
      "epoch": 0.6351843568279123,
      "grad_norm": 14.807304778247754,
      "learning_rate": 1.7674530417409823e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -456.0,
      "logps/rejected": -760.0,
      "loss": 0.18,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.203125,
      "rewards/rejected": -5.8125,
      "step": 4970
    },
    {
      "epoch": 0.6364623937631797,
      "grad_norm": 10.658628965132982,
      "learning_rate": 1.7567953436052025e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.546875,
      "logps/chosen": -460.0,
      "logps/rejected": -780.0,
      "loss": 0.2227,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.09375,
      "step": 4980
    },
    {
      "epoch": 0.6377404306984472,
      "grad_norm": 18.050402864839704,
      "learning_rate": 1.7461524412798503e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.453125,
      "logps/chosen": -446.0,
      "logps/rejected": -756.0,
      "loss": 0.2307,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.640625,
      "rewards/margins": 3.234375,
      "rewards/rejected": -5.875,
      "step": 4990
    },
    {
      "epoch": 0.6390184676337146,
      "grad_norm": 12.47102024770186,
      "learning_rate": 1.735524546645142e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.6875,
      "logps/chosen": -468.0,
      "logps/rejected": -780.0,
      "loss": 0.1968,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.390625,
      "rewards/rejected": -6.125,
      "step": 5000
    },
    {
      "epoch": 0.640296504568982,
      "grad_norm": 11.619088921859209,
      "learning_rate": 1.7249118712825246e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.546875,
      "logps/chosen": -452.0,
      "logps/rejected": -756.0,
      "loss": 0.2095,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.109375,
      "rewards/rejected": -5.875,
      "step": 5010
    },
    {
      "epoch": 0.6415745415042494,
      "grad_norm": 12.424458026510814,
      "learning_rate": 1.7143146264704522e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.546875,
      "logps/chosen": -446.0,
      "logps/rejected": -744.0,
      "loss": 0.211,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.71875,
      "rewards/margins": 3.125,
      "rewards/rejected": -5.84375,
      "step": 5020
    },
    {
      "epoch": 0.6428525784395169,
      "grad_norm": 12.38342431901027,
      "learning_rate": 1.70373302318019e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.421875,
      "logps/chosen": -448.0,
      "logps/rejected": -824.0,
      "loss": 0.2104,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.71875,
      "rewards/margins": 3.796875,
      "rewards/rejected": -6.5,
      "step": 5030
    },
    {
      "epoch": 0.6441306153747843,
      "grad_norm": 15.162610558236057,
      "learning_rate": 1.6931672720716072e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -474.0,
      "logps/rejected": -796.0,
      "loss": 0.2127,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.25,
      "step": 5040
    },
    {
      "epoch": 0.6454086523100517,
      "grad_norm": 14.483703192788319,
      "learning_rate": 1.6826175834889866e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.625,
      "logps/chosen": -452.0,
      "logps/rejected": -808.0,
      "loss": 0.2071,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.40625,
      "step": 5050
    },
    {
      "epoch": 0.6466866892453191,
      "grad_norm": 9.648128645614554,
      "learning_rate": 1.6720841674568365e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.625,
      "logps/chosen": -460.0,
      "logps/rejected": -816.0,
      "loss": 0.2157,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.78125,
      "rewards/margins": 3.75,
      "rewards/rejected": -6.53125,
      "step": 5060
    },
    {
      "epoch": 0.6479647261805866,
      "grad_norm": 23.865862346858684,
      "learning_rate": 1.6615672336757074e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.640625,
      "logps/chosen": -474.0,
      "logps/rejected": -756.0,
      "loss": 0.2171,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.1875,
      "rewards/rejected": -6.09375,
      "step": 5070
    },
    {
      "epoch": 0.649242763115854,
      "grad_norm": 9.122942043895048,
      "learning_rate": 1.6510669915180238e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.625,
      "logps/chosen": -468.0,
      "logps/rejected": -784.0,
      "loss": 0.2185,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.515625,
      "rewards/rejected": -6.28125,
      "step": 5080
    },
    {
      "epoch": 0.6505208000511215,
      "grad_norm": 8.700473889143753,
      "learning_rate": 1.6405836500239062e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.453125,
      "logps/chosen": -448.0,
      "logps/rejected": -784.0,
      "loss": 0.2013,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.546875,
      "rewards/margins": 3.65625,
      "rewards/rejected": -6.21875,
      "step": 5090
    },
    {
      "epoch": 0.6517988369863889,
      "grad_norm": 8.58272714700139,
      "learning_rate": 1.6301174178970162e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.4375,
      "logps/chosen": -484.0,
      "logps/rejected": -844.0,
      "loss": 0.1853,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -6.6875,
      "step": 5100
    },
    {
      "epoch": 0.6530768739216564,
      "grad_norm": 10.434904812366435,
      "learning_rate": 1.6196685035004015e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.5,
      "logps/chosen": -478.0,
      "logps/rejected": -800.0,
      "loss": 0.2051,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.3125,
      "step": 5110
    },
    {
      "epoch": 0.6543549108569238,
      "grad_norm": 10.150734780413945,
      "learning_rate": 1.6092371148523442e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.609375,
      "logps/chosen": -484.0,
      "logps/rejected": -796.0,
      "loss": 0.2248,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.3125,
      "step": 5120
    },
    {
      "epoch": 0.6556329477921912,
      "grad_norm": 18.90794419506982,
      "learning_rate": 1.5988234596222234e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -464.0,
      "logps/rejected": -812.0,
      "loss": 0.2086,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.78125,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.34375,
      "step": 5130
    },
    {
      "epoch": 0.6569109847274586,
      "grad_norm": 9.150386260705153,
      "learning_rate": 1.5884277451263772e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.59375,
      "logps/chosen": -456.0,
      "logps/rejected": -764.0,
      "loss": 0.2093,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.703125,
      "rewards/margins": 3.234375,
      "rewards/rejected": -5.9375,
      "step": 5140
    },
    {
      "epoch": 0.6581890216627261,
      "grad_norm": 13.406317475529743,
      "learning_rate": 1.57805017832398e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.578125,
      "logps/chosen": -440.0,
      "logps/rejected": -716.0,
      "loss": 0.2035,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.671875,
      "rewards/margins": 3.078125,
      "rewards/rejected": -5.75,
      "step": 5150
    },
    {
      "epoch": 0.6594670585979935,
      "grad_norm": 12.36253799788679,
      "learning_rate": 1.567690965812918e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.484375,
      "logps/chosen": -426.0,
      "logps/rejected": -768.0,
      "loss": 0.2115,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.09375,
      "step": 5160
    },
    {
      "epoch": 0.6607450955332609,
      "grad_norm": 14.964866572198712,
      "learning_rate": 1.5573503138256756e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.5625,
      "logps/chosen": -456.0,
      "logps/rejected": -752.0,
      "loss": 0.2271,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.71875,
      "rewards/margins": 3.21875,
      "rewards/rejected": -5.9375,
      "step": 5170
    },
    {
      "epoch": 0.6620231324685283,
      "grad_norm": 11.247531984449372,
      "learning_rate": 1.5470284282252367e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5,
      "logps/chosen": -436.0,
      "logps/rejected": -764.0,
      "loss": 0.2249,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.640625,
      "rewards/margins": 3.328125,
      "rewards/rejected": -5.96875,
      "step": 5180
    },
    {
      "epoch": 0.6633011694037958,
      "grad_norm": 13.024366783971146,
      "learning_rate": 1.5367255145009784e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.359375,
      "logps/chosen": -452.0,
      "logps/rejected": -800.0,
      "loss": 0.2148,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.71875,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.28125,
      "step": 5190
    },
    {
      "epoch": 0.6645792063390632,
      "grad_norm": 10.32305349442695,
      "learning_rate": 1.5264417777645849e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.390625,
      "logps/chosen": -434.0,
      "logps/rejected": -792.0,
      "loss": 0.2186,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.53125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.21875,
      "step": 5200
    },
    {
      "epoch": 0.6658572432743306,
      "grad_norm": 13.911746789786335,
      "learning_rate": 1.516177422745962e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -454.0,
      "logps/rejected": -780.0,
      "loss": 0.2379,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.0,
      "step": 5210
    },
    {
      "epoch": 0.667135280209598,
      "grad_norm": 13.238423892283896,
      "learning_rate": 1.5059326537891602e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -456.0,
      "logps/rejected": -760.0,
      "loss": 0.2186,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.171875,
      "rewards/rejected": -6.0,
      "step": 5220
    },
    {
      "epoch": 0.6684133171448655,
      "grad_norm": 9.962624344023618,
      "learning_rate": 1.4957076748483121e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.46875,
      "logps/chosen": -456.0,
      "logps/rejected": -820.0,
      "loss": 0.1872,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.6875,
      "rewards/margins": 3.734375,
      "rewards/rejected": -6.40625,
      "step": 5230
    },
    {
      "epoch": 0.6696913540801329,
      "grad_norm": 10.768195949098313,
      "learning_rate": 1.4855026894835636e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.609375,
      "logps/chosen": -470.0,
      "logps/rejected": -772.0,
      "loss": 0.238,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.015625,
      "rewards/rejected": -5.96875,
      "step": 5240
    },
    {
      "epoch": 0.6709693910154003,
      "grad_norm": 11.691079455919125,
      "learning_rate": 1.4753179008570304e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.46875,
      "logps/chosen": -446.0,
      "logps/rejected": -752.0,
      "loss": 0.2157,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.703125,
      "rewards/margins": 3.109375,
      "rewards/rejected": -5.8125,
      "step": 5250
    },
    {
      "epoch": 0.6722474279506677,
      "grad_norm": 11.2932938192697,
      "learning_rate": 1.4651535117287462e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.46875,
      "logps/chosen": -492.0,
      "logps/rejected": -808.0,
      "loss": 0.2059,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.1875,
      "step": 5260
    },
    {
      "epoch": 0.6735254648859352,
      "grad_norm": 16.997394730854445,
      "learning_rate": 1.4550097244526294e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -446.0,
      "logps/rejected": -772.0,
      "loss": 0.2185,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.6875,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.03125,
      "step": 5270
    },
    {
      "epoch": 0.6748035018212026,
      "grad_norm": 9.669706259899636,
      "learning_rate": 1.444886740972455e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.546875,
      "logps/chosen": -460.0,
      "logps/rejected": -772.0,
      "loss": 0.2298,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.6875,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.1875,
      "step": 5280
    },
    {
      "epoch": 0.67608153875647,
      "grad_norm": 9.437062689025364,
      "learning_rate": 1.4347847628178315e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.578125,
      "logps/chosen": -438.0,
      "logps/rejected": -728.0,
      "loss": 0.2062,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.5,
      "rewards/margins": 3.15625,
      "rewards/rejected": -5.65625,
      "step": 5290
    },
    {
      "epoch": 0.6773595756917374,
      "grad_norm": 9.847058799950773,
      "learning_rate": 1.4247039911001952e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.578125,
      "logps/chosen": -464.0,
      "logps/rejected": -760.0,
      "loss": 0.2081,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.0,
      "rewards/rejected": -5.90625,
      "step": 5300
    },
    {
      "epoch": 0.678637612627005,
      "grad_norm": 10.188554988113175,
      "learning_rate": 1.4146446265087966e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.546875,
      "logps/chosen": -476.0,
      "logps/rejected": -800.0,
      "loss": 0.1879,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.375,
      "step": 5310
    },
    {
      "epoch": 0.6799156495622724,
      "grad_norm": 12.92795398816865,
      "learning_rate": 1.4046068693067142e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -476.0,
      "logps/rejected": -780.0,
      "loss": 0.2222,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.265625,
      "rewards/rejected": -6.125,
      "step": 5320
    },
    {
      "epoch": 0.6811936864975398,
      "grad_norm": 10.209095884014243,
      "learning_rate": 1.3945909193268618e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.484375,
      "logps/chosen": -452.0,
      "logps/rejected": -768.0,
      "loss": 0.2025,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.625,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.1875,
      "step": 5330
    },
    {
      "epoch": 0.6824717234328072,
      "grad_norm": 10.681920428633404,
      "learning_rate": 1.3845969759680125e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.65625,
      "logps/chosen": -456.0,
      "logps/rejected": -760.0,
      "loss": 0.1937,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.703125,
      "rewards/margins": 3.265625,
      "rewards/rejected": -5.96875,
      "step": 5340
    },
    {
      "epoch": 0.6837497603680747,
      "grad_norm": 12.91913556754608,
      "learning_rate": 1.374625238190832e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.453125,
      "logps/chosen": -436.0,
      "logps/rejected": -764.0,
      "loss": 0.2044,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.625,
      "rewards/margins": 3.34375,
      "rewards/rejected": -5.96875,
      "step": 5350
    },
    {
      "epoch": 0.6850277973033421,
      "grad_norm": 13.167149708043858,
      "learning_rate": 1.36467590451391e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -486.0,
      "logps/rejected": -780.0,
      "loss": 0.2173,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.78125,
      "rewards/margins": 3.265625,
      "rewards/rejected": -6.0625,
      "step": 5360
    },
    {
      "epoch": 0.6863058342386095,
      "grad_norm": 13.426666394176799,
      "learning_rate": 1.354749173009815e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -468.0,
      "logps/rejected": -812.0,
      "loss": 0.2333,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.34375,
      "step": 5370
    },
    {
      "epoch": 0.6875838711738769,
      "grad_norm": 12.229831680596297,
      "learning_rate": 1.3448452413011487e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -454.0,
      "logps/rejected": -796.0,
      "loss": 0.2009,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.21875,
      "step": 5380
    },
    {
      "epoch": 0.6888619081091444,
      "grad_norm": 12.748873595679346,
      "learning_rate": 1.3349643065566117e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.484375,
      "logps/chosen": -450.0,
      "logps/rejected": -768.0,
      "loss": 0.2209,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.65625,
      "rewards/margins": 3.1875,
      "rewards/rejected": -5.84375,
      "step": 5390
    },
    {
      "epoch": 0.6901399450444118,
      "grad_norm": 10.215533367338631,
      "learning_rate": 1.325106565487077e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.53125,
      "logps/chosen": -440.0,
      "logps/rejected": -784.0,
      "loss": 0.1922,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.453125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -6.25,
      "step": 5400
    },
    {
      "epoch": 0.6914179819796792,
      "grad_norm": 14.041624035954491,
      "learning_rate": 1.3152722143416748e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.515625,
      "logps/chosen": -456.0,
      "logps/rejected": -776.0,
      "loss": 0.2148,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.65625,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.21875,
      "step": 5410
    },
    {
      "epoch": 0.6926960189149466,
      "grad_norm": 11.008377057292659,
      "learning_rate": 1.305461448903889e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -466.0,
      "logps/rejected": -776.0,
      "loss": 0.207,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.375,
      "rewards/rejected": -6.1875,
      "step": 5420
    },
    {
      "epoch": 0.6939740558502141,
      "grad_norm": 12.733796219393351,
      "learning_rate": 1.2956744644876527e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5,
      "logps/chosen": -472.0,
      "logps/rejected": -776.0,
      "loss": 0.2048,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.1875,
      "step": 5430
    },
    {
      "epoch": 0.6952520927854815,
      "grad_norm": 15.095995938186002,
      "learning_rate": 1.2859114559334643e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.375,
      "logps/chosen": -490.0,
      "logps/rejected": -824.0,
      "loss": 0.2273,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.46875,
      "step": 5440
    },
    {
      "epoch": 0.6965301297207489,
      "grad_norm": 11.418568236368777,
      "learning_rate": 1.27617261760451e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.234375,
      "logps/chosen": -452.0,
      "logps/rejected": -820.0,
      "loss": 0.1864,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.734375,
      "rewards/rejected": -6.5,
      "step": 5450
    },
    {
      "epoch": 0.6978081666560163,
      "grad_norm": 12.940479037663385,
      "learning_rate": 1.26645814338279e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -448.0,
      "logps/rejected": -764.0,
      "loss": 0.2119,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.78125,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.09375,
      "step": 5460
    },
    {
      "epoch": 0.6990862035912838,
      "grad_norm": 10.311716219671611,
      "learning_rate": 1.2567682266652603e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -458.0,
      "logps/rejected": -776.0,
      "loss": 0.2164,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.21875,
      "step": 5470
    },
    {
      "epoch": 0.7003642405265512,
      "grad_norm": 13.868109491497917,
      "learning_rate": 1.2471030603599848e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.3125,
      "logps/chosen": -452.0,
      "logps/rejected": -784.0,
      "loss": 0.2094,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.6875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.25,
      "step": 5480
    },
    {
      "epoch": 0.7016422774618186,
      "grad_norm": 14.803772463760248,
      "learning_rate": 1.2374628368822914e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.53125,
      "logps/chosen": -436.0,
      "logps/rejected": -740.0,
      "loss": 0.2143,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.234375,
      "rewards/rejected": -5.84375,
      "step": 5490
    },
    {
      "epoch": 0.702920314397086,
      "grad_norm": 11.977854961049845,
      "learning_rate": 1.2278477481509462e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -450.0,
      "logps/rejected": -752.0,
      "loss": 0.2101,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.703125,
      "rewards/margins": 3.140625,
      "rewards/rejected": -5.84375,
      "step": 5500
    },
    {
      "epoch": 0.7041983513323535,
      "grad_norm": 11.943556790431966,
      "learning_rate": 1.2182579855843247e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.59375,
      "logps/chosen": -430.0,
      "logps/rejected": -760.0,
      "loss": 0.21,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.71875,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.0625,
      "step": 5510
    },
    {
      "epoch": 0.7054763882676209,
      "grad_norm": 10.79785525614875,
      "learning_rate": 1.2086937400966129e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -468.0,
      "logps/rejected": -796.0,
      "loss": 0.2095,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.4375,
      "step": 5520
    },
    {
      "epoch": 0.7067544252028883,
      "grad_norm": 12.958894086068861,
      "learning_rate": 1.1991552020939918e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.453125,
      "logps/chosen": -470.0,
      "logps/rejected": -796.0,
      "loss": 0.2029,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.265625,
      "rewards/rejected": -6.125,
      "step": 5530
    },
    {
      "epoch": 0.7080324621381557,
      "grad_norm": 9.492831176728052,
      "learning_rate": 1.1896425614708589e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.40625,
      "logps/chosen": -448.0,
      "logps/rejected": -772.0,
      "loss": 0.199,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.0625,
      "step": 5540
    },
    {
      "epoch": 0.7093104990734233,
      "grad_norm": 10.853095578059968,
      "learning_rate": 1.1801560076060444e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -454.0,
      "logps/rejected": -796.0,
      "loss": 0.2037,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.671875,
      "rewards/rejected": -6.40625,
      "step": 5550
    },
    {
      "epoch": 0.7105885360086907,
      "grad_norm": 10.165574096025896,
      "learning_rate": 1.170695729359037e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.578125,
      "logps/chosen": -488.0,
      "logps/rejected": -776.0,
      "loss": 0.2008,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.140625,
      "rewards/rejected": -6.125,
      "step": 5560
    },
    {
      "epoch": 0.711866572943958,
      "grad_norm": 8.917862630671223,
      "learning_rate": 1.1612619150662303e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.5625,
      "logps/chosen": -478.0,
      "logps/rejected": -808.0,
      "loss": 0.2062,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.90625,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.46875,
      "step": 5570
    },
    {
      "epoch": 0.7131446098792255,
      "grad_norm": 11.700875106475753,
      "learning_rate": 1.151854752537168e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5,
      "logps/chosen": -470.0,
      "logps/rejected": -796.0,
      "loss": 0.204,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.09375,
      "rewards/rejected": -6.09375,
      "step": 5580
    },
    {
      "epoch": 0.714422646814493,
      "grad_norm": 18.89780741385279,
      "learning_rate": 1.1424744290508078e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -492.0,
      "logps/rejected": -800.0,
      "loss": 0.1906,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.4375,
      "step": 5590
    },
    {
      "epoch": 0.7157006837497604,
      "grad_norm": 15.39242311818398,
      "learning_rate": 1.1331211313517925e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.28125,
      "logps/chosen": -462.0,
      "logps/rejected": -812.0,
      "loss": 0.1959,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.78125,
      "rewards/margins": 3.65625,
      "rewards/rejected": -6.4375,
      "step": 5600
    },
    {
      "epoch": 0.7169787206850278,
      "grad_norm": 11.341856153072559,
      "learning_rate": 1.1237950456467308e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.359375,
      "logps/chosen": -502.0,
      "logps/rejected": -856.0,
      "loss": 0.2132,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.59375,
      "step": 5610
    },
    {
      "epoch": 0.7182567576202952,
      "grad_norm": 10.699908299692147,
      "learning_rate": 1.1144963576004954e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.359375,
      "logps/chosen": -468.0,
      "logps/rejected": -796.0,
      "loss": 0.2235,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.40625,
      "step": 5620
    },
    {
      "epoch": 0.7195347945555627,
      "grad_norm": 15.760227309133208,
      "learning_rate": 1.105225252332519e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.578125,
      "logps/chosen": -484.0,
      "logps/rejected": -792.0,
      "loss": 0.2276,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.90625,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.25,
      "step": 5630
    },
    {
      "epoch": 0.7208128314908301,
      "grad_norm": 14.75476244965707,
      "learning_rate": 1.0959819144131144e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.5,
      "logps/chosen": -446.0,
      "logps/rejected": -788.0,
      "loss": 0.1928,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.65625,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.21875,
      "step": 5640
    },
    {
      "epoch": 0.7220908684260975,
      "grad_norm": 10.807737408392212,
      "learning_rate": 1.0867665278597999e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.40625,
      "logps/chosen": -498.0,
      "logps/rejected": -816.0,
      "loss": 0.1992,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.53125,
      "step": 5650
    },
    {
      "epoch": 0.7233689053613649,
      "grad_norm": 11.614538762758693,
      "learning_rate": 1.0775792761336324e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.453125,
      "logps/chosen": -486.0,
      "logps/rejected": -784.0,
      "loss": 0.2023,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.28125,
      "rewards/rejected": -6.28125,
      "step": 5660
    },
    {
      "epoch": 0.7246469422966324,
      "grad_norm": 17.917741712500096,
      "learning_rate": 1.0684203421355587e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -488.0,
      "logps/rejected": -852.0,
      "loss": 0.2253,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.828125,
      "rewards/rejected": -6.65625,
      "step": 5670
    },
    {
      "epoch": 0.7259249792318998,
      "grad_norm": 9.352258211544449,
      "learning_rate": 1.0592899082027715e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.375,
      "logps/chosen": -442.0,
      "logps/rejected": -764.0,
      "loss": 0.1961,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.59375,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.03125,
      "step": 5680
    },
    {
      "epoch": 0.7272030161671672,
      "grad_norm": 14.341626084734576,
      "learning_rate": 1.050188156105081e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.46875,
      "logps/chosen": -480.0,
      "logps/rejected": -804.0,
      "loss": 0.2242,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.28125,
      "step": 5690
    },
    {
      "epoch": 0.7284810531024346,
      "grad_norm": 10.571669152090191,
      "learning_rate": 1.0411152670412975e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.15625,
      "logps/chosen": -446.0,
      "logps/rejected": -808.0,
      "loss": 0.2046,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.671875,
      "rewards/margins": 3.53125,
      "rewards/rejected": -6.1875,
      "step": 5700
    },
    {
      "epoch": 0.7297590900377021,
      "grad_norm": 13.113067954543132,
      "learning_rate": 1.03207142163562e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.484375,
      "logps/chosen": -480.0,
      "logps/rejected": -800.0,
      "loss": 0.211,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.40625,
      "step": 5710
    },
    {
      "epoch": 0.7310371269729695,
      "grad_norm": 13.594154745856802,
      "learning_rate": 1.023056799934045e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.4375,
      "logps/chosen": -478.0,
      "logps/rejected": -792.0,
      "loss": 0.2198,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.375,
      "step": 5720
    },
    {
      "epoch": 0.7323151639082369,
      "grad_norm": 12.153437818944957,
      "learning_rate": 1.0140715814007783e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -498.0,
      "logps/rejected": -780.0,
      "loss": 0.2241,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.203125,
      "rewards/margins": 3.078125,
      "rewards/rejected": -6.28125,
      "step": 5730
    },
    {
      "epoch": 0.7335932008435043,
      "grad_norm": 10.701698795347491,
      "learning_rate": 1.0051159449146643e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.421875,
      "logps/chosen": -470.0,
      "logps/rejected": -752.0,
      "loss": 0.2187,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.03125,
      "rewards/rejected": -5.96875,
      "step": 5740
    },
    {
      "epoch": 0.7348712377787718,
      "grad_norm": 12.99341874521499,
      "learning_rate": 9.961900687656242e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.546875,
      "logps/chosen": -464.0,
      "logps/rejected": -772.0,
      "loss": 0.2073,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.15625,
      "rewards/rejected": -6.125,
      "step": 5750
    },
    {
      "epoch": 0.7361492747140392,
      "grad_norm": 20.999672226991667,
      "learning_rate": 9.872941306511056e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.34375,
      "logps/chosen": -444.0,
      "logps/rejected": -768.0,
      "loss": 0.1877,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.640625,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.09375,
      "step": 5760
    },
    {
      "epoch": 0.7374273116493066,
      "grad_norm": 14.706633982915617,
      "learning_rate": 9.784283076725494e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.609375,
      "logps/chosen": -464.0,
      "logps/rejected": -780.0,
      "loss": 0.189,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.296875,
      "rewards/rejected": -6.25,
      "step": 5770
    },
    {
      "epoch": 0.738705348584574,
      "grad_norm": 11.868349045860697,
      "learning_rate": 9.695927763318568e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.46875,
      "logps/chosen": -478.0,
      "logps/rejected": -784.0,
      "loss": 0.234,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.03125,
      "rewards/margins": 3.234375,
      "rewards/rejected": -6.25,
      "step": 5780
    },
    {
      "epoch": 0.7399833855198416,
      "grad_norm": 11.008385348508503,
      "learning_rate": 9.607877125278823e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.515625,
      "logps/chosen": -488.0,
      "logps/rejected": -780.0,
      "loss": 0.2163,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.125,
      "rewards/margins": 2.96875,
      "rewards/rejected": -6.09375,
      "step": 5790
    },
    {
      "epoch": 0.741261422455109,
      "grad_norm": 16.27591688653491,
      "learning_rate": 9.520132915529269e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.390625,
      "logps/chosen": -458.0,
      "logps/rejected": -812.0,
      "loss": 0.2302,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.3125,
      "step": 5800
    },
    {
      "epoch": 0.7425394593903764,
      "grad_norm": 10.423441686869026,
      "learning_rate": 9.432696880892513e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.484375,
      "logps/chosen": -462.0,
      "logps/rejected": -780.0,
      "loss": 0.1977,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.703125,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.15625,
      "step": 5810
    },
    {
      "epoch": 0.7438174963256438,
      "grad_norm": 11.142890102670558,
      "learning_rate": 9.345570762055968e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.53125,
      "logps/chosen": -466.0,
      "logps/rejected": -768.0,
      "loss": 0.2155,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.21875,
      "rewards/rejected": -6.0625,
      "step": 5820
    },
    {
      "epoch": 0.7450955332609113,
      "grad_norm": 12.150592710154923,
      "learning_rate": 9.2587562935372e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -478.0,
      "logps/rejected": -784.0,
      "loss": 0.2107,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.265625,
      "rewards/rejected": -6.25,
      "step": 5830
    },
    {
      "epoch": 0.7463735701961787,
      "grad_norm": 16.87758055753205,
      "learning_rate": 9.172255203649429e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5,
      "logps/chosen": -494.0,
      "logps/rejected": -840.0,
      "loss": 0.2153,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.6875,
      "rewards/rejected": -6.71875,
      "step": 5840
    },
    {
      "epoch": 0.7476516071314461,
      "grad_norm": 11.623608539202431,
      "learning_rate": 9.086069214467062e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -486.0,
      "logps/rejected": -844.0,
      "loss": 0.2093,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -6.6875,
      "step": 5850
    },
    {
      "epoch": 0.7489296440667135,
      "grad_norm": 10.419591981114403,
      "learning_rate": 9.00020004179145e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.40625,
      "logps/chosen": -444.0,
      "logps/rejected": -808.0,
      "loss": 0.1972,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.3125,
      "step": 5860
    },
    {
      "epoch": 0.750207681001981,
      "grad_norm": 13.580492940326726,
      "learning_rate": 8.914649395116752e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -480.0,
      "logps/rejected": -796.0,
      "loss": 0.2124,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.40625,
      "step": 5870
    },
    {
      "epoch": 0.7514857179372484,
      "grad_norm": 11.089943534878916,
      "learning_rate": 8.829418977595815e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.375,
      "logps/chosen": -468.0,
      "logps/rejected": -780.0,
      "loss": 0.2468,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -6.0625,
      "step": 5880
    },
    {
      "epoch": 0.7527637548725158,
      "grad_norm": 9.116676950261446,
      "learning_rate": 8.744510486006376e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -470.0,
      "logps/rejected": -804.0,
      "loss": 0.2065,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.09375,
      "step": 5890
    },
    {
      "epoch": 0.7540417918077832,
      "grad_norm": 10.119597143529983,
      "learning_rate": 8.659925610717208e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -462.0,
      "logps/rejected": -816.0,
      "loss": 0.2117,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.65625,
      "rewards/rejected": -6.5,
      "step": 5900
    },
    {
      "epoch": 0.7553198287430507,
      "grad_norm": 12.621530699111664,
      "learning_rate": 8.575666035654482e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5625,
      "logps/chosen": -488.0,
      "logps/rejected": -796.0,
      "loss": 0.1991,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.40625,
      "step": 5910
    },
    {
      "epoch": 0.7565978656783181,
      "grad_norm": 9.314225992857494,
      "learning_rate": 8.491733438268281e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -478.0,
      "logps/rejected": -780.0,
      "loss": 0.1931,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.21875,
      "step": 5920
    },
    {
      "epoch": 0.7578759026135855,
      "grad_norm": 16.37857920729833,
      "learning_rate": 8.408129489499147e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -488.0,
      "logps/rejected": -812.0,
      "loss": 0.2184,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.40625,
      "step": 5930
    },
    {
      "epoch": 0.7591539395488529,
      "grad_norm": 11.99727837998784,
      "learning_rate": 8.324855853744858e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.46875,
      "logps/chosen": -484.0,
      "logps/rejected": -840.0,
      "loss": 0.2063,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -6.6875,
      "step": 5940
    },
    {
      "epoch": 0.7604319764841204,
      "grad_norm": 9.216746433900035,
      "learning_rate": 8.241914188827278e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.546875,
      "logps/chosen": -478.0,
      "logps/rejected": -832.0,
      "loss": 0.1959,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.6875,
      "rewards/margins": 3.734375,
      "rewards/rejected": -6.4375,
      "step": 5950
    },
    {
      "epoch": 0.7617100134193878,
      "grad_norm": 13.580083086430301,
      "learning_rate": 8.159306145959346e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.546875,
      "logps/chosen": -446.0,
      "logps/rejected": -824.0,
      "loss": 0.198,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.6875,
      "rewards/rejected": -6.46875,
      "step": 5960
    },
    {
      "epoch": 0.7629880503546552,
      "grad_norm": 13.400143627250559,
      "learning_rate": 8.077033369712233e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -478.0,
      "logps/rejected": -812.0,
      "loss": 0.2034,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.90625,
      "rewards/margins": 3.71875,
      "rewards/rejected": -6.625,
      "step": 5970
    },
    {
      "epoch": 0.7642660872899226,
      "grad_norm": 11.326252954210227,
      "learning_rate": 7.99509749798255e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.546875,
      "logps/chosen": -492.0,
      "logps/rejected": -764.0,
      "loss": 0.2052,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.078125,
      "rewards/rejected": -6.03125,
      "step": 5980
    },
    {
      "epoch": 0.7655441242251901,
      "grad_norm": 12.548527463686868,
      "learning_rate": 7.913500161959808e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.421875,
      "logps/chosen": -456.0,
      "logps/rejected": -816.0,
      "loss": 0.1915,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.703125,
      "rewards/margins": 3.828125,
      "rewards/rejected": -6.53125,
      "step": 5990
    },
    {
      "epoch": 0.7668221611604575,
      "grad_norm": 13.912794161523047,
      "learning_rate": 7.832242986093877e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.6875,
      "logps/chosen": -492.0,
      "logps/rejected": -824.0,
      "loss": 0.2026,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.5625,
      "step": 6000
    },
    {
      "epoch": 0.7681001980957249,
      "grad_norm": 13.669152575359544,
      "learning_rate": 7.751327588062686e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.640625,
      "logps/chosen": -460.0,
      "logps/rejected": -788.0,
      "loss": 0.1958,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.53125,
      "rewards/rejected": -6.28125,
      "step": 6010
    },
    {
      "epoch": 0.7693782350309923,
      "grad_norm": 14.456260083434469,
      "learning_rate": 7.670755578740012e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5,
      "logps/chosen": -484.0,
      "logps/rejected": -904.0,
      "loss": 0.1872,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.9375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -6.9375,
      "step": 6020
    },
    {
      "epoch": 0.7706562719662599,
      "grad_norm": 16.75764755565331,
      "learning_rate": 7.590528562163393e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5,
      "logps/chosen": -484.0,
      "logps/rejected": -828.0,
      "loss": 0.203,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.71875,
      "rewards/rejected": -6.6875,
      "step": 6030
    },
    {
      "epoch": 0.7719343089015273,
      "grad_norm": 10.950795457050495,
      "learning_rate": 7.51064813550224e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.671875,
      "logps/chosen": -472.0,
      "logps/rejected": -788.0,
      "loss": 0.1907,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.390625,
      "rewards/rejected": -6.34375,
      "step": 6040
    },
    {
      "epoch": 0.7732123458367947,
      "grad_norm": 14.947548969410093,
      "learning_rate": 7.431115889025969e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.625,
      "logps/chosen": -472.0,
      "logps/rejected": -800.0,
      "loss": 0.2056,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.90625,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.4375,
      "step": 6050
    },
    {
      "epoch": 0.7744903827720621,
      "grad_norm": 11.180981124335842,
      "learning_rate": 7.35193340607242e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.578125,
      "logps/chosen": -474.0,
      "logps/rejected": -816.0,
      "loss": 0.1956,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.671875,
      "rewards/rejected": -6.5,
      "step": 6060
    },
    {
      "epoch": 0.7757684197073296,
      "grad_norm": 17.8436799006314,
      "learning_rate": 7.27310226301627e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.5,
      "logps/chosen": -458.0,
      "logps/rejected": -792.0,
      "loss": 0.2314,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.1875,
      "step": 6070
    },
    {
      "epoch": 0.777046456642597,
      "grad_norm": 14.07707145036626,
      "learning_rate": 7.194624029237686e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.328125,
      "logps/chosen": -464.0,
      "logps/rejected": -808.0,
      "loss": 0.1898,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.34375,
      "step": 6080
    },
    {
      "epoch": 0.7783244935778644,
      "grad_norm": 14.841727421269203,
      "learning_rate": 7.116500267091077e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.703125,
      "logps/chosen": -460.0,
      "logps/rejected": -828.0,
      "loss": 0.2113,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.796875,
      "rewards/margins": 3.90625,
      "rewards/rejected": -6.71875,
      "step": 6090
    },
    {
      "epoch": 0.7796025305131318,
      "grad_norm": 15.783057307090695,
      "learning_rate": 7.038732531873972e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -476.0,
      "logps/rejected": -800.0,
      "loss": 0.2018,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.375,
      "rewards/rejected": -6.3125,
      "step": 6100
    },
    {
      "epoch": 0.7808805674483993,
      "grad_norm": 13.324934762655731,
      "learning_rate": 6.961322371796105e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -448.0,
      "logps/rejected": -784.0,
      "loss": 0.1887,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.1875,
      "step": 6110
    },
    {
      "epoch": 0.7821586043836667,
      "grad_norm": 11.586261941133285,
      "learning_rate": 6.884271327948524e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.65625,
      "logps/chosen": -458.0,
      "logps/rejected": -784.0,
      "loss": 0.1916,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.1875,
      "step": 6120
    },
    {
      "epoch": 0.7834366413189341,
      "grad_norm": 10.518632368705813,
      "learning_rate": 6.807580934272961e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.46875,
      "logps/chosen": -472.0,
      "logps/rejected": -816.0,
      "loss": 0.2095,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.40625,
      "step": 6130
    },
    {
      "epoch": 0.7847146782542015,
      "grad_norm": 10.755005442652692,
      "learning_rate": 6.731252717531288e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -474.0,
      "logps/rejected": -832.0,
      "loss": 0.1986,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.78125,
      "rewards/rejected": -6.59375,
      "step": 6140
    },
    {
      "epoch": 0.785992715189469,
      "grad_norm": 11.167558515662186,
      "learning_rate": 6.655288197275103e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.46875,
      "logps/chosen": -462.0,
      "logps/rejected": -864.0,
      "loss": 0.2102,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.828125,
      "rewards/margins": 4.0,
      "rewards/rejected": -6.84375,
      "step": 6150
    },
    {
      "epoch": 0.7872707521247364,
      "grad_norm": 10.997037338261388,
      "learning_rate": 6.579688885815488e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5,
      "logps/chosen": -476.0,
      "logps/rejected": -824.0,
      "loss": 0.1958,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.734375,
      "rewards/rejected": -6.53125,
      "step": 6160
    },
    {
      "epoch": 0.7885487890600038,
      "grad_norm": 18.392532783286537,
      "learning_rate": 6.504456288192911e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.546875,
      "logps/chosen": -494.0,
      "logps/rejected": -820.0,
      "loss": 0.1935,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.53125,
      "step": 6170
    },
    {
      "epoch": 0.7898268259952712,
      "grad_norm": 14.118633165268266,
      "learning_rate": 6.429591902147238e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.46875,
      "logps/chosen": -460.0,
      "logps/rejected": -848.0,
      "loss": 0.1921,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.828125,
      "rewards/rejected": -6.65625,
      "step": 6180
    },
    {
      "epoch": 0.7911048629305387,
      "grad_norm": 11.003000634381554,
      "learning_rate": 6.355097218087968e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5625,
      "logps/chosen": -454.0,
      "logps/rejected": -820.0,
      "loss": 0.186,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.984375,
      "rewards/rejected": -6.71875,
      "step": 6190
    },
    {
      "epoch": 0.7923828998658061,
      "grad_norm": 16.451053017395726,
      "learning_rate": 6.280973719064486e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -468.0,
      "logps/rejected": -792.0,
      "loss": 0.2099,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.21875,
      "step": 6200
    },
    {
      "epoch": 0.7936609368010735,
      "grad_norm": 14.557755810185025,
      "learning_rate": 6.207222880736618e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -504.0,
      "logps/rejected": -812.0,
      "loss": 0.218,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.59375,
      "step": 6210
    },
    {
      "epoch": 0.7949389737363409,
      "grad_norm": 16.280563573552243,
      "learning_rate": 6.133846171345189e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.375,
      "logps/chosen": -490.0,
      "logps/rejected": -844.0,
      "loss": 0.2182,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.03125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.71875,
      "step": 6220
    },
    {
      "epoch": 0.7962170106716084,
      "grad_norm": 10.122510512033415,
      "learning_rate": 6.060845051682809e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -468.0,
      "logps/rejected": -780.0,
      "loss": 0.1991,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.1875,
      "step": 6230
    },
    {
      "epoch": 0.7974950476068758,
      "grad_norm": 14.704654636546543,
      "learning_rate": 5.988220975064839e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.46875,
      "logps/chosen": -500.0,
      "logps/rejected": -816.0,
      "loss": 0.2115,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.171875,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.40625,
      "step": 6240
    },
    {
      "epoch": 0.7987730845421432,
      "grad_norm": 14.540059964735683,
      "learning_rate": 5.915975387300382e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.546875,
      "logps/chosen": -474.0,
      "logps/rejected": -792.0,
      "loss": 0.2097,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.28125,
      "step": 6250
    },
    {
      "epoch": 0.8000511214774106,
      "grad_norm": 11.755177997757091,
      "learning_rate": 5.844109726663568e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -496.0,
      "logps/rejected": -776.0,
      "loss": 0.2016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.078125,
      "rewards/rejected": -6.125,
      "step": 6260
    },
    {
      "epoch": 0.8013291584126782,
      "grad_norm": 13.264113655423577,
      "learning_rate": 5.772625423864866e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.421875,
      "logps/chosen": -486.0,
      "logps/rejected": -776.0,
      "loss": 0.2136,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.15625,
      "rewards/rejected": -6.15625,
      "step": 6270
    },
    {
      "epoch": 0.8026071953479456,
      "grad_norm": 11.50911085835101,
      "learning_rate": 5.701523902022659e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.453125,
      "logps/chosen": -492.0,
      "logps/rejected": -832.0,
      "loss": 0.2042,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.46875,
      "step": 6280
    },
    {
      "epoch": 0.803885232283213,
      "grad_norm": 12.66770927387518,
      "learning_rate": 5.6308065766348335e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.484375,
      "logps/chosen": -476.0,
      "logps/rejected": -792.0,
      "loss": 0.204,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.25,
      "step": 6290
    },
    {
      "epoch": 0.8051632692184804,
      "grad_norm": 9.8055874445722,
      "learning_rate": 5.5604748555506706e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.40625,
      "logps/chosen": -462.0,
      "logps/rejected": -836.0,
      "loss": 0.1899,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -6.78125,
      "step": 6300
    },
    {
      "epoch": 0.8064413061537479,
      "grad_norm": 12.908890335400274,
      "learning_rate": 5.490530138942806e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.4375,
      "logps/chosen": -484.0,
      "logps/rejected": -856.0,
      "loss": 0.2033,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 3.765625,
      "rewards/rejected": -6.875,
      "step": 6310
    },
    {
      "epoch": 0.8077193430890153,
      "grad_norm": 11.387706698406237,
      "learning_rate": 5.4209738192793146e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5,
      "logps/chosen": -490.0,
      "logps/rejected": -820.0,
      "loss": 0.2115,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.5,
      "step": 6320
    },
    {
      "epoch": 0.8089973800242827,
      "grad_norm": 10.203671312774231,
      "learning_rate": 5.3518072812960444e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -492.0,
      "logps/rejected": -836.0,
      "loss": 0.1882,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.640625,
      "rewards/rejected": -6.6875,
      "step": 6330
    },
    {
      "epoch": 0.8102754169595501,
      "grad_norm": 12.77406950116878,
      "learning_rate": 5.283031901969001e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.46875,
      "logps/chosen": -488.0,
      "logps/rejected": -788.0,
      "loss": 0.2196,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.296875,
      "rewards/rejected": -6.28125,
      "step": 6340
    },
    {
      "epoch": 0.8115534538948176,
      "grad_norm": 10.888244680367457,
      "learning_rate": 5.214649050486961e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.484375,
      "logps/chosen": -480.0,
      "logps/rejected": -776.0,
      "loss": 0.2173,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.15625,
      "rewards/rejected": -6.03125,
      "step": 6350
    },
    {
      "epoch": 0.812831490830085,
      "grad_norm": 12.709077484960734,
      "learning_rate": 5.146660088224209e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.453125,
      "logps/chosen": -460.0,
      "logps/rejected": -780.0,
      "loss": 0.2287,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.21875,
      "rewards/rejected": -6.0625,
      "step": 6360
    },
    {
      "epoch": 0.8141095277653524,
      "grad_norm": 12.531854090339671,
      "learning_rate": 5.079066368713425e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5625,
      "logps/chosen": -454.0,
      "logps/rejected": -808.0,
      "loss": 0.1945,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.65625,
      "rewards/rejected": -6.4375,
      "step": 6370
    },
    {
      "epoch": 0.8153875647006198,
      "grad_norm": 11.695995529582913,
      "learning_rate": 5.011869237618768e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.53125,
      "logps/chosen": -444.0,
      "logps/rejected": -792.0,
      "loss": 0.2005,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.703125,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.1875,
      "step": 6380
    },
    {
      "epoch": 0.8166656016358873,
      "grad_norm": 10.960273995518746,
      "learning_rate": 4.9450700327090535e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -486.0,
      "logps/rejected": -804.0,
      "loss": 0.2013,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.515625,
      "rewards/rejected": -6.40625,
      "step": 6390
    },
    {
      "epoch": 0.8179436385711547,
      "grad_norm": 10.350168559385803,
      "learning_rate": 4.8786700838311184e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.484375,
      "logps/chosen": -464.0,
      "logps/rejected": -792.0,
      "loss": 0.2013,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.15625,
      "rewards/rejected": -6.15625,
      "step": 6400
    },
    {
      "epoch": 0.8192216755064221,
      "grad_norm": 13.424524307911831,
      "learning_rate": 4.812670712883396e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.578125,
      "logps/chosen": -452.0,
      "logps/rejected": -792.0,
      "loss": 0.2192,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.78125,
      "rewards/margins": 3.53125,
      "rewards/rejected": -6.3125,
      "step": 6410
    },
    {
      "epoch": 0.8204997124416895,
      "grad_norm": 10.351259462785487,
      "learning_rate": 4.7470732337895335e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5,
      "logps/chosen": -470.0,
      "logps/rejected": -812.0,
      "loss": 0.1919,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.28125,
      "step": 6420
    },
    {
      "epoch": 0.821777749376957,
      "grad_norm": 13.832202931520403,
      "learning_rate": 4.681878952472276e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -466.0,
      "logps/rejected": -820.0,
      "loss": 0.1959,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.40625,
      "step": 6430
    },
    {
      "epoch": 0.8230557863122244,
      "grad_norm": 10.40643567211064,
      "learning_rate": 4.6170891668274616e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -456.0,
      "logps/rejected": -820.0,
      "loss": 0.2071,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.921875,
      "rewards/rejected": -6.6875,
      "step": 6440
    },
    {
      "epoch": 0.8243338232474918,
      "grad_norm": 12.168721926808987,
      "learning_rate": 4.552705166698165e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.546875,
      "logps/chosen": -480.0,
      "logps/rejected": -776.0,
      "loss": 0.206,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.03125,
      "rewards/margins": 2.96875,
      "rewards/rejected": -6.0,
      "step": 6450
    },
    {
      "epoch": 0.8256118601827592,
      "grad_norm": 10.43565764601295,
      "learning_rate": 4.488728233849054e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -444.0,
      "logps/rejected": -764.0,
      "loss": 0.1983,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.796875,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.125,
      "step": 6460
    },
    {
      "epoch": 0.8268898971180267,
      "grad_norm": 8.634014679425178,
      "learning_rate": 4.425159641940826e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -472.0,
      "logps/rejected": -856.0,
      "loss": 0.2088,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.921875,
      "rewards/rejected": -6.6875,
      "step": 6470
    },
    {
      "epoch": 0.8281679340532941,
      "grad_norm": 15.504546345217397,
      "learning_rate": 4.362000656504902e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.671875,
      "logps/chosen": -488.0,
      "logps/rejected": -780.0,
      "loss": 0.2116,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.25,
      "step": 6480
    },
    {
      "epoch": 0.8294459709885615,
      "grad_norm": 8.499043944485441,
      "learning_rate": 4.299252534918185e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.34375,
      "logps/chosen": -468.0,
      "logps/rejected": -860.0,
      "loss": 0.1895,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.984375,
      "rewards/rejected": -6.8125,
      "step": 6490
    },
    {
      "epoch": 0.8307240079238289,
      "grad_norm": 10.15299124404224,
      "learning_rate": 4.2369165263780595e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.40625,
      "logps/chosen": -464.0,
      "logps/rejected": -860.0,
      "loss": 0.2073,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.890625,
      "rewards/rejected": -6.625,
      "step": 6500
    },
    {
      "epoch": 0.8320020448590965,
      "grad_norm": 15.733148758450673,
      "learning_rate": 4.174993871877513e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.40625,
      "logps/chosen": -492.0,
      "logps/rejected": -820.0,
      "loss": 0.203,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.53125,
      "step": 6510
    },
    {
      "epoch": 0.8332800817943639,
      "grad_norm": 9.515128201148956,
      "learning_rate": 4.1134858041804165e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.546875,
      "logps/chosen": -494.0,
      "logps/rejected": -824.0,
      "loss": 0.2116,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.515625,
      "rewards/rejected": -6.5,
      "step": 6520
    },
    {
      "epoch": 0.8345581187296313,
      "grad_norm": 12.409452932110856,
      "learning_rate": 4.05239354779702e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.390625,
      "logps/chosen": -494.0,
      "logps/rejected": -860.0,
      "loss": 0.2161,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.609375,
      "rewards/rejected": -6.6875,
      "step": 6530
    },
    {
      "epoch": 0.8358361556648987,
      "grad_norm": 10.309600465402935,
      "learning_rate": 3.991718318959528e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.53125,
      "logps/chosen": -470.0,
      "logps/rejected": -792.0,
      "loss": 0.2044,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.34375,
      "step": 6540
    },
    {
      "epoch": 0.8371141926001662,
      "grad_norm": 10.34229486120749,
      "learning_rate": 3.931461325597924e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.4375,
      "logps/chosen": -478.0,
      "logps/rejected": -816.0,
      "loss": 0.1961,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.40625,
      "step": 6550
    },
    {
      "epoch": 0.8383922295354336,
      "grad_norm": 13.800484703982123,
      "learning_rate": 3.871623767315896e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -456.0,
      "logps/rejected": -784.0,
      "loss": 0.2138,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.25,
      "step": 6560
    },
    {
      "epoch": 0.839670266470701,
      "grad_norm": 9.643114932610025,
      "learning_rate": 3.812206835366977e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.421875,
      "logps/chosen": -464.0,
      "logps/rejected": -808.0,
      "loss": 0.205,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.640625,
      "rewards/rejected": -6.5625,
      "step": 6570
    },
    {
      "epoch": 0.8409483034059684,
      "grad_norm": 10.112770961835434,
      "learning_rate": 3.753211712630808e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.4375,
      "logps/chosen": -492.0,
      "logps/rejected": -792.0,
      "loss": 0.2104,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.40625,
      "step": 6580
    },
    {
      "epoch": 0.8422263403412359,
      "grad_norm": 10.665790385749611,
      "learning_rate": 3.6946395735896e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.421875,
      "logps/chosen": -490.0,
      "logps/rejected": -808.0,
      "loss": 0.214,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.34375,
      "step": 6590
    },
    {
      "epoch": 0.8435043772765033,
      "grad_norm": 10.09050235580377,
      "learning_rate": 3.636491584304774e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.640625,
      "logps/chosen": -462.0,
      "logps/rejected": -800.0,
      "loss": 0.2083,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.3125,
      "step": 6600
    },
    {
      "epoch": 0.8447824142117707,
      "grad_norm": 13.701302274059447,
      "learning_rate": 3.578768902393697e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -488.0,
      "logps/rejected": -784.0,
      "loss": 0.1819,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.09375,
      "rewards/rejected": -6.125,
      "step": 6610
    },
    {
      "epoch": 0.8460604511470381,
      "grad_norm": 11.219402837869524,
      "learning_rate": 3.521472677006676e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5625,
      "logps/chosen": -452.0,
      "logps/rejected": -784.0,
      "loss": 0.1894,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.796875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.375,
      "step": 6620
    },
    {
      "epoch": 0.8473384880823056,
      "grad_norm": 7.644761200025466,
      "learning_rate": 3.464604048804079e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -470.0,
      "logps/rejected": -832.0,
      "loss": 0.2077,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.53125,
      "rewards/rejected": -6.40625,
      "step": 6630
    },
    {
      "epoch": 0.848616525017573,
      "grad_norm": 11.874319787348105,
      "learning_rate": 3.408164149933587e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -450.0,
      "logps/rejected": -764.0,
      "loss": 0.2113,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.71875,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.125,
      "step": 6640
    },
    {
      "epoch": 0.8498945619528404,
      "grad_norm": 10.602952813922508,
      "learning_rate": 3.352154104007721e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.453125,
      "logps/chosen": -452.0,
      "logps/rejected": -828.0,
      "loss": 0.1861,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.609375,
      "rewards/margins": 4.0,
      "rewards/rejected": -6.625,
      "step": 6650
    },
    {
      "epoch": 0.8511725988881078,
      "grad_norm": 15.342593100171912,
      "learning_rate": 3.296575026081408e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.484375,
      "logps/chosen": -484.0,
      "logps/rejected": -796.0,
      "loss": 0.2016,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.03125,
      "rewards/margins": 3.203125,
      "rewards/rejected": -6.21875,
      "step": 6660
    },
    {
      "epoch": 0.8524506358233753,
      "grad_norm": 11.126426752165013,
      "learning_rate": 3.241428022629822e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.640625,
      "logps/chosen": -488.0,
      "logps/rejected": -768.0,
      "loss": 0.2094,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.046875,
      "rewards/margins": 2.984375,
      "rewards/rejected": -6.03125,
      "step": 6670
    },
    {
      "epoch": 0.8537286727586427,
      "grad_norm": 12.244176812427481,
      "learning_rate": 3.186714191526346e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.40625,
      "logps/chosen": -466.0,
      "logps/rejected": -820.0,
      "loss": 0.1943,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.4375,
      "step": 6680
    },
    {
      "epoch": 0.8550067096939101,
      "grad_norm": 9.272060366139213,
      "learning_rate": 3.132434622020716e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.625,
      "logps/chosen": -476.0,
      "logps/rejected": -752.0,
      "loss": 0.1977,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.09375,
      "rewards/rejected": -6.0625,
      "step": 6690
    },
    {
      "epoch": 0.8562847466291775,
      "grad_norm": 12.230736070284227,
      "learning_rate": 3.078590394717323e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.609375,
      "logps/chosen": -490.0,
      "logps/rejected": -788.0,
      "loss": 0.1957,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.3125,
      "step": 6700
    },
    {
      "epoch": 0.857562783564445,
      "grad_norm": 11.166318422981206,
      "learning_rate": 3.0251825815537204e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.375,
      "logps/chosen": -472.0,
      "logps/rejected": -808.0,
      "loss": 0.2144,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.03125,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.53125,
      "step": 6710
    },
    {
      "epoch": 0.8588408204997124,
      "grad_norm": 12.289892128433427,
      "learning_rate": 2.972212245779271e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -476.0,
      "logps/rejected": -804.0,
      "loss": 0.2097,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.4375,
      "step": 6720
    },
    {
      "epoch": 0.8601188574349798,
      "grad_norm": 12.722197877423467,
      "learning_rate": 2.9196804419339917e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -478.0,
      "logps/rejected": -848.0,
      "loss": 0.2043,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.71875,
      "rewards/rejected": -6.6875,
      "step": 6730
    },
    {
      "epoch": 0.8613968943702472,
      "grad_norm": 10.490485472883789,
      "learning_rate": 2.867588215827538e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.515625,
      "logps/chosen": -472.0,
      "logps/rejected": -812.0,
      "loss": 0.2115,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.609375,
      "rewards/rejected": -6.53125,
      "step": 6740
    },
    {
      "epoch": 0.8626749313055148,
      "grad_norm": 7.342602167198535,
      "learning_rate": 2.8159366045184124e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -452.0,
      "logps/rejected": -740.0,
      "loss": 0.1886,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.078125,
      "rewards/rejected": -5.96875,
      "step": 6750
    },
    {
      "epoch": 0.8639529682407822,
      "grad_norm": 12.744715303120733,
      "learning_rate": 2.7647266362932936e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.578125,
      "logps/chosen": -486.0,
      "logps/rejected": -812.0,
      "loss": 0.1914,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.640625,
      "rewards/rejected": -6.53125,
      "step": 6760
    },
    {
      "epoch": 0.8652310051760496,
      "grad_norm": 13.022119937076885,
      "learning_rate": 2.7139593306465758e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.46875,
      "logps/chosen": -500.0,
      "logps/rejected": -840.0,
      "loss": 0.1854,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.65625,
      "step": 6770
    },
    {
      "epoch": 0.866509042111317,
      "grad_norm": 11.496134965790445,
      "learning_rate": 2.6636356982600695e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -496.0,
      "logps/rejected": -836.0,
      "loss": 0.1942,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.171875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.75,
      "step": 6780
    },
    {
      "epoch": 0.8677870790465845,
      "grad_norm": 12.473283761985849,
      "learning_rate": 2.6137567409828864e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.71875,
      "logps/chosen": -490.0,
      "logps/rejected": -816.0,
      "loss": 0.1907,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.640625,
      "rewards/rejected": -6.6875,
      "step": 6790
    },
    {
      "epoch": 0.8690651159818519,
      "grad_norm": 14.242378204832514,
      "learning_rate": 2.5643234518114955e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.609375,
      "logps/chosen": -476.0,
      "logps/rejected": -808.0,
      "loss": 0.2112,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.5,
      "step": 6800
    },
    {
      "epoch": 0.8703431529171193,
      "grad_norm": 15.059220961717756,
      "learning_rate": 2.515336814869942e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.515625,
      "logps/chosen": -504.0,
      "logps/rejected": -840.0,
      "loss": 0.1954,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.203125,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.6875,
      "step": 6810
    },
    {
      "epoch": 0.8716211898523867,
      "grad_norm": 12.419537760921644,
      "learning_rate": 2.466797805390272e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -464.0,
      "logps/rejected": -776.0,
      "loss": 0.207,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.296875,
      "rewards/rejected": -6.21875,
      "step": 6820
    },
    {
      "epoch": 0.8728992267876542,
      "grad_norm": 11.413329128813828,
      "learning_rate": 2.4187073896930977e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.375,
      "logps/chosen": -474.0,
      "logps/rejected": -836.0,
      "loss": 0.1952,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.90625,
      "rewards/margins": 3.734375,
      "rewards/rejected": -6.625,
      "step": 6830
    },
    {
      "epoch": 0.8741772637229216,
      "grad_norm": 15.213969216012465,
      "learning_rate": 2.3710665251683775e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.53125,
      "logps/chosen": -490.0,
      "logps/rejected": -812.0,
      "loss": 0.2149,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.375,
      "step": 6840
    },
    {
      "epoch": 0.875455300658189,
      "grad_norm": 13.746535754996433,
      "learning_rate": 2.3238761602563496e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.453125,
      "logps/chosen": -484.0,
      "logps/rejected": -848.0,
      "loss": 0.206,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -6.84375,
      "step": 6850
    },
    {
      "epoch": 0.8767333375934564,
      "grad_norm": 11.705378870473332,
      "learning_rate": 2.2771372344286417e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5625,
      "logps/chosen": -492.0,
      "logps/rejected": -796.0,
      "loss": 0.2112,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.1875,
      "rewards/rejected": -6.3125,
      "step": 6860
    },
    {
      "epoch": 0.8780113745287239,
      "grad_norm": 10.340498456221402,
      "learning_rate": 2.2308506781695906e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.46875,
      "logps/chosen": -472.0,
      "logps/rejected": -792.0,
      "loss": 0.2101,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.1875,
      "step": 6870
    },
    {
      "epoch": 0.8792894114639913,
      "grad_norm": 19.083643064399936,
      "learning_rate": 2.1850174129576933e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.40625,
      "logps/chosen": -478.0,
      "logps/rejected": -820.0,
      "loss": 0.2236,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.40625,
      "step": 6880
    },
    {
      "epoch": 0.8805674483992587,
      "grad_norm": 13.811112602319664,
      "learning_rate": 2.1396383512472722e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.484375,
      "logps/chosen": -462.0,
      "logps/rejected": -824.0,
      "loss": 0.1805,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.734375,
      "rewards/rejected": -6.53125,
      "step": 6890
    },
    {
      "epoch": 0.8818454853345261,
      "grad_norm": 9.329898798953813,
      "learning_rate": 2.09471439645032e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.390625,
      "logps/chosen": -472.0,
      "logps/rejected": -872.0,
      "loss": 0.2163,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.796875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -6.875,
      "step": 6900
    },
    {
      "epoch": 0.8831235222697936,
      "grad_norm": 16.98471210386864,
      "learning_rate": 2.0502464429184933e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.53125,
      "logps/chosen": -468.0,
      "logps/rejected": -824.0,
      "loss": 0.1892,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.71875,
      "rewards/margins": 3.640625,
      "rewards/rejected": -6.375,
      "step": 6910
    },
    {
      "epoch": 0.884401559205061,
      "grad_norm": 9.852568424344767,
      "learning_rate": 2.006235375925322e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.609375,
      "logps/chosen": -484.0,
      "logps/rejected": -784.0,
      "loss": 0.1987,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.09375,
      "rewards/margins": 3.234375,
      "rewards/rejected": -6.3125,
      "step": 6920
    },
    {
      "epoch": 0.8856795961403284,
      "grad_norm": 13.364583190301358,
      "learning_rate": 1.962682071648586e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.484375,
      "logps/chosen": -476.0,
      "logps/rejected": -796.0,
      "loss": 0.2172,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.375,
      "step": 6930
    },
    {
      "epoch": 0.8869576330755958,
      "grad_norm": 12.05198739793645,
      "learning_rate": 1.919587397152861e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.453125,
      "logps/chosen": -492.0,
      "logps/rejected": -832.0,
      "loss": 0.2094,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.5625,
      "step": 6940
    },
    {
      "epoch": 0.8882356700108633,
      "grad_norm": 11.952588653960753,
      "learning_rate": 1.8769522103722763e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.21875,
      "logps/chosen": -474.0,
      "logps/rejected": -852.0,
      "loss": 0.1887,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.71875,
      "rewards/rejected": -6.65625,
      "step": 6950
    },
    {
      "epoch": 0.8895137069461307,
      "grad_norm": 8.819959065534068,
      "learning_rate": 1.8347773600934063e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -450.0,
      "logps/rejected": -808.0,
      "loss": 0.2015,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.65625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -6.5,
      "step": 6960
    },
    {
      "epoch": 0.8907917438813981,
      "grad_norm": 10.85207188058072,
      "learning_rate": 1.7930636859384064e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -448.0,
      "logps/rejected": -784.0,
      "loss": 0.2099,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.78125,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.1875,
      "step": 6970
    },
    {
      "epoch": 0.8920697808166655,
      "grad_norm": 12.40218680107,
      "learning_rate": 1.7518120183482755e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5,
      "logps/chosen": -488.0,
      "logps/rejected": -804.0,
      "loss": 0.2183,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.46875,
      "step": 6980
    },
    {
      "epoch": 0.8933478177519331,
      "grad_norm": 16.0096135694293,
      "learning_rate": 1.7110231785663094e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.375,
      "logps/chosen": -464.0,
      "logps/rejected": -844.0,
      "loss": 0.2197,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -6.625,
      "step": 6990
    },
    {
      "epoch": 0.8946258546872005,
      "grad_norm": 13.369473920908677,
      "learning_rate": 1.670697978621802e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.578125,
      "logps/chosen": -464.0,
      "logps/rejected": -812.0,
      "loss": 0.1937,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.671875,
      "rewards/rejected": -6.46875,
      "step": 7000
    },
    {
      "epoch": 0.8959038916224679,
      "grad_norm": 10.012122742124536,
      "learning_rate": 1.6308372213138244e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -484.0,
      "logps/rejected": -836.0,
      "loss": 0.1916,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.671875,
      "rewards/rejected": -6.625,
      "step": 7010
    },
    {
      "epoch": 0.8971819285577353,
      "grad_norm": 13.665099251772212,
      "learning_rate": 1.5914417001952785e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.546875,
      "logps/chosen": -482.0,
      "logps/rejected": -796.0,
      "loss": 0.2184,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.296875,
      "rewards/rejected": -6.375,
      "step": 7020
    },
    {
      "epoch": 0.8984599654930028,
      "grad_norm": 13.051809389661265,
      "learning_rate": 1.5525121995570823e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.578125,
      "logps/chosen": -496.0,
      "logps/rejected": -820.0,
      "loss": 0.1909,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.3125,
      "step": 7030
    },
    {
      "epoch": 0.8997380024282702,
      "grad_norm": 10.911871250716105,
      "learning_rate": 1.5140494944125588e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.4375,
      "logps/chosen": -484.0,
      "logps/rejected": -832.0,
      "loss": 0.1693,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.625,
      "rewards/rejected": -6.65625,
      "step": 7040
    },
    {
      "epoch": 0.9010160393635376,
      "grad_norm": 11.095010631655091,
      "learning_rate": 1.4760543504820088e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.484375,
      "logps/chosen": -498.0,
      "logps/rejected": -844.0,
      "loss": 0.2004,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 3.734375,
      "rewards/rejected": -6.84375,
      "step": 7050
    },
    {
      "epoch": 0.902294076298805,
      "grad_norm": 10.668700804530113,
      "learning_rate": 1.438527524177463e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -480.0,
      "logps/rejected": -820.0,
      "loss": 0.2111,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.46875,
      "step": 7060
    },
    {
      "epoch": 0.9035721132340725,
      "grad_norm": 9.463609508727117,
      "learning_rate": 1.4014697625876426e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5,
      "logps/chosen": -498.0,
      "logps/rejected": -824.0,
      "loss": 0.2017,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.171875,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.5,
      "step": 7070
    },
    {
      "epoch": 0.9048501501693399,
      "grad_norm": 16.093026557759046,
      "learning_rate": 1.3648818034630499e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.6875,
      "logps/chosen": -470.0,
      "logps/rejected": -788.0,
      "loss": 0.2146,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.375,
      "rewards/rejected": -6.34375,
      "step": 7080
    },
    {
      "epoch": 0.9061281871046073,
      "grad_norm": 14.850758042507245,
      "learning_rate": 1.3287643752013162e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.46875,
      "logps/chosen": -476.0,
      "logps/rejected": -800.0,
      "loss": 0.2187,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.03125,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.375,
      "step": 7090
    },
    {
      "epoch": 0.9074062240398747,
      "grad_norm": 11.031339979196975,
      "learning_rate": 1.2931181968326765e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.390625,
      "logps/chosen": -466.0,
      "logps/rejected": -816.0,
      "loss": 0.213,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.90625,
      "rewards/margins": 3.53125,
      "rewards/rejected": -6.4375,
      "step": 7100
    },
    {
      "epoch": 0.9086842609751422,
      "grad_norm": 12.769870222012521,
      "learning_rate": 1.257943978005674e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5625,
      "logps/chosen": -478.0,
      "logps/rejected": -772.0,
      "loss": 0.2152,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.15625,
      "rewards/rejected": -6.1875,
      "step": 7110
    },
    {
      "epoch": 0.9099622979104096,
      "grad_norm": 14.443925724082929,
      "learning_rate": 1.2232424189730084e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.546875,
      "logps/chosen": -480.0,
      "logps/rejected": -800.0,
      "loss": 0.2007,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.34375,
      "step": 7120
    },
    {
      "epoch": 0.911240334845677,
      "grad_norm": 15.170489157436997,
      "learning_rate": 1.189014210577624e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.4375,
      "logps/chosen": -466.0,
      "logps/rejected": -760.0,
      "loss": 0.211,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.09375,
      "rewards/rejected": -5.96875,
      "step": 7130
    },
    {
      "epoch": 0.9125183717809444,
      "grad_norm": 12.148534381619779,
      "learning_rate": 1.1552600342389363e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -488.0,
      "logps/rejected": -872.0,
      "loss": 0.1954,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.8125,
      "rewards/rejected": -6.78125,
      "step": 7140
    },
    {
      "epoch": 0.9137964087162119,
      "grad_norm": 12.41217737005298,
      "learning_rate": 1.121980561939273e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.609375,
      "logps/chosen": -484.0,
      "logps/rejected": -856.0,
      "loss": 0.1771,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.75,
      "step": 7150
    },
    {
      "epoch": 0.9150744456514793,
      "grad_norm": 15.905782996604495,
      "learning_rate": 1.0891764562104888e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5,
      "logps/chosen": -472.0,
      "logps/rejected": -812.0,
      "loss": 0.2029,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.34375,
      "step": 7160
    },
    {
      "epoch": 0.9163524825867467,
      "grad_norm": 11.630225564746283,
      "learning_rate": 1.0568483701207941e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.359375,
      "logps/chosen": -482.0,
      "logps/rejected": -784.0,
      "loss": 0.1999,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.375,
      "step": 7170
    },
    {
      "epoch": 0.9176305195220141,
      "grad_norm": 9.767134603401278,
      "learning_rate": 1.0249969472617304e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.640625,
      "logps/chosen": -456.0,
      "logps/rejected": -772.0,
      "loss": 0.1975,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.21875,
      "step": 7180
    },
    {
      "epoch": 0.9189085564572816,
      "grad_norm": 12.981163338154785,
      "learning_rate": 9.936228217353709e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.609375,
      "logps/chosen": -466.0,
      "logps/rejected": -772.0,
      "loss": 0.1851,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.15625,
      "step": 7190
    },
    {
      "epoch": 0.920186593392549,
      "grad_norm": 15.39088207240477,
      "learning_rate": 9.627266181416987e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.375,
      "logps/chosen": -476.0,
      "logps/rejected": -824.0,
      "loss": 0.1965,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.53125,
      "step": 7200
    },
    {
      "epoch": 0.9214646303278164,
      "grad_norm": 11.617094819987472,
      "learning_rate": 9.323089515661603e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.671875,
      "logps/chosen": -482.0,
      "logps/rejected": -800.0,
      "loss": 0.1857,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.90625,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.5,
      "step": 7210
    },
    {
      "epoch": 0.9227426672630838,
      "grad_norm": 10.282897416051968,
      "learning_rate": 9.023704275674394e-09,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.640625,
      "logps/chosen": -502.0,
      "logps/rejected": -824.0,
      "loss": 0.2028,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.03125,
      "rewards/margins": 3.6875,
      "rewards/rejected": -6.71875,
      "step": 7220
    },
    {
      "epoch": 0.9240207041983514,
      "grad_norm": 9.012193689775774,
      "learning_rate": 8.729116421653748e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -486.0,
      "logps/rejected": -816.0,
      "loss": 0.1764,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.5625,
      "step": 7230
    },
    {
      "epoch": 0.9252987411336188,
      "grad_norm": 10.558538457420905,
      "learning_rate": 8.439331818291261e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.625,
      "logps/chosen": -504.0,
      "logps/rejected": -800.0,
      "loss": 0.2176,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.1875,
      "rewards/margins": 3.375,
      "rewards/rejected": -6.5625,
      "step": 7240
    },
    {
      "epoch": 0.9265767780688862,
      "grad_norm": 8.840147595085451,
      "learning_rate": 8.154356234654713e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.40625,
      "logps/chosen": -476.0,
      "logps/rejected": -816.0,
      "loss": 0.2137,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.5,
      "step": 7250
    },
    {
      "epoch": 0.9278548150041536,
      "grad_norm": 9.953433410132408,
      "learning_rate": 7.874195344073297e-09,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.28125,
      "logps/chosen": -502.0,
      "logps/rejected": -856.0,
      "loss": 0.1971,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.671875,
      "rewards/rejected": -6.78125,
      "step": 7260
    },
    {
      "epoch": 0.9291328519394211,
      "grad_norm": 12.388958289713694,
      "learning_rate": 7.598854724024828e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -490.0,
      "logps/rejected": -816.0,
      "loss": 0.1975,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.5625,
      "step": 7270
    },
    {
      "epoch": 0.9304108888746885,
      "grad_norm": 14.642671164145634,
      "learning_rate": 7.328339856024407e-09,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.453125,
      "logps/chosen": -466.0,
      "logps/rejected": -804.0,
      "loss": 0.1943,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.40625,
      "step": 7280
    },
    {
      "epoch": 0.9316889258099559,
      "grad_norm": 14.619037802079363,
      "learning_rate": 7.062656125515654e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.578125,
      "logps/chosen": -476.0,
      "logps/rejected": -844.0,
      "loss": 0.1931,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.890625,
      "rewards/rejected": -6.875,
      "step": 7290
    },
    {
      "epoch": 0.9329669627452233,
      "grad_norm": 11.119232076635404,
      "learning_rate": 6.801808821763178e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.4375,
      "logps/chosen": -472.0,
      "logps/rejected": -852.0,
      "loss": 0.2056,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.859375,
      "rewards/rejected": -6.6875,
      "step": 7300
    },
    {
      "epoch": 0.9342449996804908,
      "grad_norm": 13.451205861401375,
      "learning_rate": 6.545803137747385e-09,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.515625,
      "logps/chosen": -474.0,
      "logps/rejected": -812.0,
      "loss": 0.2125,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.53125,
      "rewards/rejected": -6.53125,
      "step": 7310
    },
    {
      "epoch": 0.9355230366157582,
      "grad_norm": 16.82746542679296,
      "learning_rate": 6.294644170061258e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -458.0,
      "logps/rejected": -784.0,
      "loss": 0.2086,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.390625,
      "rewards/rejected": -6.28125,
      "step": 7320
    },
    {
      "epoch": 0.9368010735510256,
      "grad_norm": 11.816342608944979,
      "learning_rate": 6.0483369188085995e-09,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -458.0,
      "logps/rejected": -792.0,
      "loss": 0.1879,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.375,
      "step": 7330
    },
    {
      "epoch": 0.938079110486293,
      "grad_norm": 10.967703145114669,
      "learning_rate": 5.806886287504725e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.546875,
      "logps/chosen": -480.0,
      "logps/rejected": -836.0,
      "loss": 0.1986,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.5,
      "step": 7340
    },
    {
      "epoch": 0.9393571474215605,
      "grad_norm": 8.840755931694586,
      "learning_rate": 5.570297082978709e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.46875,
      "logps/chosen": -458.0,
      "logps/rejected": -832.0,
      "loss": 0.1873,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.71875,
      "rewards/margins": 3.8125,
      "rewards/rejected": -6.53125,
      "step": 7350
    },
    {
      "epoch": 0.9406351843568279,
      "grad_norm": 13.08793386078248,
      "learning_rate": 5.3385740152777634e-09,
      "logits/chosen": -2.75,
      "logits/rejected": -2.46875,
      "logps/chosen": -502.0,
      "logps/rejected": -856.0,
      "loss": 0.2169,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.875,
      "rewards/rejected": -6.875,
      "step": 7360
    },
    {
      "epoch": 0.9419132212920953,
      "grad_norm": 10.708306375386769,
      "learning_rate": 5.111721697573457e-09,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5,
      "logps/chosen": -480.0,
      "logps/rejected": -828.0,
      "loss": 0.2071,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -6.625,
      "step": 7370
    },
    {
      "epoch": 0.9431912582273627,
      "grad_norm": 8.101157614211795,
      "learning_rate": 4.889744646069837e-09,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.53125,
      "logps/chosen": -466.0,
      "logps/rejected": -816.0,
      "loss": 0.1953,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.5,
      "step": 7380
    },
    {
      "epoch": 0.9444692951626302,
      "grad_norm": 11.621422966760335,
      "learning_rate": 4.672647279913622e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -446.0,
      "logps/rejected": -816.0,
      "loss": 0.2188,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.6875,
      "rewards/rejected": -6.4375,
      "step": 7390
    },
    {
      "epoch": 0.9457473320978976,
      "grad_norm": 14.71774887048462,
      "learning_rate": 4.4604339211059824e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.390625,
      "logps/chosen": -490.0,
      "logps/rejected": -844.0,
      "loss": 0.2026,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.625,
      "step": 7400
    },
    {
      "epoch": 0.947025369033165,
      "grad_norm": 10.35523724247659,
      "learning_rate": 4.253108794416871e-09,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.4375,
      "logps/chosen": -472.0,
      "logps/rejected": -860.0,
      "loss": 0.1903,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.84375,
      "rewards/margins": 4.0,
      "rewards/rejected": -6.84375,
      "step": 7410
    },
    {
      "epoch": 0.9483034059684324,
      "grad_norm": 11.989463972076447,
      "learning_rate": 4.050676027300554e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.421875,
      "logps/chosen": -496.0,
      "logps/rejected": -804.0,
      "loss": 0.2137,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.1875,
      "rewards/rejected": -6.34375,
      "step": 7420
    },
    {
      "epoch": 0.9495814429037,
      "grad_norm": 12.437967073440545,
      "learning_rate": 3.853139649813625e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.078125,
      "logps/chosen": -488.0,
      "logps/rejected": -848.0,
      "loss": 0.2095,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.65625,
      "step": 7430
    },
    {
      "epoch": 0.9508594798389673,
      "grad_norm": 10.53127270152583,
      "learning_rate": 3.660503594534875e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -488.0,
      "logps/rejected": -856.0,
      "loss": 0.2002,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.515625,
      "rewards/rejected": -6.59375,
      "step": 7440
    },
    {
      "epoch": 0.9521375167742347,
      "grad_norm": 15.457225124049865,
      "learning_rate": 3.4727716964866595e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.609375,
      "logps/chosen": -476.0,
      "logps/rejected": -804.0,
      "loss": 0.2058,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.25,
      "step": 7450
    },
    {
      "epoch": 0.9534155537095022,
      "grad_norm": 23.12314332321969,
      "learning_rate": 3.2899476930589863e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.625,
      "logps/chosen": -498.0,
      "logps/rejected": -812.0,
      "loss": 0.2147,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.171875,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.53125,
      "step": 7460
    },
    {
      "epoch": 0.9546935906447697,
      "grad_norm": 13.065717581456365,
      "learning_rate": 3.112035223934689e-09,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5625,
      "logps/chosen": -450.0,
      "logps/rejected": -776.0,
      "loss": 0.2065,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.375,
      "rewards/rejected": -6.25,
      "step": 7470
    },
    {
      "epoch": 0.9559716275800371,
      "grad_norm": 9.92567072973857,
      "learning_rate": 2.939037831017288e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -476.0,
      "logps/rejected": -820.0,
      "loss": 0.1831,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.53125,
      "step": 7480
    },
    {
      "epoch": 0.9572496645153045,
      "grad_norm": 10.87875481014558,
      "learning_rate": 2.7709589583603254e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.546875,
      "logps/chosen": -478.0,
      "logps/rejected": -832.0,
      "loss": 0.2037,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.71875,
      "rewards/rejected": -6.6875,
      "step": 7490
    },
    {
      "epoch": 0.9585277014505719,
      "grad_norm": 20.59507724136227,
      "learning_rate": 2.6078019520988093e-09,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.75,
      "logps/chosen": -492.0,
      "logps/rejected": -808.0,
      "loss": 0.2003,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.53125,
      "step": 7500
    },
    {
      "epoch": 0.9598057383858394,
      "grad_norm": 20.569427553613547,
      "learning_rate": 2.449570060382711e-09,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.53125,
      "logps/chosen": -504.0,
      "logps/rejected": -844.0,
      "loss": 0.2245,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.78125,
      "rewards/rejected": -6.84375,
      "step": 7510
    },
    {
      "epoch": 0.9610837753211068,
      "grad_norm": 13.919589633618875,
      "learning_rate": 2.2962664333120994e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.390625,
      "logps/chosen": -470.0,
      "logps/rejected": -820.0,
      "loss": 0.1918,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.5625,
      "step": 7520
    },
    {
      "epoch": 0.9623618122563742,
      "grad_norm": 10.686523232998923,
      "learning_rate": 2.1478941228745828e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.671875,
      "logps/chosen": -484.0,
      "logps/rejected": -852.0,
      "loss": 0.2157,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.03125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.75,
      "step": 7530
    },
    {
      "epoch": 0.9636398491916416,
      "grad_norm": 9.537007805201354,
      "learning_rate": 2.0044560828845205e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -474.0,
      "logps/rejected": -876.0,
      "loss": 0.1825,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.796875,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.0,
      "step": 7540
    },
    {
      "epoch": 0.9649178861269091,
      "grad_norm": 11.06624234367653,
      "learning_rate": 1.8659551689241547e-09,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.484375,
      "logps/chosen": -484.0,
      "logps/rejected": -844.0,
      "loss": 0.2072,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.71875,
      "step": 7550
    },
    {
      "epoch": 0.9661959230621765,
      "grad_norm": 12.621095098059318,
      "learning_rate": 1.7323941382869078e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -516.0,
      "logps/rejected": -848.0,
      "loss": 0.1842,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.765625,
      "rewards/rejected": -6.90625,
      "step": 7560
    },
    {
      "epoch": 0.9674739599974439,
      "grad_norm": 15.28972535724583,
      "learning_rate": 1.603775649922312e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -472.0,
      "logps/rejected": -844.0,
      "loss": 0.1729,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.71875,
      "rewards/margins": 4.03125,
      "rewards/rejected": -6.78125,
      "step": 7570
    },
    {
      "epoch": 0.9687519969327113,
      "grad_norm": 11.721221933655565,
      "learning_rate": 1.4801022643831652e-09,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.421875,
      "logps/chosen": -508.0,
      "logps/rejected": -832.0,
      "loss": 0.2046,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.65625,
      "step": 7580
    },
    {
      "epoch": 0.9700300338679788,
      "grad_norm": 15.622764294517808,
      "learning_rate": 1.3613764437745978e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.484375,
      "logps/chosen": -472.0,
      "logps/rejected": -816.0,
      "loss": 0.1964,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.796875,
      "rewards/margins": 3.625,
      "rewards/rejected": -6.4375,
      "step": 7590
    },
    {
      "epoch": 0.9713080708032462,
      "grad_norm": 14.702285195178746,
      "learning_rate": 1.2476005517049192e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.484375,
      "logps/chosen": -484.0,
      "logps/rejected": -804.0,
      "loss": 0.1944,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.375,
      "step": 7600
    },
    {
      "epoch": 0.9725861077385136,
      "grad_norm": 12.557674078607901,
      "learning_rate": 1.138776853238682e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -498.0,
      "logps/rejected": -828.0,
      "loss": 0.2004,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.4375,
      "step": 7610
    },
    {
      "epoch": 0.973864144673781,
      "grad_norm": 12.983539021096119,
      "learning_rate": 1.0349075148516074e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -474.0,
      "logps/rejected": -812.0,
      "loss": 0.1827,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.90625,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.46875,
      "step": 7620
    },
    {
      "epoch": 0.9751421816090485,
      "grad_norm": 12.952226107004218,
      "learning_rate": 9.359946043873412e-10,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.546875,
      "logps/chosen": -496.0,
      "logps/rejected": -780.0,
      "loss": 0.2328,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.375,
      "rewards/rejected": -6.25,
      "step": 7630
    },
    {
      "epoch": 0.9764202185443159,
      "grad_norm": 15.055889199981205,
      "learning_rate": 8.420400910163494e-10,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.453125,
      "logps/chosen": -474.0,
      "logps/rejected": -808.0,
      "loss": 0.2043,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.28125,
      "step": 7640
    },
    {
      "epoch": 0.9776982554795833,
      "grad_norm": 15.964483040399076,
      "learning_rate": 7.530458451967814e-10,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.609375,
      "logps/chosen": -484.0,
      "logps/rejected": -836.0,
      "loss": 0.196,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.71875,
      "step": 7650
    },
    {
      "epoch": 0.9789762924148507,
      "grad_norm": 13.4124091634327,
      "learning_rate": 6.69013638637056e-10,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5625,
      "logps/chosen": -476.0,
      "logps/rejected": -816.0,
      "loss": 0.2015,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.640625,
      "rewards/rejected": -6.5,
      "step": 7660
    },
    {
      "epoch": 0.9802543293501182,
      "grad_norm": 15.040163473099657,
      "learning_rate": 5.899451442607784e-10,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.546875,
      "logps/chosen": -472.0,
      "logps/rejected": -840.0,
      "loss": 0.2022,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -6.625,
      "step": 7670
    },
    {
      "epoch": 0.9815323662853856,
      "grad_norm": 14.03901677519711,
      "learning_rate": 5.158419361733501e-10,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.484375,
      "logps/chosen": -504.0,
      "logps/rejected": -840.0,
      "loss": 0.2135,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.625,
      "step": 7680
    },
    {
      "epoch": 0.982810403220653,
      "grad_norm": 13.478797117751379,
      "learning_rate": 4.4670548963063327e-10,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -494.0,
      "logps/rejected": -808.0,
      "loss": 0.1944,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.40625,
      "step": 7690
    },
    {
      "epoch": 0.9840884401559205,
      "grad_norm": 14.47647348967129,
      "learning_rate": 3.82537181009529e-10,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.359375,
      "logps/chosen": -492.0,
      "logps/rejected": -876.0,
      "loss": 0.1878,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.96875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.125,
      "step": 7700
    },
    {
      "epoch": 0.985366477091188,
      "grad_norm": 17.746106540318237,
      "learning_rate": 3.233382877806945e-10,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -506.0,
      "logps/rejected": -840.0,
      "loss": 0.2122,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.65625,
      "step": 7710
    },
    {
      "epoch": 0.9866445140264554,
      "grad_norm": 11.289242528376295,
      "learning_rate": 2.6910998848306276e-10,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.671875,
      "logps/chosen": -490.0,
      "logps/rejected": -808.0,
      "loss": 0.1931,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.46875,
      "step": 7720
    },
    {
      "epoch": 0.9879225509617228,
      "grad_norm": 11.57920485888182,
      "learning_rate": 2.1985336270033405e-10,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -470.0,
      "logps/rejected": -816.0,
      "loss": 0.2025,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.609375,
      "rewards/rejected": -6.53125,
      "step": 7730
    },
    {
      "epoch": 0.9892005878969902,
      "grad_norm": 13.750778225464039,
      "learning_rate": 1.7556939103952063e-10,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.59375,
      "logps/chosen": -472.0,
      "logps/rejected": -768.0,
      "loss": 0.2053,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.203125,
      "rewards/rejected": -6.125,
      "step": 7740
    },
    {
      "epoch": 0.9904786248322577,
      "grad_norm": 10.780157788078466,
      "learning_rate": 1.3625895511137907e-10,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5625,
      "logps/chosen": -488.0,
      "logps/rejected": -820.0,
      "loss": 0.194,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -6.625,
      "step": 7750
    },
    {
      "epoch": 0.9917566617675251,
      "grad_norm": 15.518327038294338,
      "learning_rate": 1.0192283751297992e-10,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.515625,
      "logps/chosen": -452.0,
      "logps/rejected": -816.0,
      "loss": 0.1904,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.78125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.5,
      "step": 7760
    },
    {
      "epoch": 0.9930346987027925,
      "grad_norm": 12.419551999740973,
      "learning_rate": 7.256172181199783e-11,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -500.0,
      "logps/rejected": -824.0,
      "loss": 0.2081,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.625,
      "step": 7770
    },
    {
      "epoch": 0.9943127356380599,
      "grad_norm": 9.554550445127738,
      "learning_rate": 4.8176192533139206e-11,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.609375,
      "logps/chosen": -454.0,
      "logps/rejected": -804.0,
      "loss": 0.1964,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.796875,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.25,
      "step": 7780
    },
    {
      "epoch": 0.9955907725733274,
      "grad_norm": 11.093488623373322,
      "learning_rate": 2.876673514659589e-11,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.578125,
      "logps/chosen": -464.0,
      "logps/rejected": -792.0,
      "loss": 0.1859,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.53125,
      "rewards/rejected": -6.46875,
      "step": 7790
    },
    {
      "epoch": 0.9968688095085948,
      "grad_norm": 12.431609300555639,
      "learning_rate": 1.4333736058219636e-11,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.453125,
      "logps/chosen": -476.0,
      "logps/rejected": -804.0,
      "loss": 0.1917,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.3125,
      "step": 7800
    },
    {
      "epoch": 0.9981468464438622,
      "grad_norm": 19.033286012196964,
      "learning_rate": 4.8774826019448715e-12,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.421875,
      "logps/chosen": -492.0,
      "logps/rejected": -896.0,
      "loss": 0.1934,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -6.9375,
      "step": 7810
    },
    {
      "epoch": 0.9994248833791296,
      "grad_norm": 11.551976380866146,
      "learning_rate": 3.981630340710307e-13,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5,
      "logps/chosen": -466.0,
      "logps/rejected": -832.0,
      "loss": 0.1951,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.875,
      "rewards/rejected": -6.8125,
      "step": 7820
    },
    {
      "epoch": 0.9999360981532366,
      "step": 7824,
      "total_flos": 0.0,
      "train_loss": 0.0,
      "train_runtime": 4.129,
      "train_samples_per_second": 242555.672,
      "train_steps_per_second": 1894.883
    }
  ],
  "logging_steps": 10,
  "max_steps": 7824,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 10000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}