{
  "best_metric": 0.2485724240541458,
  "best_model_checkpoint": "models/llama3.2-3b-dpo-finegrained/checkpoint-10000",
  "epoch": 1.0,
  "eval_steps": 10000,
  "global_step": 13853,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 7.218652999350322e-05,
      "grad_norm": 2.7972422696966683,
      "learning_rate": 3.6075036075036073e-10,
      "logits/chosen": -0.875,
      "logits/rejected": -0.11474609375,
      "logps/chosen": -174.0,
      "logps/rejected": -188.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0007218652999350321,
      "grad_norm": 2.9446752294893495,
      "learning_rate": 3.6075036075036073e-09,
      "logits/chosen": -0.75,
      "logits/rejected": -0.328125,
      "logps/chosen": -184.0,
      "logps/rejected": -169.0,
      "loss": 0.6927,
      "rewards/accuracies": 0.2708333432674408,
      "rewards/chosen": 0.0002079010009765625,
      "rewards/margins": 0.0003833770751953125,
      "rewards/rejected": -0.00017261505126953125,
      "step": 10
    },
    {
      "epoch": 0.0014437305998700643,
      "grad_norm": 2.6581245576346353,
      "learning_rate": 7.215007215007215e-09,
      "logits/chosen": -0.87109375,
      "logits/rejected": -0.423828125,
      "logps/chosen": -184.0,
      "logps/rejected": -160.0,
      "loss": 0.6927,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": 0.000453948974609375,
      "rewards/margins": 0.0004367828369140625,
      "rewards/rejected": 1.4126300811767578e-05,
      "step": 20
    },
    {
      "epoch": 0.0021655958998050965,
      "grad_norm": 2.278271892835424,
      "learning_rate": 1.0822510822510822e-08,
      "logits/chosen": -0.84375,
      "logits/rejected": -0.400390625,
      "logps/chosen": -170.0,
      "logps/rejected": -170.0,
      "loss": 0.6925,
      "rewards/accuracies": 0.2874999940395355,
      "rewards/chosen": -0.000453948974609375,
      "rewards/margins": -0.000453948974609375,
      "rewards/rejected": -3.818422555923462e-07,
      "step": 30
    },
    {
      "epoch": 0.0028874611997401285,
      "grad_norm": 2.2866422383461305,
      "learning_rate": 1.443001443001443e-08,
      "logits/chosen": -0.87109375,
      "logits/rejected": -0.36328125,
      "logps/chosen": -191.0,
      "logps/rejected": -185.0,
      "loss": 0.6927,
      "rewards/accuracies": 0.29374998807907104,
      "rewards/chosen": 0.000530242919921875,
      "rewards/margins": 0.000514984130859375,
      "rewards/rejected": 1.6450881958007812e-05,
      "step": 40
    },
    {
      "epoch": 0.0036093264996751606,
      "grad_norm": 2.7282250029379633,
      "learning_rate": 1.8037518037518035e-08,
      "logits/chosen": -0.75,
      "logits/rejected": -0.380859375,
      "logps/chosen": -192.0,
      "logps/rejected": -166.0,
      "loss": 0.6927,
      "rewards/accuracies": 0.29374998807907104,
      "rewards/chosen": 0.0002040863037109375,
      "rewards/margins": 0.00090789794921875,
      "rewards/rejected": -0.000705718994140625,
      "step": 50
    },
    {
      "epoch": 0.004331191799610193,
      "grad_norm": 3.156770979366867,
      "learning_rate": 2.1645021645021644e-08,
      "logits/chosen": -0.765625,
      "logits/rejected": -0.33984375,
      "logps/chosen": -190.0,
      "logps/rejected": -159.0,
      "loss": 0.6927,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.000484466552734375,
      "rewards/margins": 0.00032806396484375,
      "rewards/rejected": -0.000812530517578125,
      "step": 60
    },
    {
      "epoch": 0.005053057099545225,
      "grad_norm": 2.3616471547138413,
      "learning_rate": 2.525252525252525e-08,
      "logits/chosen": -0.8671875,
      "logits/rejected": -0.263671875,
      "logps/chosen": -177.0,
      "logps/rejected": -180.0,
      "loss": 0.6928,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.00080108642578125,
      "rewards/margins": -0.0003757476806640625,
      "rewards/rejected": 0.0011749267578125,
      "step": 70
    },
    {
      "epoch": 0.005774922399480257,
      "grad_norm": 2.8865605298560837,
      "learning_rate": 2.886002886002886e-08,
      "logits/chosen": -0.83203125,
      "logits/rejected": -0.50390625,
      "logps/chosen": -197.0,
      "logps/rejected": -170.0,
      "loss": 0.6922,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": 0.001220703125,
      "rewards/margins": 0.000545501708984375,
      "rewards/rejected": 0.00067138671875,
      "step": 80
    },
    {
      "epoch": 0.006496787699415289,
      "grad_norm": 2.726867206016334,
      "learning_rate": 3.246753246753246e-08,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.3828125,
      "logps/chosen": -202.0,
      "logps/rejected": -182.0,
      "loss": 0.6926,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": 0.00165557861328125,
      "rewards/margins": 3.170967102050781e-05,
      "rewards/rejected": 0.0016326904296875,
      "step": 90
    },
    {
      "epoch": 0.007218652999350321,
      "grad_norm": 2.9974544177977775,
      "learning_rate": 3.607503607503607e-08,
      "logits/chosen": -0.87109375,
      "logits/rejected": -0.353515625,
      "logps/chosen": -200.0,
      "logps/rejected": -189.0,
      "loss": 0.692,
      "rewards/accuracies": 0.33125001192092896,
      "rewards/chosen": 0.002166748046875,
      "rewards/margins": 0.0003299713134765625,
      "rewards/rejected": 0.0018463134765625,
      "step": 100
    },
    {
      "epoch": 0.007940518299285354,
      "grad_norm": 2.7870828702044657,
      "learning_rate": 3.968253968253968e-08,
      "logits/chosen": -0.9296875,
      "logits/rejected": -0.40625,
      "logps/chosen": -181.0,
      "logps/rejected": -162.0,
      "loss": 0.6922,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.003936767578125,
      "rewards/margins": 0.0016326904296875,
      "rewards/rejected": 0.0023193359375,
      "step": 110
    },
    {
      "epoch": 0.008662383599220386,
      "grad_norm": 2.6778776361638044,
      "learning_rate": 4.329004329004329e-08,
      "logits/chosen": -0.921875,
      "logits/rejected": -0.439453125,
      "logps/chosen": -170.0,
      "logps/rejected": -162.0,
      "loss": 0.6918,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": 0.007110595703125,
      "rewards/margins": 0.00157928466796875,
      "rewards/rejected": 0.005523681640625,
      "step": 120
    },
    {
      "epoch": 0.009384248899155418,
      "grad_norm": 2.8717602343710884,
      "learning_rate": 4.68975468975469e-08,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.609375,
      "logps/chosen": -180.0,
      "logps/rejected": -175.0,
      "loss": 0.6918,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.0093994140625,
      "rewards/margins": 0.001678466796875,
      "rewards/rejected": 0.0076904296875,
      "step": 130
    },
    {
      "epoch": 0.01010611419909045,
      "grad_norm": 2.6195835919443202,
      "learning_rate": 5.05050505050505e-08,
      "logits/chosen": -0.8671875,
      "logits/rejected": -0.54296875,
      "logps/chosen": -194.0,
      "logps/rejected": -174.0,
      "loss": 0.6915,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.01214599609375,
      "rewards/margins": 0.002044677734375,
      "rewards/rejected": 0.0101318359375,
      "step": 140
    },
    {
      "epoch": 0.010827979499025482,
      "grad_norm": 3.2957818120117834,
      "learning_rate": 5.411255411255411e-08,
      "logits/chosen": -0.89453125,
      "logits/rejected": -0.46484375,
      "logps/chosen": -191.0,
      "logps/rejected": -171.0,
      "loss": 0.6913,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0137939453125,
      "rewards/margins": 0.00045013427734375,
      "rewards/rejected": 0.01336669921875,
      "step": 150
    },
    {
      "epoch": 0.011549844798960514,
      "grad_norm": 2.8352357636644334,
      "learning_rate": 5.772005772005772e-08,
      "logits/chosen": -0.8515625,
      "logits/rejected": -0.314453125,
      "logps/chosen": -191.0,
      "logps/rejected": -183.0,
      "loss": 0.6908,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.020751953125,
      "rewards/margins": 0.00494384765625,
      "rewards/rejected": 0.0157470703125,
      "step": 160
    },
    {
      "epoch": 0.012271710098895546,
      "grad_norm": 2.6093303355604625,
      "learning_rate": 6.132756132756133e-08,
      "logits/chosen": -0.87109375,
      "logits/rejected": -0.41015625,
      "logps/chosen": -189.0,
      "logps/rejected": -166.0,
      "loss": 0.6895,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.02978515625,
      "rewards/margins": 0.004913330078125,
      "rewards/rejected": 0.0247802734375,
      "step": 170
    },
    {
      "epoch": 0.012993575398830578,
      "grad_norm": 3.0165318943424926,
      "learning_rate": 6.493506493506492e-08,
      "logits/chosen": -0.86328125,
      "logits/rejected": -0.404296875,
      "logps/chosen": -210.0,
      "logps/rejected": -187.0,
      "loss": 0.69,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 0.037353515625,
      "rewards/margins": 0.00518798828125,
      "rewards/rejected": 0.031982421875,
      "step": 180
    },
    {
      "epoch": 0.01371544069876561,
      "grad_norm": 2.3952812681349065,
      "learning_rate": 6.854256854256854e-08,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.482421875,
      "logps/chosen": -186.0,
      "logps/rejected": -166.0,
      "loss": 0.6882,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.046142578125,
      "rewards/margins": 0.00946044921875,
      "rewards/rejected": 0.03662109375,
      "step": 190
    },
    {
      "epoch": 0.014437305998700642,
      "grad_norm": 2.7145840286863563,
      "learning_rate": 7.215007215007214e-08,
      "logits/chosen": -0.83203125,
      "logits/rejected": -0.390625,
      "logps/chosen": -174.0,
      "logps/rejected": -156.0,
      "loss": 0.6878,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.056640625,
      "rewards/margins": 0.01556396484375,
      "rewards/rejected": 0.041259765625,
      "step": 200
    },
    {
      "epoch": 0.015159171298635674,
      "grad_norm": 2.3227422274745844,
      "learning_rate": 7.575757575757576e-08,
      "logits/chosen": -0.88671875,
      "logits/rejected": -0.4140625,
      "logps/chosen": -175.0,
      "logps/rejected": -171.0,
      "loss": 0.6871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06494140625,
      "rewards/margins": 0.015625,
      "rewards/rejected": 0.04931640625,
      "step": 210
    },
    {
      "epoch": 0.015881036598570708,
      "grad_norm": 2.849169006278049,
      "learning_rate": 7.936507936507936e-08,
      "logits/chosen": -1.0546875,
      "logits/rejected": -0.39453125,
      "logps/chosen": -175.0,
      "logps/rejected": -184.0,
      "loss": 0.6845,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.08251953125,
      "rewards/margins": 0.0203857421875,
      "rewards/rejected": 0.062255859375,
      "step": 220
    },
    {
      "epoch": 0.01660290189850574,
      "grad_norm": 2.947016256566627,
      "learning_rate": 8.297258297258297e-08,
      "logits/chosen": -0.8671875,
      "logits/rejected": -0.4609375,
      "logps/chosen": -190.0,
      "logps/rejected": -166.0,
      "loss": 0.6812,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.107421875,
      "rewards/margins": 0.031005859375,
      "rewards/rejected": 0.076171875,
      "step": 230
    },
    {
      "epoch": 0.017324767198440772,
      "grad_norm": 2.6229394014847665,
      "learning_rate": 8.658008658008658e-08,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.69140625,
      "logps/chosen": -185.0,
      "logps/rejected": -166.0,
      "loss": 0.6809,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.1181640625,
      "rewards/margins": 0.025634765625,
      "rewards/rejected": 0.09228515625,
      "step": 240
    },
    {
      "epoch": 0.018046632498375802,
      "grad_norm": 2.7720093905367946,
      "learning_rate": 9.018759018759018e-08,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.625,
      "logps/chosen": -214.0,
      "logps/rejected": -204.0,
      "loss": 0.679,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.130859375,
      "rewards/margins": 0.025146484375,
      "rewards/rejected": 0.10546875,
      "step": 250
    },
    {
      "epoch": 0.018768497798310836,
      "grad_norm": 2.4938674483064136,
      "learning_rate": 9.37950937950938e-08,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.53515625,
      "logps/chosen": -165.0,
      "logps/rejected": -180.0,
      "loss": 0.6757,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.1416015625,
      "rewards/margins": 0.038330078125,
      "rewards/rejected": 0.103515625,
      "step": 260
    },
    {
      "epoch": 0.019490363098245866,
      "grad_norm": 6.662639325853069,
      "learning_rate": 9.74025974025974e-08,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.67578125,
      "logps/chosen": -187.0,
      "logps/rejected": -163.0,
      "loss": 0.6756,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.134765625,
      "rewards/margins": 0.034912109375,
      "rewards/rejected": 0.099609375,
      "step": 270
    },
    {
      "epoch": 0.0202122283981809,
      "grad_norm": 2.3316036607598156,
      "learning_rate": 1.01010101010101e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.443359375,
      "logps/chosen": -181.0,
      "logps/rejected": -184.0,
      "loss": 0.6759,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.1435546875,
      "rewards/margins": 0.039306640625,
      "rewards/rejected": 0.103515625,
      "step": 280
    },
    {
      "epoch": 0.02093409369811593,
      "grad_norm": 2.2722773009755777,
      "learning_rate": 1.0461760461760462e-07,
      "logits/chosen": -0.97265625,
      "logits/rejected": -0.58203125,
      "logps/chosen": -185.0,
      "logps/rejected": -165.0,
      "loss": 0.6731,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.146484375,
      "rewards/margins": 0.036865234375,
      "rewards/rejected": 0.1103515625,
      "step": 290
    },
    {
      "epoch": 0.021655958998050964,
      "grad_norm": 2.245750276448923,
      "learning_rate": 1.0822510822510822e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.64453125,
      "logps/chosen": -179.0,
      "logps/rejected": -182.0,
      "loss": 0.6751,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.142578125,
      "rewards/margins": 0.0296630859375,
      "rewards/rejected": 0.11328125,
      "step": 300
    },
    {
      "epoch": 0.022377824297985995,
      "grad_norm": 2.6175391235465084,
      "learning_rate": 1.1183261183261184e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.625,
      "logps/chosen": -163.0,
      "logps/rejected": -166.0,
      "loss": 0.6712,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.1494140625,
      "rewards/margins": 0.049560546875,
      "rewards/rejected": 0.10009765625,
      "step": 310
    },
    {
      "epoch": 0.02309968959792103,
      "grad_norm": 2.4066627744727973,
      "learning_rate": 1.1544011544011543e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.625,
      "logps/chosen": -175.0,
      "logps/rejected": -156.0,
      "loss": 0.6704,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.142578125,
      "rewards/margins": 0.04541015625,
      "rewards/rejected": 0.09716796875,
      "step": 320
    },
    {
      "epoch": 0.02382155489785606,
      "grad_norm": 2.3513485845799154,
      "learning_rate": 1.1904761904761903e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.439453125,
      "logps/chosen": -172.0,
      "logps/rejected": -162.0,
      "loss": 0.6659,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.1484375,
      "rewards/margins": 0.061767578125,
      "rewards/rejected": 0.0869140625,
      "step": 330
    },
    {
      "epoch": 0.024543420197791092,
      "grad_norm": 2.2580067158282757,
      "learning_rate": 1.2265512265512265e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -0.66015625,
      "logps/chosen": -179.0,
      "logps/rejected": -186.0,
      "loss": 0.6675,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.1513671875,
      "rewards/margins": 0.060546875,
      "rewards/rejected": 0.09130859375,
      "step": 340
    },
    {
      "epoch": 0.025265285497726123,
      "grad_norm": 2.616878081322348,
      "learning_rate": 1.2626262626262626e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.59765625,
      "logps/chosen": -187.0,
      "logps/rejected": -154.0,
      "loss": 0.6637,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.134765625,
      "rewards/margins": 0.0634765625,
      "rewards/rejected": 0.0712890625,
      "step": 350
    },
    {
      "epoch": 0.025987150797661156,
      "grad_norm": 2.409060432415145,
      "learning_rate": 1.2987012987012984e-07,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.625,
      "logps/chosen": -188.0,
      "logps/rejected": -175.0,
      "loss": 0.666,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.1240234375,
      "rewards/margins": 0.056640625,
      "rewards/rejected": 0.0673828125,
      "step": 360
    },
    {
      "epoch": 0.02670901609759619,
      "grad_norm": 2.4501831861080823,
      "learning_rate": 1.3347763347763348e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.6796875,
      "logps/chosen": -162.0,
      "logps/rejected": -163.0,
      "loss": 0.661,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.12890625,
      "rewards/margins": 0.07373046875,
      "rewards/rejected": 0.0546875,
      "step": 370
    },
    {
      "epoch": 0.02743088139753122,
      "grad_norm": 2.212553579467342,
      "learning_rate": 1.370851370851371e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.6953125,
      "logps/chosen": -183.0,
      "logps/rejected": -170.0,
      "loss": 0.6599,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.10791015625,
      "rewards/margins": 0.0693359375,
      "rewards/rejected": 0.03857421875,
      "step": 380
    },
    {
      "epoch": 0.028152746697466254,
      "grad_norm": 2.399967361936523,
      "learning_rate": 1.406926406926407e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.625,
      "logps/chosen": -170.0,
      "logps/rejected": -161.0,
      "loss": 0.6578,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11767578125,
      "rewards/margins": 0.078125,
      "rewards/rejected": 0.0400390625,
      "step": 390
    },
    {
      "epoch": 0.028874611997401285,
      "grad_norm": 2.6706978712815492,
      "learning_rate": 1.4430014430014428e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.60546875,
      "logps/chosen": -187.0,
      "logps/rejected": -183.0,
      "loss": 0.6545,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.08837890625,
      "rewards/margins": 0.08544921875,
      "rewards/rejected": 0.002655029296875,
      "step": 400
    },
    {
      "epoch": 0.02959647729733632,
      "grad_norm": 2.5619846498438634,
      "learning_rate": 1.479076479076479e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.7890625,
      "logps/chosen": -168.0,
      "logps/rejected": -156.0,
      "loss": 0.6505,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.0830078125,
      "rewards/margins": 0.0849609375,
      "rewards/rejected": -0.0020599365234375,
      "step": 410
    },
    {
      "epoch": 0.03031834259727135,
      "grad_norm": 2.5720164706506514,
      "learning_rate": 1.5151515151515152e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.7890625,
      "logps/chosen": -191.0,
      "logps/rejected": -166.0,
      "loss": 0.6485,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.06689453125,
      "rewards/margins": 0.08203125,
      "rewards/rejected": -0.01519775390625,
      "step": 420
    },
    {
      "epoch": 0.031040207897206382,
      "grad_norm": 2.4553139382857316,
      "learning_rate": 1.5512265512265513e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.73046875,
      "logps/chosen": -191.0,
      "logps/rejected": -175.0,
      "loss": 0.6445,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.053466796875,
      "rewards/margins": 0.12158203125,
      "rewards/rejected": -0.068359375,
      "step": 430
    },
    {
      "epoch": 0.031762073197141416,
      "grad_norm": 3.051084380996333,
      "learning_rate": 1.5873015873015872e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.7734375,
      "logps/chosen": -175.0,
      "logps/rejected": -180.0,
      "loss": 0.6366,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0380859375,
      "rewards/margins": 0.10693359375,
      "rewards/rejected": -0.06884765625,
      "step": 440
    },
    {
      "epoch": 0.032483938497076446,
      "grad_norm": 2.6944820958477758,
      "learning_rate": 1.6233766233766232e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.76953125,
      "logps/chosen": -188.0,
      "logps/rejected": -179.0,
      "loss": 0.6335,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.021484375,
      "rewards/margins": 0.11962890625,
      "rewards/rejected": -0.09814453125,
      "step": 450
    },
    {
      "epoch": 0.03320580379701148,
      "grad_norm": 2.5709611998153763,
      "learning_rate": 1.6594516594516593e-07,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.82421875,
      "logps/chosen": -188.0,
      "logps/rejected": -186.0,
      "loss": 0.6262,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.00927734375,
      "rewards/margins": 0.1474609375,
      "rewards/rejected": -0.138671875,
      "step": 460
    },
    {
      "epoch": 0.03392766909694651,
      "grad_norm": 2.8386033435576548,
      "learning_rate": 1.6955266955266957e-07,
      "logits/chosen": -1.4453125,
      "logits/rejected": -0.8671875,
      "logps/chosen": -208.0,
      "logps/rejected": -215.0,
      "loss": 0.6164,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0177001953125,
      "rewards/margins": 0.185546875,
      "rewards/rejected": -0.203125,
      "step": 470
    },
    {
      "epoch": 0.034649534396881544,
      "grad_norm": 2.8141257015789436,
      "learning_rate": 1.7316017316017315e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.96484375,
      "logps/chosen": -194.0,
      "logps/rejected": -194.0,
      "loss": 0.6061,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.049072265625,
      "rewards/margins": 0.2109375,
      "rewards/rejected": -0.259765625,
      "step": 480
    },
    {
      "epoch": 0.035371399696816574,
      "grad_norm": 2.9928631668743217,
      "learning_rate": 1.7676767676767676e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -1.0390625,
      "logps/chosen": -187.0,
      "logps/rejected": -194.0,
      "loss": 0.6015,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.087890625,
      "rewards/margins": 0.205078125,
      "rewards/rejected": -0.29296875,
      "step": 490
    },
    {
      "epoch": 0.036093264996751605,
      "grad_norm": 2.856505681903599,
      "learning_rate": 1.8037518037518037e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.89453125,
      "logps/chosen": -200.0,
      "logps/rejected": -223.0,
      "loss": 0.5954,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.1123046875,
      "rewards/margins": 0.2451171875,
      "rewards/rejected": -0.357421875,
      "step": 500
    },
    {
      "epoch": 0.036815130296686635,
      "grad_norm": 3.036809712740791,
      "learning_rate": 1.8398268398268395e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -0.92578125,
      "logps/chosen": -201.0,
      "logps/rejected": -205.0,
      "loss": 0.5885,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.140625,
      "rewards/margins": 0.255859375,
      "rewards/rejected": -0.396484375,
      "step": 510
    },
    {
      "epoch": 0.03753699559662167,
      "grad_norm": 3.119375343364149,
      "learning_rate": 1.875901875901876e-07,
      "logits/chosen": -1.484375,
      "logits/rejected": -1.0703125,
      "logps/chosen": -199.0,
      "logps/rejected": -218.0,
      "loss": 0.5822,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.177734375,
      "rewards/margins": 0.291015625,
      "rewards/rejected": -0.46875,
      "step": 520
    },
    {
      "epoch": 0.0382588608965567,
      "grad_norm": 2.885162200818327,
      "learning_rate": 1.911976911976912e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.1171875,
      "logps/chosen": -201.0,
      "logps/rejected": -227.0,
      "loss": 0.5772,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.1875,
      "rewards/margins": 0.32421875,
      "rewards/rejected": -0.51171875,
      "step": 530
    },
    {
      "epoch": 0.03898072619649173,
      "grad_norm": 2.999267112298604,
      "learning_rate": 1.948051948051948e-07,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.125,
      "logps/chosen": -217.0,
      "logps/rejected": -255.0,
      "loss": 0.5735,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.251953125,
      "rewards/margins": 0.4375,
      "rewards/rejected": -0.69140625,
      "step": 540
    },
    {
      "epoch": 0.03970259149642677,
      "grad_norm": 3.1053821390312657,
      "learning_rate": 1.984126984126984e-07,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.1953125,
      "logps/chosen": -233.0,
      "logps/rejected": -227.0,
      "loss": 0.5654,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.298828125,
      "rewards/margins": 0.298828125,
      "rewards/rejected": -0.59765625,
      "step": 550
    },
    {
      "epoch": 0.0404244567963618,
      "grad_norm": 3.0671768951561735,
      "learning_rate": 2.02020202020202e-07,
      "logits/chosen": -1.6875,
      "logits/rejected": -1.1875,
      "logps/chosen": -217.0,
      "logps/rejected": -256.0,
      "loss": 0.5543,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.26171875,
      "rewards/margins": 0.419921875,
      "rewards/rejected": -0.6796875,
      "step": 560
    },
    {
      "epoch": 0.04114632209629683,
      "grad_norm": 3.1660918673873604,
      "learning_rate": 2.0562770562770563e-07,
      "logits/chosen": -1.4921875,
      "logits/rejected": -1.234375,
      "logps/chosen": -227.0,
      "logps/rejected": -244.0,
      "loss": 0.5551,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.32421875,
      "rewards/margins": 0.453125,
      "rewards/rejected": -0.77734375,
      "step": 570
    },
    {
      "epoch": 0.04186818739623186,
      "grad_norm": 3.1672828289992503,
      "learning_rate": 2.0923520923520924e-07,
      "logits/chosen": -1.546875,
      "logits/rejected": -1.1953125,
      "logps/chosen": -225.0,
      "logps/rejected": -252.0,
      "loss": 0.566,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.328125,
      "rewards/margins": 0.443359375,
      "rewards/rejected": -0.76953125,
      "step": 580
    },
    {
      "epoch": 0.0425900526961669,
      "grad_norm": 3.5913509323935293,
      "learning_rate": 2.1284271284271282e-07,
      "logits/chosen": -1.515625,
      "logits/rejected": -1.3125,
      "logps/chosen": -220.0,
      "logps/rejected": -276.0,
      "loss": 0.5409,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.333984375,
      "rewards/margins": 0.59765625,
      "rewards/rejected": -0.93359375,
      "step": 590
    },
    {
      "epoch": 0.04331191799610193,
      "grad_norm": 3.3558630310205704,
      "learning_rate": 2.1645021645021643e-07,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.2734375,
      "logps/chosen": -217.0,
      "logps/rejected": -231.0,
      "loss": 0.5279,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.341796875,
      "rewards/margins": 0.365234375,
      "rewards/rejected": -0.70703125,
      "step": 600
    },
    {
      "epoch": 0.04403378329603696,
      "grad_norm": 3.5696240585362786,
      "learning_rate": 2.2005772005772004e-07,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.2890625,
      "logps/chosen": -242.0,
      "logps/rejected": -284.0,
      "loss": 0.5382,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.462890625,
      "rewards/margins": 0.51953125,
      "rewards/rejected": -0.984375,
      "step": 610
    },
    {
      "epoch": 0.04475564859597199,
      "grad_norm": 3.7224672776087075,
      "learning_rate": 2.2366522366522368e-07,
      "logits/chosen": -1.7578125,
      "logits/rejected": -1.265625,
      "logps/chosen": -249.0,
      "logps/rejected": -284.0,
      "loss": 0.539,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.546875,
      "rewards/margins": 0.6015625,
      "rewards/rejected": -1.1484375,
      "step": 620
    },
    {
      "epoch": 0.045477513895907026,
      "grad_norm": 4.344200382093463,
      "learning_rate": 2.2727272727272726e-07,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.25,
      "logps/chosen": -243.0,
      "logps/rejected": -292.0,
      "loss": 0.5351,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.5234375,
      "rewards/margins": 0.6171875,
      "rewards/rejected": -1.140625,
      "step": 630
    },
    {
      "epoch": 0.04619937919584206,
      "grad_norm": 3.7306370590115874,
      "learning_rate": 2.3088023088023087e-07,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.3125,
      "logps/chosen": -249.0,
      "logps/rejected": -306.0,
      "loss": 0.5185,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.4921875,
      "rewards/margins": 0.75390625,
      "rewards/rejected": -1.2421875,
      "step": 640
    },
    {
      "epoch": 0.04692124449577709,
      "grad_norm": 4.147030724252931,
      "learning_rate": 2.3448773448773448e-07,
      "logits/chosen": -1.6328125,
      "logits/rejected": -1.3125,
      "logps/chosen": -237.0,
      "logps/rejected": -290.0,
      "loss": 0.5022,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.458984375,
      "rewards/margins": 0.6328125,
      "rewards/rejected": -1.09375,
      "step": 650
    },
    {
      "epoch": 0.04764310979571212,
      "grad_norm": 4.193961032620522,
      "learning_rate": 2.3809523809523806e-07,
      "logits/chosen": -1.6796875,
      "logits/rejected": -1.3828125,
      "logps/chosen": -245.0,
      "logps/rejected": -292.0,
      "loss": 0.5076,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.57421875,
      "rewards/margins": 0.66796875,
      "rewards/rejected": -1.2421875,
      "step": 660
    },
    {
      "epoch": 0.048364975095647154,
      "grad_norm": 4.54243210678223,
      "learning_rate": 2.4170274170274167e-07,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.3671875,
      "logps/chosen": -228.0,
      "logps/rejected": -300.0,
      "loss": 0.5116,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.53125,
      "rewards/margins": 0.73828125,
      "rewards/rejected": -1.2734375,
      "step": 670
    },
    {
      "epoch": 0.049086840395582185,
      "grad_norm": 4.584027220658785,
      "learning_rate": 2.453102453102453e-07,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.328125,
      "logps/chosen": -247.0,
      "logps/rejected": -288.0,
      "loss": 0.5067,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.6171875,
      "rewards/margins": 0.6015625,
      "rewards/rejected": -1.21875,
      "step": 680
    },
    {
      "epoch": 0.049808705695517215,
      "grad_norm": 4.406380183123871,
      "learning_rate": 2.4891774891774894e-07,
      "logits/chosen": -1.515625,
      "logits/rejected": -1.3515625,
      "logps/chosen": -258.0,
      "logps/rejected": -308.0,
      "loss": 0.5053,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.59765625,
      "rewards/margins": 0.62109375,
      "rewards/rejected": -1.21875,
      "step": 690
    },
    {
      "epoch": 0.050530570995452245,
      "grad_norm": 4.687814253568737,
      "learning_rate": 2.525252525252525e-07,
      "logits/chosen": -1.578125,
      "logits/rejected": -1.265625,
      "logps/chosen": -243.0,
      "logps/rejected": -328.0,
      "loss": 0.4984,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.58203125,
      "rewards/margins": 0.88671875,
      "rewards/rejected": -1.46875,
      "step": 700
    },
    {
      "epoch": 0.05125243629538728,
      "grad_norm": 4.8119655192989415,
      "learning_rate": 2.5613275613275616e-07,
      "logits/chosen": -1.6328125,
      "logits/rejected": -1.328125,
      "logps/chosen": -245.0,
      "logps/rejected": -302.0,
      "loss": 0.4971,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.5859375,
      "rewards/margins": 0.703125,
      "rewards/rejected": -1.2890625,
      "step": 710
    },
    {
      "epoch": 0.05197430159532231,
      "grad_norm": 6.052904072624315,
      "learning_rate": 2.597402597402597e-07,
      "logits/chosen": -1.6953125,
      "logits/rejected": -1.421875,
      "logps/chosen": -251.0,
      "logps/rejected": -316.0,
      "loss": 0.5023,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.61328125,
      "rewards/margins": 0.75,
      "rewards/rejected": -1.359375,
      "step": 720
    },
    {
      "epoch": 0.05269616689525734,
      "grad_norm": 5.570866818111687,
      "learning_rate": 2.633477633477633e-07,
      "logits/chosen": -1.5546875,
      "logits/rejected": -1.234375,
      "logps/chosen": -256.0,
      "logps/rejected": -346.0,
      "loss": 0.4847,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.61328125,
      "rewards/margins": 0.8828125,
      "rewards/rejected": -1.4921875,
      "step": 730
    },
    {
      "epoch": 0.05341803219519238,
      "grad_norm": 5.838682503966582,
      "learning_rate": 2.6695526695526696e-07,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.359375,
      "logps/chosen": -270.0,
      "logps/rejected": -336.0,
      "loss": 0.4819,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.70703125,
      "rewards/margins": 0.890625,
      "rewards/rejected": -1.6015625,
      "step": 740
    },
    {
      "epoch": 0.05413989749512741,
      "grad_norm": 5.5396658749324095,
      "learning_rate": 2.7056277056277054e-07,
      "logits/chosen": -1.546875,
      "logits/rejected": -1.2734375,
      "logps/chosen": -247.0,
      "logps/rejected": -326.0,
      "loss": 0.4744,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.56640625,
      "rewards/margins": 0.9296875,
      "rewards/rejected": -1.5,
      "step": 750
    },
    {
      "epoch": 0.05486176279506244,
      "grad_norm": 9.526229074396465,
      "learning_rate": 2.741702741702742e-07,
      "logits/chosen": -1.578125,
      "logits/rejected": -1.3203125,
      "logps/chosen": -255.0,
      "logps/rejected": -296.0,
      "loss": 0.4796,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.64453125,
      "rewards/margins": 0.66796875,
      "rewards/rejected": -1.3125,
      "step": 760
    },
    {
      "epoch": 0.05558362809499747,
      "grad_norm": 5.81469756651447,
      "learning_rate": 2.7777777777777776e-07,
      "logits/chosen": -1.5859375,
      "logits/rejected": -1.2578125,
      "logps/chosen": -256.0,
      "logps/rejected": -314.0,
      "loss": 0.472,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.6953125,
      "rewards/margins": 0.76953125,
      "rewards/rejected": -1.4609375,
      "step": 770
    },
    {
      "epoch": 0.05630549339493251,
      "grad_norm": 4.7992757476430175,
      "learning_rate": 2.813852813852814e-07,
      "logits/chosen": -1.625,
      "logits/rejected": -1.25,
      "logps/chosen": -254.0,
      "logps/rejected": -360.0,
      "loss": 0.455,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.71484375,
      "rewards/margins": 0.95703125,
      "rewards/rejected": -1.671875,
      "step": 780
    },
    {
      "epoch": 0.05702735869486754,
      "grad_norm": 7.941076508547649,
      "learning_rate": 2.8499278499278503e-07,
      "logits/chosen": -1.5,
      "logits/rejected": -1.2109375,
      "logps/chosen": -270.0,
      "logps/rejected": -394.0,
      "loss": 0.4504,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.80078125,
      "rewards/margins": 1.203125,
      "rewards/rejected": -2.0,
      "step": 790
    },
    {
      "epoch": 0.05774922399480257,
      "grad_norm": 8.6863328809609,
      "learning_rate": 2.8860028860028856e-07,
      "logits/chosen": -1.484375,
      "logits/rejected": -1.1484375,
      "logps/chosen": -256.0,
      "logps/rejected": -332.0,
      "loss": 0.4801,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.72265625,
      "rewards/margins": 0.97265625,
      "rewards/rejected": -1.6953125,
      "step": 800
    },
    {
      "epoch": 0.0584710892947376,
      "grad_norm": 5.69286581039463,
      "learning_rate": 2.922077922077922e-07,
      "logits/chosen": -1.578125,
      "logits/rejected": -1.34375,
      "logps/chosen": -243.0,
      "logps/rejected": -340.0,
      "loss": 0.4628,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.76953125,
      "rewards/margins": 0.90234375,
      "rewards/rejected": -1.671875,
      "step": 810
    },
    {
      "epoch": 0.05919295459467264,
      "grad_norm": 4.761532874378853,
      "learning_rate": 2.958152958152958e-07,
      "logits/chosen": -1.5078125,
      "logits/rejected": -1.203125,
      "logps/chosen": -253.0,
      "logps/rejected": -354.0,
      "loss": 0.4711,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.7265625,
      "rewards/margins": 0.95703125,
      "rewards/rejected": -1.6875,
      "step": 820
    },
    {
      "epoch": 0.05991481989460767,
      "grad_norm": 6.373649034849672,
      "learning_rate": 2.994227994227994e-07,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.1484375,
      "logps/chosen": -268.0,
      "logps/rejected": -384.0,
      "loss": 0.4467,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.765625,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -2.03125,
      "step": 830
    },
    {
      "epoch": 0.0606366851945427,
      "grad_norm": 5.855984801506596,
      "learning_rate": 3.0303030303030305e-07,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.34375,
      "logps/chosen": -250.0,
      "logps/rejected": -360.0,
      "loss": 0.454,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.76953125,
      "rewards/margins": 1.0703125,
      "rewards/rejected": -1.84375,
      "step": 840
    },
    {
      "epoch": 0.06135855049447773,
      "grad_norm": 6.973545631235432,
      "learning_rate": 3.0663780663780663e-07,
      "logits/chosen": -1.6796875,
      "logits/rejected": -1.296875,
      "logps/chosen": -268.0,
      "logps/rejected": -396.0,
      "loss": 0.4396,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.81640625,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -1.984375,
      "step": 850
    },
    {
      "epoch": 0.062080415794412765,
      "grad_norm": 6.317516328518519,
      "learning_rate": 3.1024531024531027e-07,
      "logits/chosen": -1.6875,
      "logits/rejected": -1.453125,
      "logps/chosen": -276.0,
      "logps/rejected": -366.0,
      "loss": 0.4584,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.88671875,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -1.921875,
      "step": 860
    },
    {
      "epoch": 0.0628022810943478,
      "grad_norm": 7.180798239192185,
      "learning_rate": 3.138528138528138e-07,
      "logits/chosen": -1.515625,
      "logits/rejected": -1.359375,
      "logps/chosen": -278.0,
      "logps/rejected": -372.0,
      "loss": 0.449,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.953125,
      "rewards/margins": 0.98046875,
      "rewards/rejected": -1.9375,
      "step": 870
    },
    {
      "epoch": 0.06352414639428283,
      "grad_norm": 7.735093757449682,
      "learning_rate": 3.1746031746031743e-07,
      "logits/chosen": -1.59375,
      "logits/rejected": -1.3828125,
      "logps/chosen": -256.0,
      "logps/rejected": -354.0,
      "loss": 0.4342,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.8515625,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -1.875,
      "step": 880
    },
    {
      "epoch": 0.06424601169421786,
      "grad_norm": 6.387583784709454,
      "learning_rate": 3.2106782106782107e-07,
      "logits/chosen": -1.515625,
      "logits/rejected": -1.3671875,
      "logps/chosen": -284.0,
      "logps/rejected": -378.0,
      "loss": 0.423,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.96875,
      "rewards/margins": 1.09375,
      "rewards/rejected": -2.0625,
      "step": 890
    },
    {
      "epoch": 0.06496787699415289,
      "grad_norm": 6.448916258254095,
      "learning_rate": 3.2467532467532465e-07,
      "logits/chosen": -1.65625,
      "logits/rejected": -1.34375,
      "logps/chosen": -272.0,
      "logps/rejected": -364.0,
      "loss": 0.4199,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8359375,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -1.9375,
      "step": 900
    },
    {
      "epoch": 0.06568974229408793,
      "grad_norm": 6.989741466020228,
      "learning_rate": 3.282828282828283e-07,
      "logits/chosen": -1.6015625,
      "logits/rejected": -1.2734375,
      "logps/chosen": -310.0,
      "logps/rejected": -418.0,
      "loss": 0.4373,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.21875,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -2.328125,
      "step": 910
    },
    {
      "epoch": 0.06641160759402295,
      "grad_norm": 7.651962050997523,
      "learning_rate": 3.3189033189033187e-07,
      "logits/chosen": -1.6484375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -286.0,
      "logps/rejected": -376.0,
      "loss": 0.4121,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.83203125,
      "rewards/margins": 1.21875,
      "rewards/rejected": -2.046875,
      "step": 920
    },
    {
      "epoch": 0.06713347289395799,
      "grad_norm": 6.115537041094403,
      "learning_rate": 3.354978354978355e-07,
      "logits/chosen": -1.6328125,
      "logits/rejected": -1.4765625,
      "logps/chosen": -292.0,
      "logps/rejected": -384.0,
      "loss": 0.4158,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.0390625,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -2.203125,
      "step": 930
    },
    {
      "epoch": 0.06785533819389301,
      "grad_norm": 10.050173447148442,
      "learning_rate": 3.3910533910533914e-07,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.40625,
      "logps/chosen": -264.0,
      "logps/rejected": -384.0,
      "loss": 0.4239,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.80078125,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -2.0625,
      "step": 940
    },
    {
      "epoch": 0.06857720349382805,
      "grad_norm": 6.557851833078066,
      "learning_rate": 3.4271284271284267e-07,
      "logits/chosen": -1.6015625,
      "logits/rejected": -1.40625,
      "logps/chosen": -278.0,
      "logps/rejected": -358.0,
      "loss": 0.4278,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.9140625,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -1.953125,
      "step": 950
    },
    {
      "epoch": 0.06929906879376309,
      "grad_norm": 8.540295504657017,
      "learning_rate": 3.463203463203463e-07,
      "logits/chosen": -1.7578125,
      "logits/rejected": -1.4140625,
      "logps/chosen": -290.0,
      "logps/rejected": -424.0,
      "loss": 0.3999,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.0390625,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -2.484375,
      "step": 960
    },
    {
      "epoch": 0.07002093409369811,
      "grad_norm": 13.595580486598363,
      "learning_rate": 3.499278499278499e-07,
      "logits/chosen": -1.703125,
      "logits/rejected": -1.4921875,
      "logps/chosen": -296.0,
      "logps/rejected": -416.0,
      "loss": 0.4197,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.15625,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -2.5,
      "step": 970
    },
    {
      "epoch": 0.07074279939363315,
      "grad_norm": 7.487247659451693,
      "learning_rate": 3.535353535353535e-07,
      "logits/chosen": -1.65625,
      "logits/rejected": -1.4375,
      "logps/chosen": -264.0,
      "logps/rejected": -412.0,
      "loss": 0.4259,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.83984375,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -2.359375,
      "step": 980
    },
    {
      "epoch": 0.07146466469356819,
      "grad_norm": 7.77493237415594,
      "learning_rate": 3.5714285714285716e-07,
      "logits/chosen": -1.6953125,
      "logits/rejected": -1.3359375,
      "logps/chosen": -294.0,
      "logps/rejected": -394.0,
      "loss": 0.3952,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0234375,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -2.296875,
      "step": 990
    },
    {
      "epoch": 0.07218652999350321,
      "grad_norm": 7.601720758151988,
      "learning_rate": 3.6075036075036074e-07,
      "logits/chosen": -1.640625,
      "logits/rejected": -1.515625,
      "logps/chosen": -314.0,
      "logps/rejected": -446.0,
      "loss": 0.4108,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.1484375,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -2.5625,
      "step": 1000
    },
    {
      "epoch": 0.07290839529343825,
      "grad_norm": 10.52304939298257,
      "learning_rate": 3.643578643578644e-07,
      "logits/chosen": -1.6171875,
      "logits/rejected": -1.515625,
      "logps/chosen": -298.0,
      "logps/rejected": -442.0,
      "loss": 0.4031,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.1171875,
      "rewards/margins": 1.453125,
      "rewards/rejected": -2.5625,
      "step": 1010
    },
    {
      "epoch": 0.07363026059337327,
      "grad_norm": 9.63701108277932,
      "learning_rate": 3.679653679653679e-07,
      "logits/chosen": -1.7265625,
      "logits/rejected": -1.5390625,
      "logps/chosen": -298.0,
      "logps/rejected": -446.0,
      "loss": 0.4257,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.1640625,
      "rewards/margins": 1.4375,
      "rewards/rejected": -2.609375,
      "step": 1020
    },
    {
      "epoch": 0.07435212589330831,
      "grad_norm": 9.398814057739154,
      "learning_rate": 3.7157287157287154e-07,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.3984375,
      "logps/chosen": -266.0,
      "logps/rejected": -412.0,
      "loss": 0.4157,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.97265625,
      "rewards/margins": 1.3125,
      "rewards/rejected": -2.28125,
      "step": 1030
    },
    {
      "epoch": 0.07507399119324334,
      "grad_norm": 10.192374367523001,
      "learning_rate": 3.751803751803752e-07,
      "logits/chosen": -1.5390625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -302.0,
      "logps/rejected": -418.0,
      "loss": 0.4127,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.21875,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -2.4375,
      "step": 1040
    },
    {
      "epoch": 0.07579585649317837,
      "grad_norm": 41.28584187683781,
      "learning_rate": 3.7878787878787876e-07,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.296875,
      "logps/chosen": -324.0,
      "logps/rejected": -476.0,
      "loss": 0.4299,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.3671875,
      "rewards/margins": 1.34375,
      "rewards/rejected": -2.703125,
      "step": 1050
    },
    {
      "epoch": 0.0765177217931134,
      "grad_norm": 6.944055165338752,
      "learning_rate": 3.823953823953824e-07,
      "logits/chosen": -1.578125,
      "logits/rejected": -1.421875,
      "logps/chosen": -304.0,
      "logps/rejected": -416.0,
      "loss": 0.3947,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.125,
      "rewards/margins": 1.296875,
      "rewards/rejected": -2.421875,
      "step": 1060
    },
    {
      "epoch": 0.07723958709304844,
      "grad_norm": 8.85040415177623,
      "learning_rate": 3.86002886002886e-07,
      "logits/chosen": -1.59375,
      "logits/rejected": -1.3359375,
      "logps/chosen": -302.0,
      "logps/rejected": -436.0,
      "loss": 0.3904,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.140625,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -2.59375,
      "step": 1070
    },
    {
      "epoch": 0.07796145239298347,
      "grad_norm": 10.338904724977338,
      "learning_rate": 3.896103896103896e-07,
      "logits/chosen": -1.6875,
      "logits/rejected": -1.359375,
      "logps/chosen": -326.0,
      "logps/rejected": -446.0,
      "loss": 0.3915,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.3046875,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -2.8125,
      "step": 1080
    },
    {
      "epoch": 0.0786833176929185,
      "grad_norm": 8.37991361585647,
      "learning_rate": 3.9321789321789325e-07,
      "logits/chosen": -1.6640625,
      "logits/rejected": -1.21875,
      "logps/chosen": -308.0,
      "logps/rejected": -436.0,
      "loss": 0.3979,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.0546875,
      "rewards/margins": 1.625,
      "rewards/rejected": -2.671875,
      "step": 1090
    },
    {
      "epoch": 0.07940518299285354,
      "grad_norm": 12.658331566763867,
      "learning_rate": 3.968253968253968e-07,
      "logits/chosen": -1.5859375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -294.0,
      "logps/rejected": -442.0,
      "loss": 0.378,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.140625,
      "rewards/margins": 1.484375,
      "rewards/rejected": -2.625,
      "step": 1100
    },
    {
      "epoch": 0.08012704829278856,
      "grad_norm": 10.283762440436771,
      "learning_rate": 4.004329004329004e-07,
      "logits/chosen": -1.625,
      "logits/rejected": -1.2734375,
      "logps/chosen": -312.0,
      "logps/rejected": -498.0,
      "loss": 0.3921,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.2578125,
      "rewards/margins": 1.875,
      "rewards/rejected": -3.125,
      "step": 1110
    },
    {
      "epoch": 0.0808489135927236,
      "grad_norm": 11.829779911178525,
      "learning_rate": 4.04040404040404e-07,
      "logits/chosen": -1.6015625,
      "logits/rejected": -1.2578125,
      "logps/chosen": -316.0,
      "logps/rejected": -478.0,
      "loss": 0.3926,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.3125,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -2.84375,
      "step": 1120
    },
    {
      "epoch": 0.08157077889265862,
      "grad_norm": 9.559308125502714,
      "learning_rate": 4.0764790764790763e-07,
      "logits/chosen": -1.6640625,
      "logits/rejected": -1.3984375,
      "logps/chosen": -308.0,
      "logps/rejected": -440.0,
      "loss": 0.3956,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.078125,
      "rewards/margins": 1.421875,
      "rewards/rejected": -2.5,
      "step": 1130
    },
    {
      "epoch": 0.08229264419259366,
      "grad_norm": 8.503454944385053,
      "learning_rate": 4.1125541125541127e-07,
      "logits/chosen": -1.7578125,
      "logits/rejected": -1.5390625,
      "logps/chosen": -316.0,
      "logps/rejected": -428.0,
      "loss": 0.3751,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.328125,
      "rewards/margins": 1.28125,
      "rewards/rejected": -2.609375,
      "step": 1140
    },
    {
      "epoch": 0.0830145094925287,
      "grad_norm": 14.5632864224898,
      "learning_rate": 4.1486291486291485e-07,
      "logits/chosen": -1.6875,
      "logits/rejected": -1.5390625,
      "logps/chosen": -330.0,
      "logps/rejected": -474.0,
      "loss": 0.3938,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2421875,
      "rewards/margins": 1.6171875,
      "rewards/rejected": -2.859375,
      "step": 1150
    },
    {
      "epoch": 0.08373637479246372,
      "grad_norm": 11.058433655440979,
      "learning_rate": 4.184704184704185e-07,
      "logits/chosen": -1.625,
      "logits/rejected": -1.234375,
      "logps/chosen": -296.0,
      "logps/rejected": -448.0,
      "loss": 0.3949,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.203125,
      "rewards/margins": 1.4609375,
      "rewards/rejected": -2.671875,
      "step": 1160
    },
    {
      "epoch": 0.08445824009239876,
      "grad_norm": 9.638443936781867,
      "learning_rate": 4.22077922077922e-07,
      "logits/chosen": -1.546875,
      "logits/rejected": -1.3828125,
      "logps/chosen": -320.0,
      "logps/rejected": -474.0,
      "loss": 0.3972,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.390625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -3.0,
      "step": 1170
    },
    {
      "epoch": 0.0851801053923338,
      "grad_norm": 11.216359984994055,
      "learning_rate": 4.2568542568542565e-07,
      "logits/chosen": -1.6328125,
      "logits/rejected": -1.3671875,
      "logps/chosen": -296.0,
      "logps/rejected": -418.0,
      "loss": 0.3777,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.140625,
      "rewards/margins": 1.375,
      "rewards/rejected": -2.515625,
      "step": 1180
    },
    {
      "epoch": 0.08590197069226882,
      "grad_norm": 7.87576028447995,
      "learning_rate": 4.292929292929293e-07,
      "logits/chosen": -1.59375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -308.0,
      "logps/rejected": -460.0,
      "loss": 0.3739,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.265625,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -2.9375,
      "step": 1190
    },
    {
      "epoch": 0.08662383599220386,
      "grad_norm": 9.500853545858137,
      "learning_rate": 4.3290043290043287e-07,
      "logits/chosen": -1.453125,
      "logits/rejected": -1.375,
      "logps/chosen": -340.0,
      "logps/rejected": -472.0,
      "loss": 0.3662,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.3828125,
      "rewards/margins": 1.4375,
      "rewards/rejected": -2.828125,
      "step": 1200
    },
    {
      "epoch": 0.08734570129213888,
      "grad_norm": 9.02098466577823,
      "learning_rate": 4.365079365079365e-07,
      "logits/chosen": -1.65625,
      "logits/rejected": -1.3671875,
      "logps/chosen": -292.0,
      "logps/rejected": -444.0,
      "loss": 0.3714,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.2421875,
      "rewards/margins": 1.53125,
      "rewards/rejected": -2.765625,
      "step": 1210
    },
    {
      "epoch": 0.08806756659207392,
      "grad_norm": 7.13349722763848,
      "learning_rate": 4.401154401154401e-07,
      "logits/chosen": -1.6171875,
      "logits/rejected": -1.3203125,
      "logps/chosen": -332.0,
      "logps/rejected": -490.0,
      "loss": 0.3944,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3359375,
      "rewards/margins": 1.8046875,
      "rewards/rejected": -3.140625,
      "step": 1220
    },
    {
      "epoch": 0.08878943189200895,
      "grad_norm": 10.50925804486657,
      "learning_rate": 4.437229437229437e-07,
      "logits/chosen": -1.6484375,
      "logits/rejected": -1.515625,
      "logps/chosen": -282.0,
      "logps/rejected": -470.0,
      "loss": 0.3741,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.0625,
      "rewards/margins": 1.9296875,
      "rewards/rejected": -2.984375,
      "step": 1230
    },
    {
      "epoch": 0.08951129719194398,
      "grad_norm": 12.604596635238705,
      "learning_rate": 4.4733044733044736e-07,
      "logits/chosen": -1.734375,
      "logits/rejected": -1.53125,
      "logps/chosen": -342.0,
      "logps/rejected": -520.0,
      "loss": 0.3668,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.453125,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -3.28125,
      "step": 1240
    },
    {
      "epoch": 0.09023316249187902,
      "grad_norm": 9.596000301270573,
      "learning_rate": 4.509379509379509e-07,
      "logits/chosen": -1.6484375,
      "logits/rejected": -1.5234375,
      "logps/chosen": -334.0,
      "logps/rejected": -476.0,
      "loss": 0.3514,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 1.53125,
      "rewards/rejected": -3.046875,
      "step": 1250
    },
    {
      "epoch": 0.09095502779181405,
      "grad_norm": 9.165452938414273,
      "learning_rate": 4.545454545454545e-07,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.3046875,
      "logps/chosen": -338.0,
      "logps/rejected": -494.0,
      "loss": 0.3662,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -3.1875,
      "step": 1260
    },
    {
      "epoch": 0.09167689309174908,
      "grad_norm": 9.489910443968904,
      "learning_rate": 4.581529581529581e-07,
      "logits/chosen": -1.6953125,
      "logits/rejected": -1.609375,
      "logps/chosen": -342.0,
      "logps/rejected": -492.0,
      "loss": 0.3542,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.484375,
      "rewards/margins": 1.625,
      "rewards/rejected": -3.125,
      "step": 1270
    },
    {
      "epoch": 0.09239875839168411,
      "grad_norm": 8.638932596854602,
      "learning_rate": 4.6176046176046174e-07,
      "logits/chosen": -1.6171875,
      "logits/rejected": -1.484375,
      "logps/chosen": -334.0,
      "logps/rejected": -466.0,
      "loss": 0.3723,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.296875,
      "rewards/margins": 1.5546875,
      "rewards/rejected": -2.84375,
      "step": 1280
    },
    {
      "epoch": 0.09312062369161915,
      "grad_norm": 8.809176567888128,
      "learning_rate": 4.6536796536796537e-07,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.2890625,
      "logps/chosen": -334.0,
      "logps/rejected": -480.0,
      "loss": 0.3701,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.4921875,
      "rewards/margins": 1.5625,
      "rewards/rejected": -3.046875,
      "step": 1290
    },
    {
      "epoch": 0.09384248899155417,
      "grad_norm": 8.404995985635086,
      "learning_rate": 4.6897546897546896e-07,
      "logits/chosen": -1.765625,
      "logits/rejected": -1.5703125,
      "logps/chosen": -332.0,
      "logps/rejected": -482.0,
      "loss": 0.3833,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.3125,
      "rewards/margins": 1.7421875,
      "rewards/rejected": -3.0625,
      "step": 1300
    },
    {
      "epoch": 0.09456435429148921,
      "grad_norm": 10.843234383616394,
      "learning_rate": 4.725829725829726e-07,
      "logits/chosen": -1.6484375,
      "logits/rejected": -1.4765625,
      "logps/chosen": -348.0,
      "logps/rejected": -486.0,
      "loss": 0.3767,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 1.5,
      "rewards/rejected": -3.09375,
      "step": 1310
    },
    {
      "epoch": 0.09528621959142423,
      "grad_norm": 11.801739144496658,
      "learning_rate": 4.761904761904761e-07,
      "logits/chosen": -1.765625,
      "logits/rejected": -1.5546875,
      "logps/chosen": -338.0,
      "logps/rejected": -464.0,
      "loss": 0.3533,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.4921875,
      "rewards/margins": 1.421875,
      "rewards/rejected": -2.90625,
      "step": 1320
    },
    {
      "epoch": 0.09600808489135927,
      "grad_norm": 9.918268345197697,
      "learning_rate": 4.797979797979798e-07,
      "logits/chosen": -1.71875,
      "logits/rejected": -1.4609375,
      "logps/chosen": -346.0,
      "logps/rejected": -532.0,
      "loss": 0.3717,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.6875,
      "rewards/margins": 1.9765625,
      "rewards/rejected": -3.65625,
      "step": 1330
    },
    {
      "epoch": 0.09672995019129431,
      "grad_norm": 9.876498245280562,
      "learning_rate": 4.834054834054833e-07,
      "logits/chosen": -1.6875,
      "logits/rejected": -1.5078125,
      "logps/chosen": -314.0,
      "logps/rejected": -466.0,
      "loss": 0.3715,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.3359375,
      "rewards/margins": 1.46875,
      "rewards/rejected": -2.8125,
      "step": 1340
    },
    {
      "epoch": 0.09745181549122933,
      "grad_norm": 12.051580701047904,
      "learning_rate": 4.87012987012987e-07,
      "logits/chosen": -1.71875,
      "logits/rejected": -1.40625,
      "logps/chosen": -358.0,
      "logps/rejected": -512.0,
      "loss": 0.3496,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.5,
      "rewards/margins": 1.8125,
      "rewards/rejected": -3.3125,
      "step": 1350
    },
    {
      "epoch": 0.09817368079116437,
      "grad_norm": 8.098728427756836,
      "learning_rate": 4.906204906204906e-07,
      "logits/chosen": -1.71875,
      "logits/rejected": -1.421875,
      "logps/chosen": -366.0,
      "logps/rejected": -536.0,
      "loss": 0.382,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -3.421875,
      "step": 1360
    },
    {
      "epoch": 0.0988955460910994,
      "grad_norm": 9.8793727391909,
      "learning_rate": 4.942279942279942e-07,
      "logits/chosen": -1.6640625,
      "logits/rejected": -1.4921875,
      "logps/chosen": -322.0,
      "logps/rejected": -460.0,
      "loss": 0.3573,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3515625,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -3.0,
      "step": 1370
    },
    {
      "epoch": 0.09961741139103443,
      "grad_norm": 8.60139307812865,
      "learning_rate": 4.978354978354979e-07,
      "logits/chosen": -1.734375,
      "logits/rejected": -1.6640625,
      "logps/chosen": -346.0,
      "logps/rejected": -524.0,
      "loss": 0.3426,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3984375,
      "rewards/margins": 1.921875,
      "rewards/rejected": -3.3125,
      "step": 1380
    },
    {
      "epoch": 0.10033927669096947,
      "grad_norm": 9.106943350821739,
      "learning_rate": 4.999998729993963e-07,
      "logits/chosen": -1.703125,
      "logits/rejected": -1.4765625,
      "logps/chosen": -346.0,
      "logps/rejected": -510.0,
      "loss": 0.3612,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 1.84375,
      "rewards/rejected": -3.40625,
      "step": 1390
    },
    {
      "epoch": 0.10106114199090449,
      "grad_norm": 12.030063330989666,
      "learning_rate": 4.999984442440868e-07,
      "logits/chosen": -1.6328125,
      "logits/rejected": -1.40625,
      "logps/chosen": -310.0,
      "logps/rejected": -516.0,
      "loss": 0.3417,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.4140625,
      "rewards/margins": 2.140625,
      "rewards/rejected": -3.5625,
      "step": 1400
    },
    {
      "epoch": 0.10178300729083953,
      "grad_norm": 10.06694295983046,
      "learning_rate": 4.99995427991816e-07,
      "logits/chosen": -1.734375,
      "logits/rejected": -1.5,
      "logps/chosen": -356.0,
      "logps/rejected": -536.0,
      "loss": 0.3394,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 1.984375,
      "rewards/rejected": -3.59375,
      "step": 1410
    },
    {
      "epoch": 0.10250487259077457,
      "grad_norm": 9.448817692400238,
      "learning_rate": 4.999908242617371e-07,
      "logits/chosen": -1.78125,
      "logits/rejected": -1.6640625,
      "logps/chosen": -350.0,
      "logps/rejected": -540.0,
      "loss": 0.3419,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.453125,
      "rewards/margins": 1.84375,
      "rewards/rejected": -3.296875,
      "step": 1420
    },
    {
      "epoch": 0.10322673789070959,
      "grad_norm": 8.673596413636751,
      "learning_rate": 4.99984633083084e-07,
      "logits/chosen": -1.515625,
      "logits/rejected": -1.4375,
      "logps/chosen": -320.0,
      "logps/rejected": -478.0,
      "loss": 0.3655,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.4375,
      "rewards/margins": 1.515625,
      "rewards/rejected": -2.953125,
      "step": 1430
    },
    {
      "epoch": 0.10394860319064463,
      "grad_norm": 11.218785941676192,
      "learning_rate": 4.99976854495171e-07,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.4375,
      "logps/chosen": -328.0,
      "logps/rejected": -494.0,
      "loss": 0.3444,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.59375,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -3.28125,
      "step": 1440
    },
    {
      "epoch": 0.10467046849057966,
      "grad_norm": 10.858562069323627,
      "learning_rate": 4.999674885473922e-07,
      "logits/chosen": -1.8359375,
      "logits/rejected": -1.5078125,
      "logps/chosen": -354.0,
      "logps/rejected": -560.0,
      "loss": 0.3459,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.59375,
      "rewards/margins": 1.8359375,
      "rewards/rejected": -3.4375,
      "step": 1450
    },
    {
      "epoch": 0.10539233379051469,
      "grad_norm": 10.929942977968295,
      "learning_rate": 4.999565352992216e-07,
      "logits/chosen": -1.6953125,
      "logits/rejected": -1.578125,
      "logps/chosen": -340.0,
      "logps/rejected": -494.0,
      "loss": 0.3618,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.2890625,
      "rewards/margins": 1.8125,
      "rewards/rejected": -3.109375,
      "step": 1460
    },
    {
      "epoch": 0.10611419909044972,
      "grad_norm": 8.850407960614582,
      "learning_rate": 4.999439948202127e-07,
      "logits/chosen": -1.734375,
      "logits/rejected": -1.515625,
      "logps/chosen": -360.0,
      "logps/rejected": -512.0,
      "loss": 0.3387,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.703125,
      "rewards/margins": 1.9140625,
      "rewards/rejected": -3.609375,
      "step": 1470
    },
    {
      "epoch": 0.10683606439038476,
      "grad_norm": 11.569120439628279,
      "learning_rate": 4.999298671899977e-07,
      "logits/chosen": -1.640625,
      "logits/rejected": -1.5234375,
      "logps/chosen": -360.0,
      "logps/rejected": -528.0,
      "loss": 0.348,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.546875,
      "rewards/margins": 1.984375,
      "rewards/rejected": -3.53125,
      "step": 1480
    },
    {
      "epoch": 0.10755792969031978,
      "grad_norm": 13.134584276330152,
      "learning_rate": 4.999141524982877e-07,
      "logits/chosen": -1.640625,
      "logits/rejected": -1.5703125,
      "logps/chosen": -352.0,
      "logps/rejected": -576.0,
      "loss": 0.337,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 2.359375,
      "rewards/rejected": -4.0,
      "step": 1490
    },
    {
      "epoch": 0.10827979499025482,
      "grad_norm": 11.128753256545982,
      "learning_rate": 4.998968508448714e-07,
      "logits/chosen": -1.6953125,
      "logits/rejected": -1.4375,
      "logps/chosen": -364.0,
      "logps/rejected": -532.0,
      "loss": 0.3522,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -3.515625,
      "step": 1500
    },
    {
      "epoch": 0.10900166029018984,
      "grad_norm": 12.357381196341924,
      "learning_rate": 4.998779623396146e-07,
      "logits/chosen": -1.734375,
      "logits/rejected": -1.46875,
      "logps/chosen": -390.0,
      "logps/rejected": -560.0,
      "loss": 0.3307,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.7734375,
      "rewards/margins": 2.078125,
      "rewards/rejected": -3.84375,
      "step": 1510
    },
    {
      "epoch": 0.10972352559012488,
      "grad_norm": 11.586008020522211,
      "learning_rate": 4.9985748710246e-07,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.375,
      "logps/chosen": -338.0,
      "logps/rejected": -502.0,
      "loss": 0.359,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 1.75,
      "rewards/rejected": -3.296875,
      "step": 1520
    },
    {
      "epoch": 0.11044539089005992,
      "grad_norm": 9.91417030534485,
      "learning_rate": 4.998354252634257e-07,
      "logits/chosen": -1.6171875,
      "logits/rejected": -1.46875,
      "logps/chosen": -358.0,
      "logps/rejected": -524.0,
      "loss": 0.3122,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.671875,
      "rewards/margins": 1.921875,
      "rewards/rejected": -3.59375,
      "step": 1530
    },
    {
      "epoch": 0.11116725618999494,
      "grad_norm": 11.9152934106706,
      "learning_rate": 4.998117769626051e-07,
      "logits/chosen": -1.6015625,
      "logits/rejected": -1.5390625,
      "logps/chosen": -352.0,
      "logps/rejected": -540.0,
      "loss": 0.3244,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.671875,
      "rewards/margins": 1.9375,
      "rewards/rejected": -3.609375,
      "step": 1540
    },
    {
      "epoch": 0.11188912148992998,
      "grad_norm": 8.481054674797894,
      "learning_rate": 4.997865423501657e-07,
      "logits/chosen": -1.7109375,
      "logits/rejected": -1.515625,
      "logps/chosen": -326.0,
      "logps/rejected": -476.0,
      "loss": 0.3527,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.5,
      "rewards/margins": 1.609375,
      "rewards/rejected": -3.109375,
      "step": 1550
    },
    {
      "epoch": 0.11261098678986502,
      "grad_norm": 13.86973588243038,
      "learning_rate": 4.997597215863477e-07,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.375,
      "logps/chosen": -358.0,
      "logps/rejected": -580.0,
      "loss": 0.3275,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 2.515625,
      "rewards/rejected": -4.09375,
      "step": 1560
    },
    {
      "epoch": 0.11333285208980004,
      "grad_norm": 12.779326116493381,
      "learning_rate": 4.99731314841464e-07,
      "logits/chosen": -1.671875,
      "logits/rejected": -1.53125,
      "logps/chosen": -354.0,
      "logps/rejected": -512.0,
      "loss": 0.315,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.640625,
      "rewards/margins": 1.78125,
      "rewards/rejected": -3.421875,
      "step": 1570
    },
    {
      "epoch": 0.11405471738973508,
      "grad_norm": 9.1229629854827,
      "learning_rate": 4.997013222958978e-07,
      "logits/chosen": -1.5546875,
      "logits/rejected": -1.4453125,
      "logps/chosen": -336.0,
      "logps/rejected": -506.0,
      "loss": 0.3351,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.375,
      "rewards/margins": 1.9296875,
      "rewards/rejected": -3.296875,
      "step": 1580
    },
    {
      "epoch": 0.1147765826896701,
      "grad_norm": 9.40616816702191,
      "learning_rate": 4.99669744140103e-07,
      "logits/chosen": -1.6015625,
      "logits/rejected": -1.5390625,
      "logps/chosen": -334.0,
      "logps/rejected": -510.0,
      "loss": 0.3276,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.6875,
      "rewards/margins": 1.84375,
      "rewards/rejected": -3.53125,
      "step": 1590
    },
    {
      "epoch": 0.11549844798960514,
      "grad_norm": 10.53142171546676,
      "learning_rate": 4.996365805746015e-07,
      "logits/chosen": -1.7421875,
      "logits/rejected": -1.484375,
      "logps/chosen": -352.0,
      "logps/rejected": -580.0,
      "loss": 0.3154,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.734375,
      "rewards/margins": 2.3125,
      "rewards/rejected": -4.0625,
      "step": 1600
    },
    {
      "epoch": 0.11622031328954018,
      "grad_norm": 9.055745196277467,
      "learning_rate": 4.996018318099829e-07,
      "logits/chosen": -1.71875,
      "logits/rejected": -1.4140625,
      "logps/chosen": -354.0,
      "logps/rejected": -540.0,
      "loss": 0.3178,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 1.9765625,
      "rewards/rejected": -3.671875,
      "step": 1610
    },
    {
      "epoch": 0.1169421785894752,
      "grad_norm": 12.457017675622993,
      "learning_rate": 4.995654980669028e-07,
      "logits/chosen": -1.78125,
      "logits/rejected": -1.390625,
      "logps/chosen": -376.0,
      "logps/rejected": -636.0,
      "loss": 0.3264,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.75,
      "rewards/margins": 2.734375,
      "rewards/rejected": -4.5,
      "step": 1620
    },
    {
      "epoch": 0.11766404388941024,
      "grad_norm": 9.359756811193828,
      "learning_rate": 4.995275795760816e-07,
      "logits/chosen": -1.6328125,
      "logits/rejected": -1.5859375,
      "logps/chosen": -346.0,
      "logps/rejected": -508.0,
      "loss": 0.3304,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 1.703125,
      "rewards/rejected": -3.3125,
      "step": 1630
    },
    {
      "epoch": 0.11838590918934527,
      "grad_norm": 10.884772237956211,
      "learning_rate": 4.994880765783026e-07,
      "logits/chosen": -1.75,
      "logits/rejected": -1.671875,
      "logps/chosen": -370.0,
      "logps/rejected": -520.0,
      "loss": 0.3152,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.703125,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -3.390625,
      "step": 1640
    },
    {
      "epoch": 0.1191077744892803,
      "grad_norm": 8.127699043869297,
      "learning_rate": 4.99446989324411e-07,
      "logits/chosen": -1.734375,
      "logits/rejected": -1.4921875,
      "logps/chosen": -376.0,
      "logps/rejected": -564.0,
      "loss": 0.307,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 2.09375,
      "rewards/rejected": -3.828125,
      "step": 1650
    },
    {
      "epoch": 0.11982963978921533,
      "grad_norm": 11.007870522484678,
      "learning_rate": 4.99404318075312e-07,
      "logits/chosen": -1.6953125,
      "logits/rejected": -1.546875,
      "logps/chosen": -360.0,
      "logps/rejected": -502.0,
      "loss": 0.3267,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.65625,
      "rewards/margins": 1.75,
      "rewards/rejected": -3.40625,
      "step": 1660
    },
    {
      "epoch": 0.12055150508915037,
      "grad_norm": 12.484986181324327,
      "learning_rate": 4.993600631019689e-07,
      "logits/chosen": -1.7734375,
      "logits/rejected": -1.5234375,
      "logps/chosen": -372.0,
      "logps/rejected": -548.0,
      "loss": 0.3334,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 1.8125,
      "rewards/rejected": -3.78125,
      "step": 1670
    },
    {
      "epoch": 0.1212733703890854,
      "grad_norm": 11.79909205254396,
      "learning_rate": 4.993142246854024e-07,
      "logits/chosen": -1.78125,
      "logits/rejected": -1.6171875,
      "logps/chosen": -340.0,
      "logps/rejected": -548.0,
      "loss": 0.3179,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 2.046875,
      "rewards/rejected": -3.671875,
      "step": 1680
    },
    {
      "epoch": 0.12199523568902043,
      "grad_norm": 8.936845141815601,
      "learning_rate": 4.992668031166876e-07,
      "logits/chosen": -1.8046875,
      "logits/rejected": -1.6796875,
      "logps/chosen": -368.0,
      "logps/rejected": -556.0,
      "loss": 0.3255,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 2.09375,
      "rewards/rejected": -3.953125,
      "step": 1690
    },
    {
      "epoch": 0.12271710098895545,
      "grad_norm": 11.979038208132922,
      "learning_rate": 4.992177986969526e-07,
      "logits/chosen": -1.6953125,
      "logits/rejected": -1.5390625,
      "logps/chosen": -382.0,
      "logps/rejected": -564.0,
      "loss": 0.3332,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 1.796875,
      "rewards/rejected": -3.734375,
      "step": 1700
    },
    {
      "epoch": 0.12343896628889049,
      "grad_norm": 10.746986218839295,
      "learning_rate": 4.991672117373769e-07,
      "logits/chosen": -1.75,
      "logits/rejected": -1.578125,
      "logps/chosen": -372.0,
      "logps/rejected": -540.0,
      "loss": 0.337,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -3.484375,
      "step": 1710
    },
    {
      "epoch": 0.12416083158882553,
      "grad_norm": 9.630930893493563,
      "learning_rate": 4.991150425591891e-07,
      "logits/chosen": -1.7109375,
      "logits/rejected": -1.4765625,
      "logps/chosen": -376.0,
      "logps/rejected": -576.0,
      "loss": 0.3248,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 2.078125,
      "rewards/rejected": -4.0625,
      "step": 1720
    },
    {
      "epoch": 0.12488269688876055,
      "grad_norm": 9.333819482151602,
      "learning_rate": 4.99061291493665e-07,
      "logits/chosen": -1.90625,
      "logits/rejected": -1.7109375,
      "logps/chosen": -370.0,
      "logps/rejected": -564.0,
      "loss": 0.3253,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.859375,
      "rewards/margins": 2.078125,
      "rewards/rejected": -3.9375,
      "step": 1730
    },
    {
      "epoch": 0.1256045621886956,
      "grad_norm": 9.567265468073371,
      "learning_rate": 4.99005958882125e-07,
      "logits/chosen": -1.7421875,
      "logits/rejected": -1.6484375,
      "logps/chosen": -356.0,
      "logps/rejected": -556.0,
      "loss": 0.3446,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.7578125,
      "rewards/margins": 2.0625,
      "rewards/rejected": -3.828125,
      "step": 1740
    },
    {
      "epoch": 0.12632642748863063,
      "grad_norm": 10.311375373059471,
      "learning_rate": 4.989490450759331e-07,
      "logits/chosen": -1.875,
      "logits/rejected": -1.671875,
      "logps/chosen": -370.0,
      "logps/rejected": -564.0,
      "loss": 0.3042,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.734375,
      "rewards/margins": 2.171875,
      "rewards/rejected": -3.90625,
      "step": 1750
    },
    {
      "epoch": 0.12704829278856566,
      "grad_norm": 8.528340273461628,
      "learning_rate": 4.988905504364933e-07,
      "logits/chosen": -1.7890625,
      "logits/rejected": -1.703125,
      "logps/chosen": -402.0,
      "logps/rejected": -612.0,
      "loss": 0.2979,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 2.359375,
      "rewards/rejected": -4.3125,
      "step": 1760
    },
    {
      "epoch": 0.12777015808850067,
      "grad_norm": 9.172308318188918,
      "learning_rate": 4.988304753352482e-07,
      "logits/chosen": -1.8515625,
      "logits/rejected": -1.671875,
      "logps/chosen": -366.0,
      "logps/rejected": -580.0,
      "loss": 0.306,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 2.109375,
      "rewards/rejected": -4.03125,
      "step": 1770
    },
    {
      "epoch": 0.1284920233884357,
      "grad_norm": 9.676743834174015,
      "learning_rate": 4.987688201536764e-07,
      "logits/chosen": -1.765625,
      "logits/rejected": -1.59375,
      "logps/chosen": -382.0,
      "logps/rejected": -584.0,
      "loss": 0.3299,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.859375,
      "rewards/margins": 2.25,
      "rewards/rejected": -4.09375,
      "step": 1780
    },
    {
      "epoch": 0.12921388868837075,
      "grad_norm": 12.43360641083241,
      "learning_rate": 4.987055852832899e-07,
      "logits/chosen": -1.875,
      "logits/rejected": -1.6484375,
      "logps/chosen": -386.0,
      "logps/rejected": -612.0,
      "loss": 0.3033,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.828125,
      "rewards/margins": 2.359375,
      "rewards/rejected": -4.1875,
      "step": 1790
    },
    {
      "epoch": 0.12993575398830579,
      "grad_norm": 10.457822792817186,
      "learning_rate": 4.986407711256319e-07,
      "logits/chosen": -1.8203125,
      "logits/rejected": -1.6640625,
      "logps/chosen": -386.0,
      "logps/rejected": -592.0,
      "loss": 0.316,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.03125,
      "rewards/margins": 2.0625,
      "rewards/rejected": -4.09375,
      "step": 1800
    },
    {
      "epoch": 0.13065761928824082,
      "grad_norm": 9.092999698264402,
      "learning_rate": 4.98574378092274e-07,
      "logits/chosen": -1.8203125,
      "logits/rejected": -1.7421875,
      "logps/chosen": -378.0,
      "logps/rejected": -584.0,
      "loss": 0.2978,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 2.375,
      "rewards/rejected": -4.21875,
      "step": 1810
    },
    {
      "epoch": 0.13137948458817586,
      "grad_norm": 9.944028923592004,
      "learning_rate": 4.985064066048139e-07,
      "logits/chosen": -1.828125,
      "logits/rejected": -1.6015625,
      "logps/chosen": -372.0,
      "logps/rejected": -568.0,
      "loss": 0.322,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 2.015625,
      "rewards/rejected": -3.84375,
      "step": 1820
    },
    {
      "epoch": 0.13210134988811087,
      "grad_norm": 16.71668582293834,
      "learning_rate": 4.984368570948724e-07,
      "logits/chosen": -1.890625,
      "logits/rejected": -1.734375,
      "logps/chosen": -394.0,
      "logps/rejected": -620.0,
      "loss": 0.3113,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.046875,
      "rewards/margins": 2.234375,
      "rewards/rejected": -4.28125,
      "step": 1830
    },
    {
      "epoch": 0.1328232151880459,
      "grad_norm": 11.007937989569053,
      "learning_rate": 4.983657300040907e-07,
      "logits/chosen": -1.8359375,
      "logits/rejected": -1.7421875,
      "logps/chosen": -384.0,
      "logps/rejected": -544.0,
      "loss": 0.3161,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.109375,
      "rewards/margins": 1.78125,
      "rewards/rejected": -3.890625,
      "step": 1840
    },
    {
      "epoch": 0.13354508048798094,
      "grad_norm": 11.600955889575422,
      "learning_rate": 4.982930257841278e-07,
      "logits/chosen": -1.96875,
      "logits/rejected": -1.671875,
      "logps/chosen": -374.0,
      "logps/rejected": -592.0,
      "loss": 0.3284,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.875,
      "rewards/margins": 2.265625,
      "rewards/rejected": -4.125,
      "step": 1850
    },
    {
      "epoch": 0.13426694578791598,
      "grad_norm": 11.043505707474354,
      "learning_rate": 4.982187448966575e-07,
      "logits/chosen": -1.984375,
      "logits/rejected": -1.8359375,
      "logps/chosen": -400.0,
      "logps/rejected": -616.0,
      "loss": 0.3186,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.046875,
      "rewards/margins": 2.234375,
      "rewards/rejected": -4.28125,
      "step": 1860
    },
    {
      "epoch": 0.13498881108785102,
      "grad_norm": 10.94848741299333,
      "learning_rate": 4.981428878133656e-07,
      "logits/chosen": -1.9765625,
      "logits/rejected": -1.6640625,
      "logps/chosen": -354.0,
      "logps/rejected": -588.0,
      "loss": 0.3111,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.6875,
      "rewards/margins": 2.421875,
      "rewards/rejected": -4.125,
      "step": 1870
    },
    {
      "epoch": 0.13571067638778603,
      "grad_norm": 10.816892739400435,
      "learning_rate": 4.980654550159463e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -1.9921875,
      "logps/chosen": -398.0,
      "logps/rejected": -612.0,
      "loss": 0.2951,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 2.515625,
      "rewards/rejected": -4.5,
      "step": 1880
    },
    {
      "epoch": 0.13643254168772107,
      "grad_norm": 9.710511313172901,
      "learning_rate": 4.979864469961004e-07,
      "logits/chosen": -1.8984375,
      "logits/rejected": -1.78125,
      "logps/chosen": -372.0,
      "logps/rejected": -572.0,
      "loss": 0.3061,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.90625,
      "rewards/margins": 2.15625,
      "rewards/rejected": -4.0625,
      "step": 1890
    },
    {
      "epoch": 0.1371544069876561,
      "grad_norm": 9.371273937626315,
      "learning_rate": 4.979058642555307e-07,
      "logits/chosen": -1.9765625,
      "logits/rejected": -1.8203125,
      "logps/chosen": -374.0,
      "logps/rejected": -548.0,
      "loss": 0.3272,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 2.0,
      "rewards/rejected": -3.796875,
      "step": 1900
    },
    {
      "epoch": 0.13787627228759114,
      "grad_norm": 8.585561544330783,
      "learning_rate": 4.978237073059399e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.8984375,
      "logps/chosen": -384.0,
      "logps/rejected": -584.0,
      "loss": 0.3186,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.015625,
      "rewards/margins": 2.09375,
      "rewards/rejected": -4.09375,
      "step": 1910
    },
    {
      "epoch": 0.13859813758752618,
      "grad_norm": 10.44295508614073,
      "learning_rate": 4.977399766690269e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.8984375,
      "logps/chosen": -400.0,
      "logps/rejected": -660.0,
      "loss": 0.2926,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.96875,
      "rewards/margins": 2.765625,
      "rewards/rejected": -4.75,
      "step": 1920
    },
    {
      "epoch": 0.1393200028874612,
      "grad_norm": 9.31955352135655,
      "learning_rate": 4.976546728764836e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.953125,
      "logps/chosen": -408.0,
      "logps/rejected": -616.0,
      "loss": 0.3112,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.140625,
      "rewards/margins": 2.28125,
      "rewards/rejected": -4.4375,
      "step": 1930
    },
    {
      "epoch": 0.14004186818739622,
      "grad_norm": 8.811759679783911,
      "learning_rate": 4.975677964699912e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -2.0625,
      "logps/chosen": -404.0,
      "logps/rejected": -628.0,
      "loss": 0.2893,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.296875,
      "rewards/margins": 2.328125,
      "rewards/rejected": -4.625,
      "step": 1940
    },
    {
      "epoch": 0.14076373348733126,
      "grad_norm": 12.005197746857558,
      "learning_rate": 4.974793480012174e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.859375,
      "logps/chosen": -410.0,
      "logps/rejected": -612.0,
      "loss": 0.2929,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.046875,
      "rewards/margins": 2.234375,
      "rewards/rejected": -4.28125,
      "step": 1950
    },
    {
      "epoch": 0.1414855987872663,
      "grad_norm": 12.131307061733434,
      "learning_rate": 4.973893280318125e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -1.9296875,
      "logps/chosen": -404.0,
      "logps/rejected": -624.0,
      "loss": 0.3203,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.203125,
      "rewards/margins": 2.3125,
      "rewards/rejected": -4.5,
      "step": 1960
    },
    {
      "epoch": 0.14220746408720134,
      "grad_norm": 11.085132469193848,
      "learning_rate": 4.972977371334056e-07,
      "logits/chosen": -2.046875,
      "logits/rejected": -1.9375,
      "logps/chosen": -408.0,
      "logps/rejected": -616.0,
      "loss": 0.3155,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.078125,
      "rewards/margins": 2.109375,
      "rewards/rejected": -4.1875,
      "step": 1970
    },
    {
      "epoch": 0.14292932938713637,
      "grad_norm": 10.713520075634671,
      "learning_rate": 4.972045758876014e-07,
      "logits/chosen": -1.9296875,
      "logits/rejected": -1.90625,
      "logps/chosen": -412.0,
      "logps/rejected": -600.0,
      "loss": 0.2836,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.21875,
      "rewards/margins": 1.9375,
      "rewards/rejected": -4.15625,
      "step": 1980
    },
    {
      "epoch": 0.14365119468707138,
      "grad_norm": 12.619154159082733,
      "learning_rate": 4.971098448859766e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -2.046875,
      "logps/chosen": -424.0,
      "logps/rejected": -624.0,
      "loss": 0.3133,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.34375,
      "rewards/margins": 2.203125,
      "rewards/rejected": -4.53125,
      "step": 1990
    },
    {
      "epoch": 0.14437305998700642,
      "grad_norm": 11.280740464124674,
      "learning_rate": 4.970135447300752e-07,
      "logits/chosen": -1.96875,
      "logits/rejected": -1.78125,
      "logps/chosen": -410.0,
      "logps/rejected": -628.0,
      "loss": 0.2965,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.1875,
      "rewards/margins": 2.046875,
      "rewards/rejected": -4.21875,
      "step": 2000
    },
    {
      "epoch": 0.14509492528694146,
      "grad_norm": 10.989401075759773,
      "learning_rate": 4.969156760314064e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.859375,
      "logps/chosen": -434.0,
      "logps/rejected": -676.0,
      "loss": 0.3111,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.4375,
      "rewards/margins": 2.421875,
      "rewards/rejected": -4.875,
      "step": 2010
    },
    {
      "epoch": 0.1458167905868765,
      "grad_norm": 9.884000471558048,
      "learning_rate": 4.968162394114387e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -1.890625,
      "logps/chosen": -400.0,
      "logps/rejected": -620.0,
      "loss": 0.2985,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.125,
      "rewards/margins": 2.375,
      "rewards/rejected": -4.5,
      "step": 2020
    },
    {
      "epoch": 0.14653865588681153,
      "grad_norm": 9.903209784892821,
      "learning_rate": 4.967152355015974e-07,
      "logits/chosen": -1.8828125,
      "logits/rejected": -1.9453125,
      "logps/chosen": -362.0,
      "logps/rejected": -556.0,
      "loss": 0.3068,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.953125,
      "rewards/margins": 2.0625,
      "rewards/rejected": -4.03125,
      "step": 2030
    },
    {
      "epoch": 0.14726052118674654,
      "grad_norm": 9.889316892988317,
      "learning_rate": 4.966126649432603e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -1.9140625,
      "logps/chosen": -386.0,
      "logps/rejected": -632.0,
      "loss": 0.2943,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.1875,
      "rewards/margins": 2.265625,
      "rewards/rejected": -4.46875,
      "step": 2040
    },
    {
      "epoch": 0.14798238648668158,
      "grad_norm": 9.14070408877811,
      "learning_rate": 4.96508528387753e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -1.90625,
      "logps/chosen": -402.0,
      "logps/rejected": -604.0,
      "loss": 0.3151,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.3125,
      "rewards/margins": 2.0625,
      "rewards/rejected": -4.375,
      "step": 2050
    },
    {
      "epoch": 0.14870425178661661,
      "grad_norm": 10.437422004578684,
      "learning_rate": 4.964028264963456e-07,
      "logits/chosen": -2.046875,
      "logits/rejected": -1.96875,
      "logps/chosen": -390.0,
      "logps/rejected": -668.0,
      "loss": 0.3116,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 2.671875,
      "rewards/rejected": -4.625,
      "step": 2060
    },
    {
      "epoch": 0.14942611708655165,
      "grad_norm": 9.97242816389023,
      "learning_rate": 4.962955599402481e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.828125,
      "logps/chosen": -422.0,
      "logps/rejected": -624.0,
      "loss": 0.2891,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.359375,
      "rewards/margins": 2.078125,
      "rewards/rejected": -4.4375,
      "step": 2070
    },
    {
      "epoch": 0.1501479823864867,
      "grad_norm": 11.765371405718236,
      "learning_rate": 4.961867294006059e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -1.9765625,
      "logps/chosen": -430.0,
      "logps/rejected": -600.0,
      "loss": 0.345,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.40625,
      "rewards/margins": 1.875,
      "rewards/rejected": -4.28125,
      "step": 2080
    },
    {
      "epoch": 0.15086984768642173,
      "grad_norm": 9.097861673410657,
      "learning_rate": 4.96076335568496e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.0625,
      "logps/chosen": -426.0,
      "logps/rejected": -640.0,
      "loss": 0.2796,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.28125,
      "rewards/margins": 2.34375,
      "rewards/rejected": -4.625,
      "step": 2090
    },
    {
      "epoch": 0.15159171298635674,
      "grad_norm": 12.08689034279979,
      "learning_rate": 4.959643791449222e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -2.125,
      "logps/chosen": -438.0,
      "logps/rejected": -648.0,
      "loss": 0.3167,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.296875,
      "rewards/margins": 2.421875,
      "rewards/rejected": -4.6875,
      "step": 2100
    },
    {
      "epoch": 0.15231357828629177,
      "grad_norm": 11.892805741781155,
      "learning_rate": 4.958508608408109e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -1.9140625,
      "logps/chosen": -390.0,
      "logps/rejected": -560.0,
      "loss": 0.3088,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.109375,
      "rewards/margins": 1.859375,
      "rewards/rejected": -3.96875,
      "step": 2110
    },
    {
      "epoch": 0.1530354435862268,
      "grad_norm": 10.964564102574641,
      "learning_rate": 4.957357813770064e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -1.890625,
      "logps/chosen": -412.0,
      "logps/rejected": -624.0,
      "loss": 0.277,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.171875,
      "rewards/margins": 2.25,
      "rewards/rejected": -4.40625,
      "step": 2120
    },
    {
      "epoch": 0.15375730888616185,
      "grad_norm": 11.394723854575908,
      "learning_rate": 4.956191414842665e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.0,
      "logps/chosen": -428.0,
      "logps/rejected": -680.0,
      "loss": 0.2761,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.421875,
      "rewards/margins": 2.578125,
      "rewards/rejected": -5.0,
      "step": 2130
    },
    {
      "epoch": 0.15447917418609688,
      "grad_norm": 10.626058357182513,
      "learning_rate": 4.955009419032575e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -2.03125,
      "logps/chosen": -424.0,
      "logps/rejected": -632.0,
      "loss": 0.3033,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.234375,
      "rewards/margins": 2.28125,
      "rewards/rejected": -4.53125,
      "step": 2140
    },
    {
      "epoch": 0.1552010394860319,
      "grad_norm": 9.80786382526639,
      "learning_rate": 4.953811833845503e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -2.09375,
      "logps/chosen": -418.0,
      "logps/rejected": -612.0,
      "loss": 0.3059,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.359375,
      "rewards/margins": 2.03125,
      "rewards/rejected": -4.40625,
      "step": 2150
    },
    {
      "epoch": 0.15592290478596693,
      "grad_norm": 12.25179726334292,
      "learning_rate": 4.952598666886145e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.140625,
      "logps/chosen": -422.0,
      "logps/rejected": -628.0,
      "loss": 0.2791,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.296875,
      "rewards/margins": 2.171875,
      "rewards/rejected": -4.46875,
      "step": 2160
    },
    {
      "epoch": 0.15664477008590197,
      "grad_norm": 13.496384072412704,
      "learning_rate": 4.951369925858147e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.046875,
      "logps/chosen": -426.0,
      "logps/rejected": -628.0,
      "loss": 0.2886,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.375,
      "rewards/margins": 2.234375,
      "rewards/rejected": -4.59375,
      "step": 2170
    },
    {
      "epoch": 0.157366635385837,
      "grad_norm": 9.094987338350231,
      "learning_rate": 4.950125618564046e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.1875,
      "logps/chosen": -436.0,
      "logps/rejected": -636.0,
      "loss": 0.3019,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.4375,
      "rewards/margins": 2.046875,
      "rewards/rejected": -4.46875,
      "step": 2180
    },
    {
      "epoch": 0.15808850068577204,
      "grad_norm": 8.849183913356645,
      "learning_rate": 4.948865752905228e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -2.109375,
      "logps/chosen": -406.0,
      "logps/rejected": -588.0,
      "loss": 0.3056,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.140625,
      "rewards/margins": 1.9765625,
      "rewards/rejected": -4.125,
      "step": 2190
    },
    {
      "epoch": 0.15881036598570708,
      "grad_norm": 17.153439185865572,
      "learning_rate": 4.947590336881874e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -1.96875,
      "logps/chosen": -422.0,
      "logps/rejected": -636.0,
      "loss": 0.3014,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.171875,
      "rewards/margins": 2.421875,
      "rewards/rejected": -4.59375,
      "step": 2200
    },
    {
      "epoch": 0.1595322312856421,
      "grad_norm": 12.107673212010969,
      "learning_rate": 4.946299378592912e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -2.0625,
      "logps/chosen": -436.0,
      "logps/rejected": -696.0,
      "loss": 0.3051,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.375,
      "rewards/margins": 2.640625,
      "rewards/rejected": -5.0,
      "step": 2210
    },
    {
      "epoch": 0.16025409658557713,
      "grad_norm": 15.352742668675448,
      "learning_rate": 4.944992886235961e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -2.109375,
      "logps/chosen": -442.0,
      "logps/rejected": -656.0,
      "loss": 0.2918,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.390625,
      "rewards/margins": 2.421875,
      "rewards/rejected": -4.8125,
      "step": 2220
    },
    {
      "epoch": 0.16097596188551216,
      "grad_norm": 13.801989108510753,
      "learning_rate": 4.943670868107284e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.1875,
      "logps/chosen": -404.0,
      "logps/rejected": -656.0,
      "loss": 0.2956,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.125,
      "rewards/margins": 2.609375,
      "rewards/rejected": -4.71875,
      "step": 2230
    },
    {
      "epoch": 0.1616978271854472,
      "grad_norm": 12.816456031522316,
      "learning_rate": 4.942333332601731e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -2.109375,
      "logps/chosen": -434.0,
      "logps/rejected": -672.0,
      "loss": 0.2885,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.40625,
      "rewards/margins": 2.609375,
      "rewards/rejected": -5.0,
      "step": 2240
    },
    {
      "epoch": 0.16241969248538224,
      "grad_norm": 11.94989840869529,
      "learning_rate": 4.940980288212691e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.171875,
      "logps/chosen": -448.0,
      "logps/rejected": -652.0,
      "loss": 0.27,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.46875,
      "rewards/margins": 2.296875,
      "rewards/rejected": -4.75,
      "step": 2250
    },
    {
      "epoch": 0.16314155778531725,
      "grad_norm": 11.962904954028824,
      "learning_rate": 4.939611743532031e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.3125,
      "logps/chosen": -422.0,
      "logps/rejected": -644.0,
      "loss": 0.2662,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.265625,
      "rewards/margins": 2.421875,
      "rewards/rejected": -4.6875,
      "step": 2260
    },
    {
      "epoch": 0.16386342308525229,
      "grad_norm": 10.28931740651005,
      "learning_rate": 4.93822770725005e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.203125,
      "logps/chosen": -420.0,
      "logps/rejected": -604.0,
      "loss": 0.2965,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.25,
      "rewards/margins": 2.140625,
      "rewards/rejected": -4.40625,
      "step": 2270
    },
    {
      "epoch": 0.16458528838518732,
      "grad_norm": 10.42623586388014,
      "learning_rate": 4.936828188155413e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.25,
      "logps/chosen": -404.0,
      "logps/rejected": -624.0,
      "loss": 0.2836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.1875,
      "rewards/margins": 2.296875,
      "rewards/rejected": -4.46875,
      "step": 2280
    },
    {
      "epoch": 0.16530715368512236,
      "grad_norm": 12.697774685988367,
      "learning_rate": 4.935413195135106e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.328125,
      "logps/chosen": -458.0,
      "logps/rejected": -740.0,
      "loss": 0.297,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.609375,
      "rewards/margins": 3.0,
      "rewards/rejected": -5.59375,
      "step": 2290
    },
    {
      "epoch": 0.1660290189850574,
      "grad_norm": 10.358734058098408,
      "learning_rate": 4.933982737174374e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.125,
      "logps/chosen": -402.0,
      "logps/rejected": -620.0,
      "loss": 0.3014,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.21875,
      "rewards/margins": 2.109375,
      "rewards/rejected": -4.34375,
      "step": 2300
    },
    {
      "epoch": 0.1667508842849924,
      "grad_norm": 13.59430771481954,
      "learning_rate": 4.932536823356664e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.21875,
      "logps/chosen": -438.0,
      "logps/rejected": -664.0,
      "loss": 0.2772,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.625,
      "rewards/margins": 2.265625,
      "rewards/rejected": -4.875,
      "step": 2310
    },
    {
      "epoch": 0.16747274958492744,
      "grad_norm": 9.295319915823654,
      "learning_rate": 4.931075462863567e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.09375,
      "logps/chosen": -446.0,
      "logps/rejected": -716.0,
      "loss": 0.3012,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.515625,
      "rewards/margins": 2.71875,
      "rewards/rejected": -5.21875,
      "step": 2320
    },
    {
      "epoch": 0.16819461488486248,
      "grad_norm": 9.41311409027581,
      "learning_rate": 4.929598664974762e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.25,
      "logps/chosen": -446.0,
      "logps/rejected": -676.0,
      "loss": 0.2853,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.34375,
      "rewards/margins": 2.65625,
      "rewards/rejected": -5.0,
      "step": 2330
    },
    {
      "epoch": 0.16891648018479752,
      "grad_norm": 11.65247902232989,
      "learning_rate": 4.928106439067958e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.9921875,
      "logps/chosen": -412.0,
      "logps/rejected": -660.0,
      "loss": 0.2929,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.34375,
      "rewards/margins": 2.46875,
      "rewards/rejected": -4.8125,
      "step": 2340
    },
    {
      "epoch": 0.16963834548473256,
      "grad_norm": 11.66832125135654,
      "learning_rate": 4.926598794618829e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.21875,
      "logps/chosen": -404.0,
      "logps/rejected": -620.0,
      "loss": 0.2883,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.140625,
      "rewards/margins": 2.453125,
      "rewards/rejected": -4.59375,
      "step": 2350
    },
    {
      "epoch": 0.1703602107846676,
      "grad_norm": 15.534670056063364,
      "learning_rate": 4.92507574120096e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -2.09375,
      "logps/chosen": -448.0,
      "logps/rejected": -644.0,
      "loss": 0.3048,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.5625,
      "rewards/margins": 2.109375,
      "rewards/rejected": -4.6875,
      "step": 2360
    },
    {
      "epoch": 0.1710820760846026,
      "grad_norm": 11.236064628760017,
      "learning_rate": 4.923537288485779e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -2.09375,
      "logps/chosen": -430.0,
      "logps/rejected": -668.0,
      "loss": 0.275,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.53125,
      "rewards/margins": 2.5,
      "rewards/rejected": -5.03125,
      "step": 2370
    },
    {
      "epoch": 0.17180394138453764,
      "grad_norm": 13.154161861157965,
      "learning_rate": 4.921983446242506e-07,
      "logits/chosen": -2.25,
      "logits/rejected": -2.265625,
      "logps/chosen": -490.0,
      "logps/rejected": -736.0,
      "loss": 0.289,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.8125,
      "rewards/margins": 2.75,
      "rewards/rejected": -5.5625,
      "step": 2380
    },
    {
      "epoch": 0.17252580668447268,
      "grad_norm": 9.650949634453966,
      "learning_rate": 4.92041422433808e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.109375,
      "logps/chosen": -410.0,
      "logps/rejected": -724.0,
      "loss": 0.2766,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.28125,
      "rewards/margins": 3.109375,
      "rewards/rejected": -5.375,
      "step": 2390
    },
    {
      "epoch": 0.17324767198440771,
      "grad_norm": 10.842950734509005,
      "learning_rate": 4.918829632737103e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.046875,
      "logps/chosen": -400.0,
      "logps/rejected": -652.0,
      "loss": 0.2896,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.265625,
      "rewards/margins": 2.53125,
      "rewards/rejected": -4.8125,
      "step": 2400
    },
    {
      "epoch": 0.17396953728434275,
      "grad_norm": 13.83920905975613,
      "learning_rate": 4.917229681501774e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.234375,
      "logps/chosen": -440.0,
      "logps/rejected": -656.0,
      "loss": 0.2897,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.421875,
      "rewards/margins": 2.375,
      "rewards/rejected": -4.78125,
      "step": 2410
    },
    {
      "epoch": 0.17469140258427776,
      "grad_norm": 9.557859907380454,
      "learning_rate": 4.91561438079183e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.34375,
      "logps/chosen": -456.0,
      "logps/rejected": -708.0,
      "loss": 0.293,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.625,
      "rewards/margins": 2.484375,
      "rewards/rejected": -5.125,
      "step": 2420
    },
    {
      "epoch": 0.1754132678842128,
      "grad_norm": 11.816715024101674,
      "learning_rate": 4.913983740864473e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.09375,
      "logps/chosen": -446.0,
      "logps/rejected": -664.0,
      "loss": 0.2745,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.421875,
      "rewards/margins": 2.234375,
      "rewards/rejected": -4.65625,
      "step": 2430
    },
    {
      "epoch": 0.17613513318414784,
      "grad_norm": 12.069738630529791,
      "learning_rate": 4.91233777207431e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.1875,
      "logps/chosen": -438.0,
      "logps/rejected": -672.0,
      "loss": 0.2964,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.625,
      "rewards/margins": 2.390625,
      "rewards/rejected": -5.03125,
      "step": 2440
    },
    {
      "epoch": 0.17685699848408287,
      "grad_norm": 14.061779566229111,
      "learning_rate": 4.910676484873292e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.359375,
      "logps/chosen": -446.0,
      "logps/rejected": -672.0,
      "loss": 0.2942,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.515625,
      "rewards/margins": 2.4375,
      "rewards/rejected": -4.9375,
      "step": 2450
    },
    {
      "epoch": 0.1775788637840179,
      "grad_norm": 13.782201199172336,
      "learning_rate": 4.908999889810633e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.203125,
      "logps/chosen": -448.0,
      "logps/rejected": -700.0,
      "loss": 0.2754,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.53125,
      "rewards/margins": 2.65625,
      "rewards/rejected": -5.1875,
      "step": 2460
    },
    {
      "epoch": 0.17830072908395295,
      "grad_norm": 14.155628793519401,
      "learning_rate": 4.907307997532761e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.4375,
      "logps/chosen": -466.0,
      "logps/rejected": -768.0,
      "loss": 0.2606,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.75,
      "rewards/margins": 2.984375,
      "rewards/rejected": -5.75,
      "step": 2470
    },
    {
      "epoch": 0.17902259438388796,
      "grad_norm": 11.099722888021903,
      "learning_rate": 4.905600818783237e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.296875,
      "logps/chosen": -502.0,
      "logps/rejected": -748.0,
      "loss": 0.2843,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.890625,
      "rewards/margins": 2.875,
      "rewards/rejected": -5.78125,
      "step": 2480
    },
    {
      "epoch": 0.179744459683823,
      "grad_norm": 9.803506633400223,
      "learning_rate": 4.903878364402691e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.34375,
      "logps/chosen": -444.0,
      "logps/rejected": -672.0,
      "loss": 0.2774,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.421875,
      "rewards/margins": 2.609375,
      "rewards/rejected": -5.03125,
      "step": 2490
    },
    {
      "epoch": 0.18046632498375803,
      "grad_norm": 10.846425928819453,
      "learning_rate": 4.902140645328759e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.296875,
      "logps/chosen": -412.0,
      "logps/rejected": -652.0,
      "loss": 0.2831,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.390625,
      "rewards/margins": 2.546875,
      "rewards/rejected": -4.9375,
      "step": 2500
    },
    {
      "epoch": 0.18118819028369307,
      "grad_norm": 16.759887774805073,
      "learning_rate": 4.900387672596003e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.34375,
      "logps/chosen": -446.0,
      "logps/rejected": -700.0,
      "loss": 0.2681,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.578125,
      "rewards/margins": 2.6875,
      "rewards/rejected": -5.25,
      "step": 2510
    },
    {
      "epoch": 0.1819100555836281,
      "grad_norm": 14.68071330748384,
      "learning_rate": 4.898619457335848e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -2.265625,
      "logps/chosen": -458.0,
      "logps/rejected": -684.0,
      "loss": 0.2771,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.625,
      "rewards/margins": 2.46875,
      "rewards/rejected": -5.09375,
      "step": 2520
    },
    {
      "epoch": 0.18263192088356311,
      "grad_norm": 14.4018067051319,
      "learning_rate": 4.896836010776509e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.375,
      "logps/chosen": -448.0,
      "logps/rejected": -756.0,
      "loss": 0.2807,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.53125,
      "rewards/margins": 3.0625,
      "rewards/rejected": -5.59375,
      "step": 2530
    },
    {
      "epoch": 0.18335378618349815,
      "grad_norm": 10.743360547245137,
      "learning_rate": 4.895037344242921e-07,
      "logits/chosen": -2.25,
      "logits/rejected": -2.375,
      "logps/chosen": -450.0,
      "logps/rejected": -688.0,
      "loss": 0.2867,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.578125,
      "rewards/margins": 2.5625,
      "rewards/rejected": -5.15625,
      "step": 2540
    },
    {
      "epoch": 0.1840756514834332,
      "grad_norm": 8.989162171013234,
      "learning_rate": 4.893223469156664e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.296875,
      "logps/chosen": -454.0,
      "logps/rejected": -676.0,
      "loss": 0.2615,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.609375,
      "rewards/margins": 2.3125,
      "rewards/rejected": -4.9375,
      "step": 2550
    },
    {
      "epoch": 0.18479751678336823,
      "grad_norm": 15.99275201538434,
      "learning_rate": 4.891394397035896e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.3125,
      "logps/chosen": -480.0,
      "logps/rejected": -696.0,
      "loss": 0.2767,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.90625,
      "rewards/margins": 2.328125,
      "rewards/rejected": -5.25,
      "step": 2560
    },
    {
      "epoch": 0.18551938208330326,
      "grad_norm": 9.200278835722967,
      "learning_rate": 4.889550139495275e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.375,
      "logps/chosen": -428.0,
      "logps/rejected": -716.0,
      "loss": 0.2441,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.390625,
      "rewards/margins": 2.984375,
      "rewards/rejected": -5.375,
      "step": 2570
    },
    {
      "epoch": 0.1862412473832383,
      "grad_norm": 10.40828341000086,
      "learning_rate": 4.887690708245887e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.265625,
      "logps/chosen": -412.0,
      "logps/rejected": -692.0,
      "loss": 0.2627,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.1875,
      "rewards/margins": 2.78125,
      "rewards/rejected": -4.96875,
      "step": 2580
    },
    {
      "epoch": 0.1869631126831733,
      "grad_norm": 12.947016795780353,
      "learning_rate": 4.885816115095171e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.3125,
      "logps/chosen": -442.0,
      "logps/rejected": -768.0,
      "loss": 0.2853,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.53125,
      "rewards/margins": 3.1875,
      "rewards/rejected": -5.71875,
      "step": 2590
    },
    {
      "epoch": 0.18768497798310835,
      "grad_norm": 10.213794299370996,
      "learning_rate": 4.883926371946843e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.296875,
      "logps/chosen": -470.0,
      "logps/rejected": -716.0,
      "loss": 0.2972,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.75,
      "rewards/margins": 2.515625,
      "rewards/rejected": -5.28125,
      "step": 2600
    },
    {
      "epoch": 0.18840684328304338,
      "grad_norm": 8.748591180432715,
      "learning_rate": 4.882021490800826e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.234375,
      "logps/chosen": -422.0,
      "logps/rejected": -620.0,
      "loss": 0.2745,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.21875,
      "rewards/margins": 2.09375,
      "rewards/rejected": -4.3125,
      "step": 2610
    },
    {
      "epoch": 0.18912870858297842,
      "grad_norm": 14.080860793036742,
      "learning_rate": 4.880101483753167e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.09375,
      "logps/chosen": -412.0,
      "logps/rejected": -700.0,
      "loss": 0.2908,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.4375,
      "rewards/margins": 2.6875,
      "rewards/rejected": -5.125,
      "step": 2620
    },
    {
      "epoch": 0.18985057388291346,
      "grad_norm": 10.721396459348382,
      "learning_rate": 4.878166362995963e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.28125,
      "logps/chosen": -418.0,
      "logps/rejected": -668.0,
      "loss": 0.3121,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.234375,
      "rewards/margins": 2.4375,
      "rewards/rejected": -4.6875,
      "step": 2630
    },
    {
      "epoch": 0.19057243918284847,
      "grad_norm": 12.989225976467061,
      "learning_rate": 4.876216140817285e-07,
      "logits/chosen": -2.25,
      "logits/rejected": -2.15625,
      "logps/chosen": -424.0,
      "logps/rejected": -656.0,
      "loss": 0.2783,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.328125,
      "rewards/margins": 2.375,
      "rewards/rejected": -4.6875,
      "step": 2640
    },
    {
      "epoch": 0.1912943044827835,
      "grad_norm": 11.698899032693152,
      "learning_rate": 4.874250829601094e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.390625,
      "logps/chosen": -496.0,
      "logps/rejected": -712.0,
      "loss": 0.2471,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.765625,
      "rewards/margins": 2.453125,
      "rewards/rejected": -5.21875,
      "step": 2650
    },
    {
      "epoch": 0.19201616978271854,
      "grad_norm": 11.56011926756224,
      "learning_rate": 4.872270441827174e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.171875,
      "logps/chosen": -460.0,
      "logps/rejected": -700.0,
      "loss": 0.2703,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.625,
      "rewards/margins": 2.515625,
      "rewards/rejected": -5.15625,
      "step": 2660
    },
    {
      "epoch": 0.19273803508265358,
      "grad_norm": 11.399494678887148,
      "learning_rate": 4.870274990071038e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.390625,
      "logps/chosen": -400.0,
      "logps/rejected": -636.0,
      "loss": 0.2763,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.328125,
      "rewards/margins": 2.453125,
      "rewards/rejected": -4.78125,
      "step": 2670
    },
    {
      "epoch": 0.19345990038258862,
      "grad_norm": 9.260136695607668,
      "learning_rate": 4.868264487003862e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.1875,
      "logps/chosen": -440.0,
      "logps/rejected": -732.0,
      "loss": 0.2773,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.5625,
      "rewards/margins": 2.828125,
      "rewards/rejected": -5.40625,
      "step": 2680
    },
    {
      "epoch": 0.19418176568252365,
      "grad_norm": 9.99668184202405,
      "learning_rate": 4.866238945392393e-07,
      "logits/chosen": -2.25,
      "logits/rejected": -2.15625,
      "logps/chosen": -442.0,
      "logps/rejected": -664.0,
      "loss": 0.2767,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.484375,
      "rewards/margins": 2.5,
      "rewards/rejected": -4.96875,
      "step": 2690
    },
    {
      "epoch": 0.19490363098245866,
      "grad_norm": 9.88609667677639,
      "learning_rate": 4.864198378098877e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.28125,
      "logps/chosen": -414.0,
      "logps/rejected": -708.0,
      "loss": 0.2571,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.296875,
      "rewards/margins": 3.0,
      "rewards/rejected": -5.3125,
      "step": 2700
    },
    {
      "epoch": 0.1956254962823937,
      "grad_norm": 15.497949591327552,
      "learning_rate": 4.862142798080973e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.359375,
      "logps/chosen": -466.0,
      "logps/rejected": -680.0,
      "loss": 0.2718,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.828125,
      "rewards/margins": 2.375,
      "rewards/rejected": -5.21875,
      "step": 2710
    },
    {
      "epoch": 0.19634736158232874,
      "grad_norm": 8.87107110055295,
      "learning_rate": 4.860072218391669e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -2.328125,
      "logps/chosen": -434.0,
      "logps/rejected": -720.0,
      "loss": 0.2514,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.453125,
      "rewards/margins": 2.984375,
      "rewards/rejected": -5.4375,
      "step": 2720
    },
    {
      "epoch": 0.19706922688226378,
      "grad_norm": 9.881490353848653,
      "learning_rate": 4.857986652179203e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.28125,
      "logps/chosen": -486.0,
      "logps/rejected": -752.0,
      "loss": 0.2743,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.796875,
      "rewards/margins": 2.703125,
      "rewards/rejected": -5.5,
      "step": 2730
    },
    {
      "epoch": 0.1977910921821988,
      "grad_norm": 11.801073627615567,
      "learning_rate": 4.855886112686977e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.421875,
      "logps/chosen": -448.0,
      "logps/rejected": -728.0,
      "loss": 0.2397,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.5625,
      "rewards/margins": 2.84375,
      "rewards/rejected": -5.40625,
      "step": 2740
    },
    {
      "epoch": 0.19851295748213382,
      "grad_norm": 11.007267794347985,
      "learning_rate": 4.853770613253476e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.421875,
      "logps/chosen": -452.0,
      "logps/rejected": -712.0,
      "loss": 0.2655,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.765625,
      "rewards/margins": 2.84375,
      "rewards/rejected": -5.59375,
      "step": 2750
    },
    {
      "epoch": 0.19923482278206886,
      "grad_norm": 11.435358287783314,
      "learning_rate": 4.851640167312178e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.53125,
      "logps/chosen": -462.0,
      "logps/rejected": -728.0,
      "loss": 0.2544,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.703125,
      "rewards/margins": 2.65625,
      "rewards/rejected": -5.375,
      "step": 2760
    },
    {
      "epoch": 0.1999566880820039,
      "grad_norm": 12.377882031181327,
      "learning_rate": 4.849494788391473e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.265625,
      "logps/chosen": -466.0,
      "logps/rejected": -760.0,
      "loss": 0.2605,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.046875,
      "rewards/rejected": -5.8125,
      "step": 2770
    },
    {
      "epoch": 0.20067855338193893,
      "grad_norm": 13.770666426550322,
      "learning_rate": 4.847334490114576e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.578125,
      "logps/chosen": -450.0,
      "logps/rejected": -696.0,
      "loss": 0.2616,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.703125,
      "rewards/margins": 2.65625,
      "rewards/rejected": -5.375,
      "step": 2780
    },
    {
      "epoch": 0.20140041868187397,
      "grad_norm": 9.391404172910784,
      "learning_rate": 4.84515928619944e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.375,
      "logps/chosen": -466.0,
      "logps/rejected": -776.0,
      "loss": 0.2496,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.703125,
      "rewards/margins": 3.09375,
      "rewards/rejected": -5.78125,
      "step": 2790
    },
    {
      "epoch": 0.20212228398180898,
      "grad_norm": 14.686701862821584,
      "learning_rate": 4.84296919045867e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.265625,
      "logps/chosen": -438.0,
      "logps/rejected": -732.0,
      "loss": 0.2628,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.46875,
      "rewards/margins": 2.734375,
      "rewards/rejected": -5.1875,
      "step": 2800
    },
    {
      "epoch": 0.20284414928174402,
      "grad_norm": 11.490603703306945,
      "learning_rate": 4.840764216799433e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.09375,
      "logps/chosen": -438.0,
      "logps/rejected": -700.0,
      "loss": 0.2829,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.453125,
      "rewards/margins": 2.703125,
      "rewards/rejected": -5.15625,
      "step": 2810
    },
    {
      "epoch": 0.20356601458167906,
      "grad_norm": 13.312504542654146,
      "learning_rate": 4.838544379223373e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.46875,
      "logps/chosen": -464.0,
      "logps/rejected": -716.0,
      "loss": 0.2977,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.703125,
      "rewards/margins": 2.6875,
      "rewards/rejected": -5.40625,
      "step": 2820
    },
    {
      "epoch": 0.2042878798816141,
      "grad_norm": 12.670987065802498,
      "learning_rate": 4.836309691826518e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.59375,
      "logps/chosen": -442.0,
      "logps/rejected": -712.0,
      "loss": 0.2934,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.640625,
      "rewards/margins": 2.75,
      "rewards/rejected": -5.375,
      "step": 2830
    },
    {
      "epoch": 0.20500974518154913,
      "grad_norm": 11.60451245846052,
      "learning_rate": 4.834060168799195e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.25,
      "logps/chosen": -462.0,
      "logps/rejected": -672.0,
      "loss": 0.2723,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.609375,
      "rewards/margins": 2.359375,
      "rewards/rejected": -4.96875,
      "step": 2840
    },
    {
      "epoch": 0.20573161048148417,
      "grad_norm": 10.084682945653388,
      "learning_rate": 4.831795824425938e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.46875,
      "logps/chosen": -458.0,
      "logps/rejected": -688.0,
      "loss": 0.2875,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.59375,
      "rewards/margins": 2.53125,
      "rewards/rejected": -5.125,
      "step": 2850
    },
    {
      "epoch": 0.20645347578141918,
      "grad_norm": 17.15048261659515,
      "learning_rate": 4.829516673085394e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.46875,
      "logps/chosen": -464.0,
      "logps/rejected": -736.0,
      "loss": 0.2552,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.734375,
      "rewards/margins": 2.796875,
      "rewards/rejected": -5.53125,
      "step": 2860
    },
    {
      "epoch": 0.20717534108135421,
      "grad_norm": 12.469884733860969,
      "learning_rate": 4.827222729250236e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.359375,
      "logps/chosen": -452.0,
      "logps/rejected": -692.0,
      "loss": 0.2826,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.5,
      "rewards/margins": 2.71875,
      "rewards/rejected": -5.21875,
      "step": 2870
    },
    {
      "epoch": 0.20789720638128925,
      "grad_norm": 9.185412944423907,
      "learning_rate": 4.824914007487072e-07,
      "logits/chosen": -2.25,
      "logits/rejected": -2.328125,
      "logps/chosen": -416.0,
      "logps/rejected": -680.0,
      "loss": 0.2466,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.359375,
      "rewards/margins": 2.609375,
      "rewards/rejected": -4.96875,
      "step": 2880
    },
    {
      "epoch": 0.2086190716812243,
      "grad_norm": 9.763909974617468,
      "learning_rate": 4.822590522456347e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.375,
      "logps/chosen": -448.0,
      "logps/rejected": -684.0,
      "loss": 0.2761,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.609375,
      "rewards/margins": 2.5625,
      "rewards/rejected": -5.15625,
      "step": 2890
    },
    {
      "epoch": 0.20934093698115933,
      "grad_norm": 8.838099413342011,
      "learning_rate": 4.820252288912254e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.609375,
      "logps/chosen": -464.0,
      "logps/rejected": -680.0,
      "loss": 0.2509,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.6875,
      "rewards/margins": 2.3125,
      "rewards/rejected": -5.0,
      "step": 2900
    },
    {
      "epoch": 0.21006280228109434,
      "grad_norm": 8.648328885741636,
      "learning_rate": 4.817899321702642e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.546875,
      "logps/chosen": -450.0,
      "logps/rejected": -720.0,
      "loss": 0.259,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.375,
      "rewards/margins": 3.03125,
      "rewards/rejected": -5.40625,
      "step": 2910
    },
    {
      "epoch": 0.21078466758102937,
      "grad_norm": 16.897919304351465,
      "learning_rate": 4.815531635768916e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.484375,
      "logps/chosen": -448.0,
      "logps/rejected": -748.0,
      "loss": 0.2633,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.578125,
      "rewards/margins": 2.9375,
      "rewards/rejected": -5.5,
      "step": 2920
    },
    {
      "epoch": 0.2115065328809644,
      "grad_norm": 14.689397820328814,
      "learning_rate": 4.813149246145946e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.375,
      "logps/chosen": -432.0,
      "logps/rejected": -712.0,
      "loss": 0.2578,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.4375,
      "rewards/margins": 2.90625,
      "rewards/rejected": -5.34375,
      "step": 2930
    },
    {
      "epoch": 0.21222839818089945,
      "grad_norm": 10.03106976493699,
      "learning_rate": 4.810752167961971e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.5,
      "logps/chosen": -440.0,
      "logps/rejected": -696.0,
      "loss": 0.2584,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.578125,
      "rewards/margins": 2.5,
      "rewards/rejected": -5.09375,
      "step": 2940
    },
    {
      "epoch": 0.21295026348083448,
      "grad_norm": 8.043227016631931,
      "learning_rate": 4.808340416438505e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.546875,
      "logps/chosen": -462.0,
      "logps/rejected": -680.0,
      "loss": 0.2637,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.765625,
      "rewards/margins": 2.484375,
      "rewards/rejected": -5.25,
      "step": 2950
    },
    {
      "epoch": 0.21367212878076952,
      "grad_norm": 12.501105696870356,
      "learning_rate": 4.805914006890234e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.671875,
      "logps/chosen": -480.0,
      "logps/rejected": -740.0,
      "loss": 0.2703,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.796875,
      "rewards/margins": 2.734375,
      "rewards/rejected": -5.53125,
      "step": 2960
    },
    {
      "epoch": 0.21439399408070453,
      "grad_norm": 11.254651074225874,
      "learning_rate": 4.803472954724928e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.65625,
      "logps/chosen": -458.0,
      "logps/rejected": -684.0,
      "loss": 0.263,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.71875,
      "rewards/margins": 2.4375,
      "rewards/rejected": -5.15625,
      "step": 2970
    },
    {
      "epoch": 0.21511585938063957,
      "grad_norm": 18.241498797423027,
      "learning_rate": 4.801017275443331e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.59375,
      "logps/chosen": -472.0,
      "logps/rejected": -732.0,
      "loss": 0.2789,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.828125,
      "rewards/margins": 2.796875,
      "rewards/rejected": -5.625,
      "step": 2980
    },
    {
      "epoch": 0.2158377246805746,
      "grad_norm": 13.440869821315093,
      "learning_rate": 4.798546984639076e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.484375,
      "logps/chosen": -446.0,
      "logps/rejected": -716.0,
      "loss": 0.2514,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.59375,
      "rewards/margins": 2.796875,
      "rewards/rejected": -5.375,
      "step": 2990
    },
    {
      "epoch": 0.21655958998050964,
      "grad_norm": 14.300279918623843,
      "learning_rate": 4.796062097998578e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.515625,
      "logps/chosen": -474.0,
      "logps/rejected": -732.0,
      "loss": 0.2699,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.765625,
      "rewards/margins": 2.828125,
      "rewards/rejected": -5.59375,
      "step": 3000
    },
    {
      "epoch": 0.21728145528044468,
      "grad_norm": 19.906489309756488,
      "learning_rate": 4.793562631300932e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.53125,
      "logps/chosen": -490.0,
      "logps/rejected": -708.0,
      "loss": 0.2565,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.015625,
      "rewards/margins": 2.265625,
      "rewards/rejected": -5.28125,
      "step": 3010
    },
    {
      "epoch": 0.2180033205803797,
      "grad_norm": 10.531191648041075,
      "learning_rate": 4.791048600417824e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.546875,
      "logps/chosen": -442.0,
      "logps/rejected": -716.0,
      "loss": 0.2671,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.59375,
      "rewards/margins": 2.875,
      "rewards/rejected": -5.46875,
      "step": 3020
    },
    {
      "epoch": 0.21872518588031473,
      "grad_norm": 12.443889254173175,
      "learning_rate": 4.788520021313416e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.53125,
      "logps/chosen": -456.0,
      "logps/rejected": -724.0,
      "loss": 0.2409,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.625,
      "rewards/margins": 2.75,
      "rewards/rejected": -5.375,
      "step": 3030
    },
    {
      "epoch": 0.21944705118024976,
      "grad_norm": 12.42319433769227,
      "learning_rate": 4.785976910044255e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.703125,
      "logps/chosen": -470.0,
      "logps/rejected": -756.0,
      "loss": 0.2335,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.125,
      "rewards/rejected": -5.9375,
      "step": 3040
    },
    {
      "epoch": 0.2201689164801848,
      "grad_norm": 13.777766110799076,
      "learning_rate": 4.783419282759168e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.390625,
      "logps/chosen": -472.0,
      "logps/rejected": -724.0,
      "loss": 0.2581,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.8125,
      "rewards/margins": 2.765625,
      "rewards/rejected": -5.5625,
      "step": 3050
    },
    {
      "epoch": 0.22089078178011984,
      "grad_norm": 11.358944857031117,
      "learning_rate": 4.780847155699157e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.4375,
      "logps/chosen": -468.0,
      "logps/rejected": -724.0,
      "loss": 0.2492,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.75,
      "rewards/margins": 2.71875,
      "rewards/rejected": -5.46875,
      "step": 3060
    },
    {
      "epoch": 0.22161264708005488,
      "grad_norm": 16.10778837148524,
      "learning_rate": 4.778260545197301e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.5625,
      "logps/chosen": -472.0,
      "logps/rejected": -724.0,
      "loss": 0.2676,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.9375,
      "rewards/margins": 2.625,
      "rewards/rejected": -5.5625,
      "step": 3070
    },
    {
      "epoch": 0.22233451237998988,
      "grad_norm": 11.746080585673342,
      "learning_rate": 4.775659467678645e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.671875,
      "logps/chosen": -446.0,
      "logps/rejected": -688.0,
      "loss": 0.257,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.53125,
      "rewards/margins": 2.6875,
      "rewards/rejected": -5.21875,
      "step": 3080
    },
    {
      "epoch": 0.22305637767992492,
      "grad_norm": 15.325381253397913,
      "learning_rate": 4.773043939660105e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.609375,
      "logps/chosen": -458.0,
      "logps/rejected": -700.0,
      "loss": 0.2715,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.640625,
      "rewards/margins": 2.578125,
      "rewards/rejected": -5.21875,
      "step": 3090
    },
    {
      "epoch": 0.22377824297985996,
      "grad_norm": 18.599645534716352,
      "learning_rate": 4.770413977750353e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.609375,
      "logps/chosen": -496.0,
      "logps/rejected": -764.0,
      "loss": 0.2536,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.96875,
      "rewards/margins": 2.890625,
      "rewards/rejected": -5.875,
      "step": 3100
    },
    {
      "epoch": 0.224500108279795,
      "grad_norm": 8.96291689253792,
      "learning_rate": 4.7677695986497225e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -482.0,
      "logps/rejected": -720.0,
      "loss": 0.2748,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.765625,
      "rewards/margins": 2.53125,
      "rewards/rejected": -5.28125,
      "step": 3110
    },
    {
      "epoch": 0.22522197357973003,
      "grad_norm": 10.362651250346739,
      "learning_rate": 4.7651108191500896e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.375,
      "logps/chosen": -466.0,
      "logps/rejected": -732.0,
      "loss": 0.2502,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.578125,
      "rewards/margins": 2.765625,
      "rewards/rejected": -5.34375,
      "step": 3120
    },
    {
      "epoch": 0.22594383887966504,
      "grad_norm": 9.5061194727635,
      "learning_rate": 4.762437656134778e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.71875,
      "logps/chosen": -454.0,
      "logps/rejected": -696.0,
      "loss": 0.2597,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.59375,
      "rewards/margins": 2.65625,
      "rewards/rejected": -5.25,
      "step": 3130
    },
    {
      "epoch": 0.22666570417960008,
      "grad_norm": 15.379283168608554,
      "learning_rate": 4.7597501265784466e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.578125,
      "logps/chosen": -496.0,
      "logps/rejected": -736.0,
      "loss": 0.2471,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.078125,
      "rewards/margins": 2.515625,
      "rewards/rejected": -5.59375,
      "step": 3140
    },
    {
      "epoch": 0.22738756947953512,
      "grad_norm": 13.207358353876185,
      "learning_rate": 4.757048247546982e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.609375,
      "logps/chosen": -442.0,
      "logps/rejected": -692.0,
      "loss": 0.2752,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.671875,
      "rewards/margins": 2.546875,
      "rewards/rejected": -5.21875,
      "step": 3150
    },
    {
      "epoch": 0.22810943477947015,
      "grad_norm": 12.680159027731989,
      "learning_rate": 4.7543320361973884e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.578125,
      "logps/chosen": -422.0,
      "logps/rejected": -668.0,
      "loss": 0.2534,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.40625,
      "rewards/margins": 2.625,
      "rewards/rejected": -5.03125,
      "step": 3160
    },
    {
      "epoch": 0.2288313000794052,
      "grad_norm": 14.16757412808312,
      "learning_rate": 4.751601509777683e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.765625,
      "logps/chosen": -456.0,
      "logps/rejected": -760.0,
      "loss": 0.2435,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.765625,
      "rewards/margins": 2.96875,
      "rewards/rejected": -5.75,
      "step": 3170
    },
    {
      "epoch": 0.2295531653793402,
      "grad_norm": 16.953418019709876,
      "learning_rate": 4.748856685626784e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.671875,
      "logps/chosen": -488.0,
      "logps/rejected": -772.0,
      "loss": 0.2663,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.890625,
      "rewards/margins": 2.96875,
      "rewards/rejected": -5.84375,
      "step": 3180
    },
    {
      "epoch": 0.23027503067927524,
      "grad_norm": 8.905205262666545,
      "learning_rate": 4.7460975811743986e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5,
      "logps/chosen": -484.0,
      "logps/rejected": -744.0,
      "loss": 0.2483,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.78125,
      "rewards/margins": 2.78125,
      "rewards/rejected": -5.5625,
      "step": 3190
    },
    {
      "epoch": 0.23099689597921028,
      "grad_norm": 12.494067533198487,
      "learning_rate": 4.743324213940917e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.8125,
      "logps/chosen": -480.0,
      "logps/rejected": -732.0,
      "loss": 0.2617,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 2.640625,
      "rewards/rejected": -5.5625,
      "step": 3200
    },
    {
      "epoch": 0.2317187612791453,
      "grad_norm": 11.684223200632987,
      "learning_rate": 4.740536601537295e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.828125,
      "logps/chosen": -506.0,
      "logps/rejected": -736.0,
      "loss": 0.2722,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.0625,
      "rewards/margins": 2.484375,
      "rewards/rejected": -5.53125,
      "step": 3210
    },
    {
      "epoch": 0.23244062657908035,
      "grad_norm": 13.4382238542573,
      "learning_rate": 4.73773476166495e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.71875,
      "logps/chosen": -474.0,
      "logps/rejected": -724.0,
      "loss": 0.2388,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.78125,
      "rewards/margins": 2.671875,
      "rewards/rejected": -5.4375,
      "step": 3220
    },
    {
      "epoch": 0.2331624918790154,
      "grad_norm": 11.637178803211128,
      "learning_rate": 4.73491871211564e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -484.0,
      "logps/rejected": -776.0,
      "loss": 0.2568,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.015625,
      "rewards/rejected": -5.90625,
      "step": 3230
    },
    {
      "epoch": 0.2338843571789504,
      "grad_norm": 12.359492405437395,
      "learning_rate": 4.7320884707713573e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.546875,
      "logps/chosen": -456.0,
      "logps/rejected": -768.0,
      "loss": 0.2387,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.6875,
      "rewards/margins": 3.1875,
      "rewards/rejected": -5.875,
      "step": 3240
    },
    {
      "epoch": 0.23460622247888543,
      "grad_norm": 11.807012351185746,
      "learning_rate": 4.7292440556042116e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.78125,
      "logps/chosen": -442.0,
      "logps/rejected": -760.0,
      "loss": 0.2616,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.671875,
      "rewards/margins": 3.296875,
      "rewards/rejected": -5.96875,
      "step": 3250
    },
    {
      "epoch": 0.23532808777882047,
      "grad_norm": 10.114982406199337,
      "learning_rate": 4.726385484676318e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.6875,
      "logps/chosen": -430.0,
      "logps/rejected": -676.0,
      "loss": 0.2583,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.453125,
      "rewards/margins": 2.5625,
      "rewards/rejected": -5.0,
      "step": 3260
    },
    {
      "epoch": 0.2360499530787555,
      "grad_norm": 13.86470772911654,
      "learning_rate": 4.723512776139679e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.75,
      "logps/chosen": -426.0,
      "logps/rejected": -672.0,
      "loss": 0.2676,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.625,
      "rewards/margins": 2.328125,
      "rewards/rejected": -4.96875,
      "step": 3270
    },
    {
      "epoch": 0.23677181837869055,
      "grad_norm": 10.771417577248654,
      "learning_rate": 4.7206259482360734e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.71875,
      "logps/chosen": -440.0,
      "logps/rejected": -672.0,
      "loss": 0.2554,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.484375,
      "rewards/margins": 2.5,
      "rewards/rejected": -5.0,
      "step": 3280
    },
    {
      "epoch": 0.23749368367862556,
      "grad_norm": 10.50765536959762,
      "learning_rate": 4.7177250192969364e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.5625,
      "logps/chosen": -452.0,
      "logps/rejected": -708.0,
      "loss": 0.2404,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.5,
      "rewards/margins": 2.78125,
      "rewards/rejected": -5.28125,
      "step": 3290
    },
    {
      "epoch": 0.2382155489785606,
      "grad_norm": 12.675349969572869,
      "learning_rate": 4.714810007743247e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.71875,
      "logps/chosen": -480.0,
      "logps/rejected": -736.0,
      "loss": 0.2538,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.78125,
      "rewards/margins": 2.84375,
      "rewards/rejected": -5.625,
      "step": 3300
    },
    {
      "epoch": 0.23893741427849563,
      "grad_norm": 11.128989432844534,
      "learning_rate": 4.7118809320854077e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.796875,
      "logps/chosen": -438.0,
      "logps/rejected": -716.0,
      "loss": 0.2453,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.625,
      "rewards/margins": 2.96875,
      "rewards/rejected": -5.59375,
      "step": 3310
    },
    {
      "epoch": 0.23965927957843067,
      "grad_norm": 15.178248588954864,
      "learning_rate": 4.7089378109231294e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.859375,
      "logps/chosen": -496.0,
      "logps/rejected": -780.0,
      "loss": 0.2622,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.171875,
      "rewards/margins": 2.828125,
      "rewards/rejected": -6.0,
      "step": 3320
    },
    {
      "epoch": 0.2403811448783657,
      "grad_norm": 9.996885840127206,
      "learning_rate": 4.7059806629453116e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.90625,
      "logps/chosen": -472.0,
      "logps/rejected": -724.0,
      "loss": 0.2543,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.671875,
      "rewards/margins": 2.8125,
      "rewards/rejected": -5.46875,
      "step": 3330
    },
    {
      "epoch": 0.24110301017830074,
      "grad_norm": 10.262318540512673,
      "learning_rate": 4.7030095069299257e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.71875,
      "logps/chosen": -496.0,
      "logps/rejected": -764.0,
      "loss": 0.2229,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.921875,
      "rewards/margins": 2.859375,
      "rewards/rejected": -5.78125,
      "step": 3340
    },
    {
      "epoch": 0.24182487547823575,
      "grad_norm": 11.012162462170245,
      "learning_rate": 4.700024361743893e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -3.03125,
      "logps/chosen": -480.0,
      "logps/rejected": -768.0,
      "loss": 0.2384,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.046875,
      "rewards/rejected": -5.96875,
      "step": 3350
    },
    {
      "epoch": 0.2425467407781708,
      "grad_norm": 12.109737067913255,
      "learning_rate": 4.697025246342967e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.8125,
      "logps/chosen": -460.0,
      "logps/rejected": -708.0,
      "loss": 0.2369,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.765625,
      "rewards/margins": 2.8125,
      "rewards/rejected": -5.5625,
      "step": 3360
    },
    {
      "epoch": 0.24326860607810583,
      "grad_norm": 15.032805570285488,
      "learning_rate": 4.6940121797716127e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.796875,
      "logps/chosen": -472.0,
      "logps/rejected": -764.0,
      "loss": 0.2488,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.03125,
      "rewards/rejected": -5.90625,
      "step": 3370
    },
    {
      "epoch": 0.24399047137804086,
      "grad_norm": 14.74411971460415,
      "learning_rate": 4.6909851811628853e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.9375,
      "logps/chosen": -504.0,
      "logps/rejected": -784.0,
      "loss": 0.2222,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.03125,
      "rewards/rejected": -6.125,
      "step": 3380
    },
    {
      "epoch": 0.2447123366779759,
      "grad_norm": 10.795391451105882,
      "learning_rate": 4.68794426973831e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.78125,
      "logps/chosen": -472.0,
      "logps/rejected": -768.0,
      "loss": 0.2591,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.03125,
      "rewards/rejected": -5.875,
      "step": 3390
    },
    {
      "epoch": 0.2454342019779109,
      "grad_norm": 9.935149455720076,
      "learning_rate": 4.684889464807755e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.75,
      "logps/chosen": -460.0,
      "logps/rejected": -756.0,
      "loss": 0.2455,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.734375,
      "rewards/margins": 2.890625,
      "rewards/rejected": -5.625,
      "step": 3400
    },
    {
      "epoch": 0.24615606727784595,
      "grad_norm": 10.451118195815205,
      "learning_rate": 4.681820785769317e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.734375,
      "logps/chosen": -478.0,
      "logps/rejected": -740.0,
      "loss": 0.2239,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.796875,
      "rewards/margins": 2.890625,
      "rewards/rejected": -5.6875,
      "step": 3410
    },
    {
      "epoch": 0.24687793257778098,
      "grad_norm": 12.213445541387253,
      "learning_rate": 4.678738252109192e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.78125,
      "logps/chosen": -476.0,
      "logps/rejected": -744.0,
      "loss": 0.2568,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 2.828125,
      "rewards/rejected": -5.75,
      "step": 3420
    },
    {
      "epoch": 0.24759979787771602,
      "grad_norm": 13.253740393281229,
      "learning_rate": 4.675641883401553e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.625,
      "logps/chosen": -470.0,
      "logps/rejected": -760.0,
      "loss": 0.2457,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.796875,
      "rewards/margins": 2.984375,
      "rewards/rejected": -5.78125,
      "step": 3430
    },
    {
      "epoch": 0.24832166317765106,
      "grad_norm": 10.595762189942786,
      "learning_rate": 4.672531699308425e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.734375,
      "logps/chosen": -462.0,
      "logps/rejected": -748.0,
      "loss": 0.2527,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.875,
      "rewards/margins": 2.90625,
      "rewards/rejected": -5.78125,
      "step": 3440
    },
    {
      "epoch": 0.2490435284775861,
      "grad_norm": 9.493996818423415,
      "learning_rate": 4.669407719579562e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.8125,
      "logps/chosen": -476.0,
      "logps/rejected": -712.0,
      "loss": 0.2569,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.734375,
      "rewards/margins": 2.65625,
      "rewards/rejected": -5.40625,
      "step": 3450
    },
    {
      "epoch": 0.2497653937775211,
      "grad_norm": 11.78357170396975,
      "learning_rate": 4.666269964052322e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.796875,
      "logps/chosen": -480.0,
      "logps/rejected": -760.0,
      "loss": 0.2407,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.78125,
      "rewards/margins": 3.09375,
      "rewards/rejected": -5.875,
      "step": 3460
    },
    {
      "epoch": 0.25048725907745617,
      "grad_norm": 9.998441634418365,
      "learning_rate": 4.6631184526515384e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.953125,
      "logps/chosen": -494.0,
      "logps/rejected": -756.0,
      "loss": 0.2468,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.96875,
      "rewards/margins": 2.796875,
      "rewards/rejected": -5.75,
      "step": 3470
    },
    {
      "epoch": 0.2512091243773912,
      "grad_norm": 13.711576277859297,
      "learning_rate": 4.6599532053893936e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.796875,
      "logps/chosen": -474.0,
      "logps/rejected": -712.0,
      "loss": 0.2435,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.796875,
      "rewards/margins": 2.53125,
      "rewards/rejected": -5.3125,
      "step": 3480
    },
    {
      "epoch": 0.2519309896773262,
      "grad_norm": 8.910619779758456,
      "learning_rate": 4.6567742423652955e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.796875,
      "logps/chosen": -488.0,
      "logps/rejected": -744.0,
      "loss": 0.2389,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.921875,
      "rewards/margins": 2.734375,
      "rewards/rejected": -5.65625,
      "step": 3490
    },
    {
      "epoch": 0.25265285497726125,
      "grad_norm": 9.203688078434835,
      "learning_rate": 4.6535815837657456e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.75,
      "logps/chosen": -482.0,
      "logps/rejected": -760.0,
      "loss": 0.2433,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.859375,
      "rewards/margins": 2.90625,
      "rewards/rejected": -5.75,
      "step": 3500
    },
    {
      "epoch": 0.25337472027719626,
      "grad_norm": 12.187312607080647,
      "learning_rate": 4.6503752498642133e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -496.0,
      "logps/rejected": -732.0,
      "loss": 0.243,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 2.59375,
      "rewards/rejected": -5.6875,
      "step": 3510
    },
    {
      "epoch": 0.25409658557713133,
      "grad_norm": 15.103693341686443,
      "learning_rate": 4.6471552610210073e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.9375,
      "logps/chosen": -504.0,
      "logps/rejected": -800.0,
      "loss": 0.248,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.078125,
      "rewards/rejected": -6.1875,
      "step": 3520
    },
    {
      "epoch": 0.25481845087706634,
      "grad_norm": 12.514956106781375,
      "learning_rate": 4.6439216376831447e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.75,
      "logps/chosen": -466.0,
      "logps/rejected": -744.0,
      "loss": 0.2408,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.75,
      "rewards/margins": 2.8125,
      "rewards/rejected": -5.5625,
      "step": 3530
    },
    {
      "epoch": 0.25554031617700135,
      "grad_norm": 17.087371262780874,
      "learning_rate": 4.6406744003842213e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.90625,
      "logps/chosen": -488.0,
      "logps/rejected": -732.0,
      "loss": 0.2502,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 2.65625,
      "rewards/rejected": -5.625,
      "step": 3540
    },
    {
      "epoch": 0.2562621814769364,
      "grad_norm": 14.0133252859323,
      "learning_rate": 4.6374135697442833e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.953125,
      "logps/chosen": -470.0,
      "logps/rejected": -772.0,
      "loss": 0.2313,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.90625,
      "rewards/margins": 3.046875,
      "rewards/rejected": -5.9375,
      "step": 3550
    },
    {
      "epoch": 0.2569840467768714,
      "grad_norm": 9.805653629926068,
      "learning_rate": 4.634139166469695e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.671875,
      "logps/chosen": -502.0,
      "logps/rejected": -788.0,
      "loss": 0.2529,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.046875,
      "rewards/margins": 2.78125,
      "rewards/rejected": -5.8125,
      "step": 3560
    },
    {
      "epoch": 0.2577059120768065,
      "grad_norm": 10.32820134535909,
      "learning_rate": 4.6308512113530063e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.546875,
      "logps/chosen": -446.0,
      "logps/rejected": -692.0,
      "loss": 0.2609,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.59375,
      "rewards/margins": 2.5625,
      "rewards/rejected": -5.15625,
      "step": 3570
    },
    {
      "epoch": 0.2584277773767415,
      "grad_norm": 9.747854763230055,
      "learning_rate": 4.627549725272822e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.734375,
      "logps/chosen": -492.0,
      "logps/rejected": -772.0,
      "loss": 0.2627,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.953125,
      "rewards/margins": 2.9375,
      "rewards/rejected": -5.875,
      "step": 3580
    },
    {
      "epoch": 0.2591496426766765,
      "grad_norm": 9.407917516321,
      "learning_rate": 4.62423472919367e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.46875,
      "logps/chosen": -460.0,
      "logps/rejected": -696.0,
      "loss": 0.2565,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.75,
      "rewards/margins": 2.46875,
      "rewards/rejected": -5.21875,
      "step": 3590
    },
    {
      "epoch": 0.25987150797661157,
      "grad_norm": 15.685569932906636,
      "learning_rate": 4.6209062441658654e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.65625,
      "logps/chosen": -468.0,
      "logps/rejected": -712.0,
      "loss": 0.2547,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.796875,
      "rewards/margins": 2.59375,
      "rewards/rejected": -5.375,
      "step": 3600
    },
    {
      "epoch": 0.2605933732765466,
      "grad_norm": 9.84659765567554,
      "learning_rate": 4.6175642913253783e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.53125,
      "logps/chosen": -472.0,
      "logps/rejected": -720.0,
      "loss": 0.2465,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.8125,
      "rewards/margins": 2.71875,
      "rewards/rejected": -5.53125,
      "step": 3610
    },
    {
      "epoch": 0.26131523857648165,
      "grad_norm": 9.774477108958665,
      "learning_rate": 4.6142088918937016e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.65625,
      "logps/chosen": -476.0,
      "logps/rejected": -728.0,
      "loss": 0.2435,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.875,
      "rewards/margins": 2.734375,
      "rewards/rejected": -5.59375,
      "step": 3620
    },
    {
      "epoch": 0.26203710387641665,
      "grad_norm": 8.96456035262789,
      "learning_rate": 4.6108400671777135e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.578125,
      "logps/chosen": -484.0,
      "logps/rejected": -752.0,
      "loss": 0.2334,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.859375,
      "rewards/margins": 2.90625,
      "rewards/rejected": -5.78125,
      "step": 3630
    },
    {
      "epoch": 0.2627589691763517,
      "grad_norm": 13.092282437254031,
      "learning_rate": 4.607457838569544e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.625,
      "logps/chosen": -510.0,
      "logps/rejected": -852.0,
      "loss": 0.2248,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.203125,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.4375,
      "step": 3640
    },
    {
      "epoch": 0.26348083447628673,
      "grad_norm": 12.222481192662093,
      "learning_rate": 4.6040622275464355e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5625,
      "logps/chosen": -468.0,
      "logps/rejected": -788.0,
      "loss": 0.2308,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.140625,
      "rewards/rejected": -6.0625,
      "step": 3650
    },
    {
      "epoch": 0.26420269977622174,
      "grad_norm": 11.32191550018901,
      "learning_rate": 4.6006532556706124e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.90625,
      "logps/chosen": -496.0,
      "logps/rejected": -780.0,
      "loss": 0.2384,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.265625,
      "rewards/margins": 2.90625,
      "rewards/rejected": -6.1875,
      "step": 3660
    },
    {
      "epoch": 0.2649245650761568,
      "grad_norm": 13.16728810398072,
      "learning_rate": 4.5972309445891386e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.515625,
      "logps/chosen": -496.0,
      "logps/rejected": -772.0,
      "loss": 0.2374,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.046875,
      "rewards/rejected": -6.125,
      "step": 3670
    },
    {
      "epoch": 0.2656464303760918,
      "grad_norm": 13.594824373280135,
      "learning_rate": 4.593795316033783e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.78125,
      "logps/chosen": -508.0,
      "logps/rejected": -784.0,
      "loss": 0.2499,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.234375,
      "rewards/margins": 2.90625,
      "rewards/rejected": -6.125,
      "step": 3680
    },
    {
      "epoch": 0.2663682956760269,
      "grad_norm": 12.41704299278977,
      "learning_rate": 4.5903463918208815e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -480.0,
      "logps/rejected": -788.0,
      "loss": 0.2468,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.140625,
      "rewards/rejected": -6.09375,
      "step": 3690
    },
    {
      "epoch": 0.2670901609759619,
      "grad_norm": 14.989493937540907,
      "learning_rate": 4.586884193851197e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.6875,
      "logps/chosen": -464.0,
      "logps/rejected": -712.0,
      "loss": 0.2503,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.859375,
      "rewards/margins": 2.421875,
      "rewards/rejected": -5.28125,
      "step": 3700
    },
    {
      "epoch": 0.2678120262758969,
      "grad_norm": 12.51758186067069,
      "learning_rate": 4.5834087441097806e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.65625,
      "logps/chosen": -464.0,
      "logps/rejected": -808.0,
      "loss": 0.2266,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.25,
      "step": 3710
    },
    {
      "epoch": 0.26853389157583196,
      "grad_norm": 9.77581503477085,
      "learning_rate": 4.5799200646658345e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.703125,
      "logps/chosen": -460.0,
      "logps/rejected": -752.0,
      "loss": 0.2154,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.921875,
      "rewards/margins": 2.921875,
      "rewards/rejected": -5.84375,
      "step": 3720
    },
    {
      "epoch": 0.26925575687576697,
      "grad_norm": 11.062875212711075,
      "learning_rate": 4.576418177672568e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.8125,
      "logps/chosen": -524.0,
      "logps/rejected": -788.0,
      "loss": 0.2493,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.15625,
      "rewards/margins": 2.890625,
      "rewards/rejected": -6.0625,
      "step": 3730
    },
    {
      "epoch": 0.26997762217570204,
      "grad_norm": 9.391695304310463,
      "learning_rate": 4.5729031053670596e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.5625,
      "logps/chosen": -456.0,
      "logps/rejected": -728.0,
      "loss": 0.2442,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.59375,
      "rewards/margins": 3.03125,
      "rewards/rejected": -5.625,
      "step": 3740
    },
    {
      "epoch": 0.27069948747563705,
      "grad_norm": 11.75430775377124,
      "learning_rate": 4.569374870070114e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.875,
      "logps/chosen": -496.0,
      "logps/rejected": -792.0,
      "loss": 0.2384,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.125,
      "rewards/rejected": -6.1875,
      "step": 3750
    },
    {
      "epoch": 0.27142135277557206,
      "grad_norm": 8.329747357841415,
      "learning_rate": 4.565833494186122e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.84375,
      "logps/chosen": -478.0,
      "logps/rejected": -752.0,
      "loss": 0.2397,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.921875,
      "rewards/margins": 2.875,
      "rewards/rejected": -5.78125,
      "step": 3760
    },
    {
      "epoch": 0.2721432180755071,
      "grad_norm": 9.582418010453875,
      "learning_rate": 4.562279000202919e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.484375,
      "logps/chosen": -478.0,
      "logps/rejected": -732.0,
      "loss": 0.2421,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.734375,
      "rewards/margins": 2.890625,
      "rewards/rejected": -5.625,
      "step": 3770
    },
    {
      "epoch": 0.27286508337544213,
      "grad_norm": 10.907277810869974,
      "learning_rate": 4.5587114106916366e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.71875,
      "logps/chosen": -480.0,
      "logps/rejected": -788.0,
      "loss": 0.241,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.09375,
      "rewards/rejected": -5.96875,
      "step": 3780
    },
    {
      "epoch": 0.2735869486753772,
      "grad_norm": 11.20683247743551,
      "learning_rate": 4.5551307483065664e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.875,
      "logps/chosen": -516.0,
      "logps/rejected": -836.0,
      "loss": 0.2428,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.21875,
      "rewards/margins": 3.390625,
      "rewards/rejected": -6.59375,
      "step": 3790
    },
    {
      "epoch": 0.2743088139753122,
      "grad_norm": 13.043933823709054,
      "learning_rate": 4.5515370357850136e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.78125,
      "logps/chosen": -488.0,
      "logps/rejected": -820.0,
      "loss": 0.2315,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.3125,
      "step": 3800
    },
    {
      "epoch": 0.2750306792752472,
      "grad_norm": 8.951383438963486,
      "learning_rate": 4.547930295947151e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.8125,
      "logps/chosen": -472.0,
      "logps/rejected": -740.0,
      "loss": 0.2209,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.03125,
      "rewards/rejected": -5.78125,
      "step": 3810
    },
    {
      "epoch": 0.2757525445751823,
      "grad_norm": 9.69258579643795,
      "learning_rate": 4.5443105516958737e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -3.03125,
      "logps/chosen": -504.0,
      "logps/rejected": -824.0,
      "loss": 0.2253,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.09375,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.5625,
      "step": 3820
    },
    {
      "epoch": 0.2764744098751173,
      "grad_norm": 13.524060862816272,
      "learning_rate": 4.5406778260166586e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.84375,
      "logps/chosen": -498.0,
      "logps/rejected": -800.0,
      "loss": 0.2318,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.28125,
      "rewards/rejected": -6.40625,
      "step": 3830
    },
    {
      "epoch": 0.27719627517505235,
      "grad_norm": 11.798281233796194,
      "learning_rate": 4.5370321419774117e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.453125,
      "logps/chosen": -474.0,
      "logps/rejected": -756.0,
      "loss": 0.2488,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.9375,
      "rewards/margins": 2.796875,
      "rewards/rejected": -5.75,
      "step": 3840
    },
    {
      "epoch": 0.27791814047498736,
      "grad_norm": 10.351200240344673,
      "learning_rate": 4.5333735227283274e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.671875,
      "logps/chosen": -490.0,
      "logps/rejected": -768.0,
      "loss": 0.2228,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.125,
      "rewards/margins": 2.84375,
      "rewards/rejected": -5.96875,
      "step": 3850
    },
    {
      "epoch": 0.2786400057749224,
      "grad_norm": 10.58721609591468,
      "learning_rate": 4.5297019915017375e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.921875,
      "logps/chosen": -510.0,
      "logps/rejected": -836.0,
      "loss": 0.2066,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.265625,
      "rewards/rejected": -6.65625,
      "step": 3860
    },
    {
      "epoch": 0.27936187107485744,
      "grad_norm": 13.537478691659441,
      "learning_rate": 4.5260175716119655e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.71875,
      "logps/chosen": -528.0,
      "logps/rejected": -788.0,
      "loss": 0.2483,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.140625,
      "rewards/margins": 2.90625,
      "rewards/rejected": -6.03125,
      "step": 3870
    },
    {
      "epoch": 0.28008373637479245,
      "grad_norm": 11.119135119357038,
      "learning_rate": 4.522320286455179e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.703125,
      "logps/chosen": -452.0,
      "logps/rejected": -692.0,
      "loss": 0.2453,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.75,
      "rewards/margins": 2.6875,
      "rewards/rejected": -5.4375,
      "step": 3880
    },
    {
      "epoch": 0.2808056016747275,
      "grad_norm": 13.116830958448425,
      "learning_rate": 4.5186101595092403e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.8125,
      "logps/chosen": -470.0,
      "logps/rejected": -728.0,
      "loss": 0.25,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.84375,
      "rewards/margins": 2.6875,
      "rewards/rejected": -5.53125,
      "step": 3890
    },
    {
      "epoch": 0.2815274669746625,
      "grad_norm": 10.821311447109608,
      "learning_rate": 4.5148872143335566e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.625,
      "logps/chosen": -492.0,
      "logps/rejected": -796.0,
      "loss": 0.2302,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.171875,
      "rewards/rejected": -6.15625,
      "step": 3900
    },
    {
      "epoch": 0.2822493322745976,
      "grad_norm": 11.460354434691167,
      "learning_rate": 4.5111514745689307e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.875,
      "logps/chosen": -468.0,
      "logps/rejected": -784.0,
      "loss": 0.2348,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.25,
      "step": 3910
    },
    {
      "epoch": 0.2829711975745326,
      "grad_norm": 13.18839407473639,
      "learning_rate": 4.507402963937414e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.734375,
      "logps/chosen": -498.0,
      "logps/rejected": -800.0,
      "loss": 0.2029,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.109375,
      "rewards/rejected": -6.125,
      "step": 3920
    },
    {
      "epoch": 0.2836930628744676,
      "grad_norm": 16.44744701013159,
      "learning_rate": 4.5036417062421506e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -3.0,
      "logps/chosen": -544.0,
      "logps/rejected": -888.0,
      "loss": 0.2594,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.625,
      "rewards/rejected": -7.125,
      "step": 3930
    },
    {
      "epoch": 0.28441492817440267,
      "grad_norm": 13.041057194840082,
      "learning_rate": 4.4998677253672297e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.796875,
      "logps/chosen": -524.0,
      "logps/rejected": -808.0,
      "loss": 0.2428,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.25,
      "rewards/margins": 2.96875,
      "rewards/rejected": -6.21875,
      "step": 3940
    },
    {
      "epoch": 0.2851367934743377,
      "grad_norm": 12.748707203561708,
      "learning_rate": 4.496081045277533e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.765625,
      "logps/chosen": -484.0,
      "logps/rejected": -788.0,
      "loss": 0.2405,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.96875,
      "rewards/margins": 2.75,
      "rewards/rejected": -5.71875,
      "step": 3950
    },
    {
      "epoch": 0.28585865877427274,
      "grad_norm": 11.81274411549332,
      "learning_rate": 4.492281690018583e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.765625,
      "logps/chosen": -474.0,
      "logps/rejected": -728.0,
      "loss": 0.2197,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.859375,
      "rewards/margins": 2.8125,
      "rewards/rejected": -5.6875,
      "step": 3960
    },
    {
      "epoch": 0.28658052407420775,
      "grad_norm": 11.535098057046644,
      "learning_rate": 4.4884696837163905e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.75,
      "logps/chosen": -504.0,
      "logps/rejected": -772.0,
      "loss": 0.2466,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.0,
      "rewards/margins": 2.875,
      "rewards/rejected": -5.875,
      "step": 3970
    },
    {
      "epoch": 0.28730238937414276,
      "grad_norm": 11.242175467186813,
      "learning_rate": 4.484645050577299e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.609375,
      "logps/chosen": -484.0,
      "logps/rejected": -796.0,
      "loss": 0.2374,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.09375,
      "rewards/rejected": -6.09375,
      "step": 3980
    },
    {
      "epoch": 0.28802425467407783,
      "grad_norm": 11.060505089940248,
      "learning_rate": 4.480807814887833e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.8125,
      "logps/chosen": -452.0,
      "logps/rejected": -696.0,
      "loss": 0.2366,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.71875,
      "rewards/margins": 2.703125,
      "rewards/rejected": -5.40625,
      "step": 3990
    },
    {
      "epoch": 0.28874611997401284,
      "grad_norm": 10.263434370988277,
      "learning_rate": 4.476958001014544e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.828125,
      "logps/chosen": -458.0,
      "logps/rejected": -752.0,
      "loss": 0.2395,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.046875,
      "rewards/rejected": -5.75,
      "step": 4000
    },
    {
      "epoch": 0.2894679852739479,
      "grad_norm": 11.051254929442925,
      "learning_rate": 4.4730956334038565e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.890625,
      "logps/chosen": -496.0,
      "logps/rejected": -792.0,
      "loss": 0.2086,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.25,
      "step": 4010
    },
    {
      "epoch": 0.2901898505738829,
      "grad_norm": 10.937580543605891,
      "learning_rate": 4.46922073658191e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.90625,
      "logps/chosen": -492.0,
      "logps/rejected": -828.0,
      "loss": 0.2223,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.03125,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.59375,
      "step": 4020
    },
    {
      "epoch": 0.2909117158738179,
      "grad_norm": 12.078102339541845,
      "learning_rate": 4.465333335154406e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -486.0,
      "logps/rejected": -764.0,
      "loss": 0.247,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.03125,
      "rewards/margins": 2.796875,
      "rewards/rejected": -5.8125,
      "step": 4030
    },
    {
      "epoch": 0.291633581173753,
      "grad_norm": 11.440475094446068,
      "learning_rate": 4.461433453806449e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.65625,
      "logps/chosen": -466.0,
      "logps/rejected": -748.0,
      "loss": 0.2296,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.71875,
      "rewards/margins": 2.90625,
      "rewards/rejected": -5.625,
      "step": 4040
    },
    {
      "epoch": 0.292355446473688,
      "grad_norm": 14.360771639444092,
      "learning_rate": 4.457521117302392e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.90625,
      "logps/chosen": -464.0,
      "logps/rejected": -752.0,
      "loss": 0.2514,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.890625,
      "rewards/margins": 2.953125,
      "rewards/rejected": -5.84375,
      "step": 4050
    },
    {
      "epoch": 0.29307731177362306,
      "grad_norm": 11.263443329618648,
      "learning_rate": 4.45359635048568e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.859375,
      "logps/chosen": -478.0,
      "logps/rejected": -748.0,
      "loss": 0.2349,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.96875,
      "rewards/margins": 2.6875,
      "rewards/rejected": -5.65625,
      "step": 4060
    },
    {
      "epoch": 0.29379917707355807,
      "grad_norm": 10.337277710057334,
      "learning_rate": 4.4496591782786883e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.75,
      "logps/chosen": -482.0,
      "logps/rejected": -792.0,
      "loss": 0.2665,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.140625,
      "rewards/rejected": -6.15625,
      "step": 4070
    },
    {
      "epoch": 0.2945210423734931,
      "grad_norm": 14.95890807575268,
      "learning_rate": 4.44570962568257e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.90625,
      "logps/chosen": -508.0,
      "logps/rejected": -840.0,
      "loss": 0.2363,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.1875,
      "rewards/margins": 3.53125,
      "rewards/rejected": -6.71875,
      "step": 4080
    },
    {
      "epoch": 0.29524290767342815,
      "grad_norm": 10.788798203049051,
      "learning_rate": 4.4417477177770905e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.984375,
      "logps/chosen": -472.0,
      "logps/rejected": -768.0,
      "loss": 0.2294,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.0625,
      "rewards/rejected": -5.96875,
      "step": 4090
    },
    {
      "epoch": 0.29596477297336315,
      "grad_norm": 11.655360773186286,
      "learning_rate": 4.4377734797204747e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.578125,
      "logps/chosen": -448.0,
      "logps/rejected": -776.0,
      "loss": 0.224,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.734375,
      "rewards/margins": 3.375,
      "rewards/rejected": -6.125,
      "step": 4100
    },
    {
      "epoch": 0.2966866382732982,
      "grad_norm": 9.468204960246823,
      "learning_rate": 4.433786936749241e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.609375,
      "logps/chosen": -492.0,
      "logps/rejected": -788.0,
      "loss": 0.2273,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.125,
      "step": 4110
    },
    {
      "epoch": 0.29740850357323323,
      "grad_norm": 9.9199628803366,
      "learning_rate": 4.429788114178049e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.6875,
      "logps/chosen": -480.0,
      "logps/rejected": -760.0,
      "loss": 0.2186,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 2.9375,
      "rewards/rejected": -5.875,
      "step": 4120
    },
    {
      "epoch": 0.2981303688731683,
      "grad_norm": 9.759952777278889,
      "learning_rate": 4.425777037399529e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.96875,
      "logps/chosen": -510.0,
      "logps/rejected": -796.0,
      "loss": 0.2191,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.09375,
      "rewards/rejected": -6.25,
      "step": 4130
    },
    {
      "epoch": 0.2988522341731033,
      "grad_norm": 17.245979700732356,
      "learning_rate": 4.42175373188413e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.78125,
      "logps/chosen": -520.0,
      "logps/rejected": -812.0,
      "loss": 0.2302,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.109375,
      "rewards/rejected": -6.5,
      "step": 4140
    },
    {
      "epoch": 0.2995740994730383,
      "grad_norm": 10.989889057696143,
      "learning_rate": 4.4177182231799513e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.484375,
      "logps/chosen": -500.0,
      "logps/rejected": -836.0,
      "loss": 0.2261,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.171875,
      "rewards/margins": 3.234375,
      "rewards/rejected": -6.40625,
      "step": 4150
    },
    {
      "epoch": 0.3002959647729734,
      "grad_norm": 10.349736275068137,
      "learning_rate": 4.413670536912584e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.71875,
      "logps/chosen": -532.0,
      "logps/rejected": -796.0,
      "loss": 0.2378,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.15625,
      "rewards/margins": 2.90625,
      "rewards/rejected": -6.0625,
      "step": 4160
    },
    {
      "epoch": 0.3010178300729084,
      "grad_norm": 8.080593230158266,
      "learning_rate": 4.4096106987849457e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.734375,
      "logps/chosen": -488.0,
      "logps/rejected": -812.0,
      "loss": 0.2223,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.3125,
      "step": 4170
    },
    {
      "epoch": 0.30173969537284345,
      "grad_norm": 15.975559808656076,
      "learning_rate": 4.405538734577121e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.75,
      "logps/chosen": -498.0,
      "logps/rejected": -784.0,
      "loss": 0.2266,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.125,
      "rewards/margins": 2.859375,
      "rewards/rejected": -5.96875,
      "step": 4180
    },
    {
      "epoch": 0.30246156067277846,
      "grad_norm": 11.931268777483961,
      "learning_rate": 4.401454670146193e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.921875,
      "logps/chosen": -510.0,
      "logps/rejected": -820.0,
      "loss": 0.236,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.375,
      "rewards/rejected": -6.4375,
      "step": 4190
    },
    {
      "epoch": 0.30318342597271347,
      "grad_norm": 10.482285587356756,
      "learning_rate": 4.3973585314260836e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.59375,
      "logps/chosen": -468.0,
      "logps/rejected": -776.0,
      "loss": 0.2222,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.046875,
      "rewards/rejected": -5.9375,
      "step": 4200
    },
    {
      "epoch": 0.30390529127264854,
      "grad_norm": 16.028200114457633,
      "learning_rate": 4.393250344427385e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5,
      "logps/chosen": -496.0,
      "logps/rejected": -768.0,
      "loss": 0.2269,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.15625,
      "rewards/margins": 2.859375,
      "rewards/rejected": -6.03125,
      "step": 4210
    },
    {
      "epoch": 0.30462715657258355,
      "grad_norm": 10.120911168237928,
      "learning_rate": 4.389130135237196e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.890625,
      "logps/chosen": -504.0,
      "logps/rejected": -840.0,
      "loss": 0.2599,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.6875,
      "step": 4220
    },
    {
      "epoch": 0.3053490218725186,
      "grad_norm": 10.866383253636467,
      "learning_rate": 4.38499793001896e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.65625,
      "logps/chosen": -488.0,
      "logps/rejected": -752.0,
      "loss": 0.2393,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.015625,
      "rewards/margins": 2.796875,
      "rewards/rejected": -5.8125,
      "step": 4230
    },
    {
      "epoch": 0.3060708871724536,
      "grad_norm": 11.354499209587136,
      "learning_rate": 4.3808537550122913e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.796875,
      "logps/chosen": -486.0,
      "logps/rejected": -716.0,
      "loss": 0.2501,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.0,
      "rewards/margins": 2.46875,
      "rewards/rejected": -5.46875,
      "step": 4240
    },
    {
      "epoch": 0.30679275247238863,
      "grad_norm": 9.307627201905396,
      "learning_rate": 4.376697636532816e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.796875,
      "logps/chosen": -478.0,
      "logps/rejected": -780.0,
      "loss": 0.2099,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.046875,
      "rewards/rejected": -6.03125,
      "step": 4250
    },
    {
      "epoch": 0.3075146177723237,
      "grad_norm": 12.334959306396991,
      "learning_rate": 4.3725296009719985e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.96875,
      "logps/chosen": -464.0,
      "logps/rejected": -784.0,
      "loss": 0.2123,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.09375,
      "step": 4260
    },
    {
      "epoch": 0.3082364830722587,
      "grad_norm": 8.216528187102487,
      "learning_rate": 4.3683496747969804e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.9375,
      "logps/chosen": -480.0,
      "logps/rejected": -816.0,
      "loss": 0.2276,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.46875,
      "step": 4270
    },
    {
      "epoch": 0.30895834837219377,
      "grad_norm": 10.133915655575697,
      "learning_rate": 4.364157884550406e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.046875,
      "logps/chosen": -502.0,
      "logps/rejected": -832.0,
      "loss": 0.2298,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.390625,
      "rewards/rejected": -6.46875,
      "step": 4280
    },
    {
      "epoch": 0.3096802136721288,
      "grad_norm": 14.958027801447614,
      "learning_rate": 4.359954256850259e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.8125,
      "logps/chosen": -450.0,
      "logps/rejected": -752.0,
      "loss": 0.2598,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.796875,
      "rewards/margins": 3.03125,
      "rewards/rejected": -5.84375,
      "step": 4290
    },
    {
      "epoch": 0.3104020789720638,
      "grad_norm": 15.945935398802646,
      "learning_rate": 4.35573881838969e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -3.046875,
      "logps/chosen": -520.0,
      "logps/rejected": -788.0,
      "loss": 0.2417,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.3125,
      "rewards/margins": 2.90625,
      "rewards/rejected": -6.21875,
      "step": 4300
    },
    {
      "epoch": 0.31112394427199885,
      "grad_norm": 12.522934819936081,
      "learning_rate": 4.3515115959368476e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.03125,
      "logps/chosen": -506.0,
      "logps/rejected": -764.0,
      "loss": 0.2347,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.0625,
      "rewards/margins": 2.890625,
      "rewards/rejected": -5.9375,
      "step": 4310
    },
    {
      "epoch": 0.31184580957193386,
      "grad_norm": 13.962655607680325,
      "learning_rate": 4.3472726163347116e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.078125,
      "logps/chosen": -490.0,
      "logps/rejected": -764.0,
      "loss": 0.2297,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.015625,
      "rewards/rejected": -6.03125,
      "step": 4320
    },
    {
      "epoch": 0.31256767487186893,
      "grad_norm": 10.117053048486753,
      "learning_rate": 4.3430219065009177e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.71875,
      "logps/chosen": -490.0,
      "logps/rejected": -792.0,
      "loss": 0.2301,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.984375,
      "rewards/margins": 3.09375,
      "rewards/rejected": -6.09375,
      "step": 4330
    },
    {
      "epoch": 0.31328954017180394,
      "grad_norm": 12.08059874452895,
      "learning_rate": 4.3387594934275896e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.75,
      "logps/chosen": -520.0,
      "logps/rejected": -836.0,
      "loss": 0.2247,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.21875,
      "rewards/rejected": -6.46875,
      "step": 4340
    },
    {
      "epoch": 0.31401140547173895,
      "grad_norm": 12.700419777870017,
      "learning_rate": 4.334485404181167e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.84375,
      "logps/chosen": -496.0,
      "logps/rejected": -792.0,
      "loss": 0.2249,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.09375,
      "rewards/rejected": -6.21875,
      "step": 4350
    },
    {
      "epoch": 0.314733270771674,
      "grad_norm": 12.12493695834519,
      "learning_rate": 4.3301996659022334e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -500.0,
      "logps/rejected": -800.0,
      "loss": 0.2412,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.109375,
      "rewards/rejected": -6.15625,
      "step": 4360
    },
    {
      "epoch": 0.315455136071609,
      "grad_norm": 14.366699793628479,
      "learning_rate": 4.3259023058053444e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.953125,
      "logps/chosen": -496.0,
      "logps/rejected": -768.0,
      "loss": 0.2192,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 2.796875,
      "rewards/rejected": -5.90625,
      "step": 4370
    },
    {
      "epoch": 0.3161770013715441,
      "grad_norm": 12.332624089738163,
      "learning_rate": 4.3215933511788544e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.0625,
      "logps/chosen": -502.0,
      "logps/rejected": -780.0,
      "loss": 0.2381,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 2.921875,
      "rewards/rejected": -6.1875,
      "step": 4380
    },
    {
      "epoch": 0.3168988666714791,
      "grad_norm": 10.468719104334246,
      "learning_rate": 4.317272829384743e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.96875,
      "logps/chosen": -496.0,
      "logps/rejected": -764.0,
      "loss": 0.2184,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.09375,
      "rewards/margins": 2.78125,
      "rewards/rejected": -5.875,
      "step": 4390
    },
    {
      "epoch": 0.31762073197141416,
      "grad_norm": 10.290141687548948,
      "learning_rate": 4.3129407678584414e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -3.0625,
      "logps/chosen": -454.0,
      "logps/rejected": -720.0,
      "loss": 0.2377,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.671875,
      "rewards/margins": 2.859375,
      "rewards/rejected": -5.53125,
      "step": 4400
    },
    {
      "epoch": 0.31834259727134917,
      "grad_norm": 14.6934151314309,
      "learning_rate": 4.3085971941086585e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.859375,
      "logps/chosen": -484.0,
      "logps/rejected": -768.0,
      "loss": 0.2164,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.984375,
      "rewards/margins": 2.921875,
      "rewards/rejected": -5.90625,
      "step": 4410
    },
    {
      "epoch": 0.3190644625712842,
      "grad_norm": 9.91674001308468,
      "learning_rate": 4.3042421357172076e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.6875,
      "logps/chosen": -532.0,
      "logps/rejected": -884.0,
      "loss": 0.2179,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.84375,
      "step": 4420
    },
    {
      "epoch": 0.31978632787121924,
      "grad_norm": 11.495675955116727,
      "learning_rate": 4.2998756203388285e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.984375,
      "logps/chosen": -506.0,
      "logps/rejected": -800.0,
      "loss": 0.2395,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.28125,
      "step": 4430
    },
    {
      "epoch": 0.32050819317115425,
      "grad_norm": 11.432987943608707,
      "learning_rate": 4.2954976757010133e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.75,
      "logps/chosen": -512.0,
      "logps/rejected": -800.0,
      "loss": 0.2227,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.171875,
      "rewards/margins": 3.078125,
      "rewards/rejected": -6.25,
      "step": 4440
    },
    {
      "epoch": 0.3212300584710893,
      "grad_norm": 13.610387053428951,
      "learning_rate": 4.2911083296038285e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.953125,
      "logps/chosen": -508.0,
      "logps/rejected": -800.0,
      "loss": 0.2197,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.203125,
      "rewards/rejected": -6.34375,
      "step": 4450
    },
    {
      "epoch": 0.32195192377102433,
      "grad_norm": 12.562790757305583,
      "learning_rate": 4.2867076099197446e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.765625,
      "logps/chosen": -506.0,
      "logps/rejected": -796.0,
      "loss": 0.2021,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.265625,
      "rewards/rejected": -6.34375,
      "step": 4460
    },
    {
      "epoch": 0.32267378907095934,
      "grad_norm": 13.047560394628606,
      "learning_rate": 4.2822955445934505e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -3.15625,
      "logps/chosen": -528.0,
      "logps/rejected": -848.0,
      "loss": 0.2042,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.75,
      "step": 4470
    },
    {
      "epoch": 0.3233956543708944,
      "grad_norm": 19.50594714399454,
      "learning_rate": 4.277872161641681e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.953125,
      "logps/chosen": -520.0,
      "logps/rejected": -864.0,
      "loss": 0.2164,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.203125,
      "rewards/margins": 3.53125,
      "rewards/rejected": -6.75,
      "step": 4480
    },
    {
      "epoch": 0.3241175196708294,
      "grad_norm": 10.3524278780653,
      "learning_rate": 4.273437489153041e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.859375,
      "logps/chosen": -510.0,
      "logps/rejected": -820.0,
      "loss": 0.2271,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.375,
      "rewards/rejected": -6.5,
      "step": 4490
    },
    {
      "epoch": 0.3248393849707645,
      "grad_norm": 11.195057709514954,
      "learning_rate": 4.2689915552878207e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.734375,
      "logps/chosen": -498.0,
      "logps/rejected": -780.0,
      "loss": 0.2239,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.90625,
      "rewards/margins": 3.0625,
      "rewards/rejected": -5.96875,
      "step": 4500
    },
    {
      "epoch": 0.3255612502706995,
      "grad_norm": 10.759408860514071,
      "learning_rate": 4.2645343882778215e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.203125,
      "logps/chosen": -496.0,
      "logps/rejected": -800.0,
      "loss": 0.2328,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.21875,
      "rewards/rejected": -6.34375,
      "step": 4510
    },
    {
      "epoch": 0.3262831155706345,
      "grad_norm": 10.257842921839272,
      "learning_rate": 4.260066016426177e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.96875,
      "logps/chosen": -498.0,
      "logps/rejected": -808.0,
      "loss": 0.2016,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.203125,
      "rewards/rejected": -6.34375,
      "step": 4520
    },
    {
      "epoch": 0.32700498087056956,
      "grad_norm": 14.681944390001153,
      "learning_rate": 4.25558646810717e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -3.0,
      "logps/chosen": -502.0,
      "logps/rejected": -800.0,
      "loss": 0.1942,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.171875,
      "rewards/margins": 3.125,
      "rewards/rejected": -6.28125,
      "step": 4530
    },
    {
      "epoch": 0.32772684617050457,
      "grad_norm": 14.130179880482482,
      "learning_rate": 4.251095771766055e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.765625,
      "logps/chosen": -520.0,
      "logps/rejected": -856.0,
      "loss": 0.2112,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.59375,
      "step": 4540
    },
    {
      "epoch": 0.32844871147043964,
      "grad_norm": 8.8696970691209,
      "learning_rate": 4.246593955918877e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.09375,
      "logps/chosen": -502.0,
      "logps/rejected": -800.0,
      "loss": 0.2442,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.1875,
      "rewards/margins": 3.125,
      "rewards/rejected": -6.3125,
      "step": 4550
    },
    {
      "epoch": 0.32917057677037465,
      "grad_norm": 7.742190542236623,
      "learning_rate": 4.2420810491522896e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.890625,
      "logps/chosen": -470.0,
      "logps/rejected": -736.0,
      "loss": 0.2294,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.0,
      "rewards/margins": 2.703125,
      "rewards/rejected": -5.71875,
      "step": 4560
    },
    {
      "epoch": 0.32989244207030965,
      "grad_norm": 9.718776216062784,
      "learning_rate": 4.237557080123373e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.734375,
      "logps/chosen": -494.0,
      "logps/rejected": -744.0,
      "loss": 0.2504,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.046875,
      "rewards/margins": 2.796875,
      "rewards/rejected": -5.84375,
      "step": 4570
    },
    {
      "epoch": 0.3306143073702447,
      "grad_norm": 14.717048285874382,
      "learning_rate": 4.2330220775594567e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.6875,
      "logps/chosen": -490.0,
      "logps/rejected": -764.0,
      "loss": 0.2379,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.015625,
      "rewards/rejected": -5.96875,
      "step": 4580
    },
    {
      "epoch": 0.33133617267017973,
      "grad_norm": 11.892469546919582,
      "learning_rate": 4.228476070257929e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -536.0,
      "logps/rejected": -800.0,
      "loss": 0.207,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 2.8125,
      "rewards/rejected": -6.3125,
      "step": 4590
    },
    {
      "epoch": 0.3320580379701148,
      "grad_norm": 13.30490570233992,
      "learning_rate": 4.223919087086062e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.9375,
      "logps/chosen": -510.0,
      "logps/rejected": -848.0,
      "loss": 0.2175,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.53125,
      "rewards/rejected": -6.84375,
      "step": 4600
    },
    {
      "epoch": 0.3327799032700498,
      "grad_norm": 9.670999787060145,
      "learning_rate": 4.2193511569808225e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.0,
      "logps/chosen": -480.0,
      "logps/rejected": -808.0,
      "loss": 0.2263,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.28125,
      "step": 4610
    },
    {
      "epoch": 0.3335017685699848,
      "grad_norm": 6.3786382315588055,
      "learning_rate": 4.214772308948693e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.796875,
      "logps/chosen": -458.0,
      "logps/rejected": -752.0,
      "loss": 0.2325,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.6875,
      "rewards/margins": 3.171875,
      "rewards/rejected": -5.84375,
      "step": 4620
    },
    {
      "epoch": 0.3342236338699199,
      "grad_norm": 16.096306093490902,
      "learning_rate": 4.2101825720654827e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.78125,
      "logps/chosen": -496.0,
      "logps/rejected": -768.0,
      "loss": 0.2157,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.09375,
      "rewards/margins": 2.859375,
      "rewards/rejected": -5.9375,
      "step": 4630
    },
    {
      "epoch": 0.3349454991698549,
      "grad_norm": 8.872588796580702,
      "learning_rate": 4.205581975476146e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.03125,
      "logps/chosen": -498.0,
      "logps/rejected": -824.0,
      "loss": 0.2271,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.390625,
      "rewards/rejected": -6.53125,
      "step": 4640
    },
    {
      "epoch": 0.33566736446978995,
      "grad_norm": 14.866846482464831,
      "learning_rate": 4.200970548394598e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -3.265625,
      "logps/chosen": -520.0,
      "logps/rejected": -812.0,
      "loss": 0.2202,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.234375,
      "rewards/rejected": -6.34375,
      "step": 4650
    },
    {
      "epoch": 0.33638922976972496,
      "grad_norm": 14.331548961773473,
      "learning_rate": 4.196348320103527e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.859375,
      "logps/chosen": -470.0,
      "logps/rejected": -768.0,
      "loss": 0.2332,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.0,
      "rewards/margins": 2.84375,
      "rewards/rejected": -5.84375,
      "step": 4660
    },
    {
      "epoch": 0.33711109506966,
      "grad_norm": 13.28952726836952,
      "learning_rate": 4.191715319954209e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.0625,
      "logps/chosen": -484.0,
      "logps/rejected": -796.0,
      "loss": 0.2187,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.203125,
      "rewards/rejected": -6.25,
      "step": 4670
    },
    {
      "epoch": 0.33783296036959504,
      "grad_norm": 12.088871078688804,
      "learning_rate": 4.187071577366321e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.0625,
      "logps/chosen": -498.0,
      "logps/rejected": -768.0,
      "loss": 0.2014,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.171875,
      "rewards/margins": 2.921875,
      "rewards/rejected": -6.09375,
      "step": 4680
    },
    {
      "epoch": 0.33855482566953005,
      "grad_norm": 9.272870455966427,
      "learning_rate": 4.182417121827755e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.015625,
      "logps/chosen": -508.0,
      "logps/rejected": -804.0,
      "loss": 0.2087,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.34375,
      "rewards/margins": 2.9375,
      "rewards/rejected": -6.28125,
      "step": 4690
    },
    {
      "epoch": 0.3392766909694651,
      "grad_norm": 14.53535476522131,
      "learning_rate": 4.177751982894433e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -3.15625,
      "logps/chosen": -496.0,
      "logps/rejected": -784.0,
      "loss": 0.2225,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.03125,
      "rewards/margins": 3.296875,
      "rewards/rejected": -6.3125,
      "step": 4700
    },
    {
      "epoch": 0.3399985562694001,
      "grad_norm": 13.814416257393315,
      "learning_rate": 4.1730761901901135e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.75,
      "logps/chosen": -536.0,
      "logps/rejected": -904.0,
      "loss": 0.2252,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.765625,
      "rewards/rejected": -7.15625,
      "step": 4710
    },
    {
      "epoch": 0.3407204215693352,
      "grad_norm": 13.134993375278981,
      "learning_rate": 4.168389773406209e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.1875,
      "logps/chosen": -536.0,
      "logps/rejected": -840.0,
      "loss": 0.2112,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.21875,
      "rewards/rejected": -6.6875,
      "step": 4720
    },
    {
      "epoch": 0.3414422868692702,
      "grad_norm": 12.69266212969094,
      "learning_rate": 4.1636927623015927e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.03125,
      "logps/chosen": -488.0,
      "logps/rejected": -816.0,
      "loss": 0.2143,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.4375,
      "step": 4730
    },
    {
      "epoch": 0.3421641521692052,
      "grad_norm": 14.083621564521064,
      "learning_rate": 4.158985186702415e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.8125,
      "logps/chosen": -476.0,
      "logps/rejected": -828.0,
      "loss": 0.2344,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.34375,
      "step": 4740
    },
    {
      "epoch": 0.34288601746914027,
      "grad_norm": 12.025034242421366,
      "learning_rate": 4.1542670765019104e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.765625,
      "logps/chosen": -470.0,
      "logps/rejected": -776.0,
      "loss": 0.2093,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.8125,
      "rewards/margins": 3.28125,
      "rewards/rejected": -6.09375,
      "step": 4750
    },
    {
      "epoch": 0.3436078827690753,
      "grad_norm": 8.427739832626886,
      "learning_rate": 4.149538461660206e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.796875,
      "logps/chosen": -460.0,
      "logps/rejected": -736.0,
      "loss": 0.231,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.875,
      "rewards/margins": 2.765625,
      "rewards/rejected": -5.625,
      "step": 4760
    },
    {
      "epoch": 0.34432974806901034,
      "grad_norm": 10.453843705855519,
      "learning_rate": 4.144799372204136e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.953125,
      "logps/chosen": -482.0,
      "logps/rejected": -792.0,
      "loss": 0.1966,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.21875,
      "step": 4770
    },
    {
      "epoch": 0.34505161336894535,
      "grad_norm": 11.147779764040918,
      "learning_rate": 4.140049838227049e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.96875,
      "logps/chosen": -504.0,
      "logps/rejected": -832.0,
      "loss": 0.2149,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.515625,
      "rewards/rejected": -6.5625,
      "step": 4780
    },
    {
      "epoch": 0.34577347866888036,
      "grad_norm": 14.194229222790018,
      "learning_rate": 4.135289889888615e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -3.15625,
      "logps/chosen": -512.0,
      "logps/rejected": -868.0,
      "loss": 0.2028,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.71875,
      "rewards/rejected": -6.84375,
      "step": 4790
    },
    {
      "epoch": 0.34649534396881543,
      "grad_norm": 11.93797460277735,
      "learning_rate": 4.130519557414636e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.953125,
      "logps/chosen": -490.0,
      "logps/rejected": -824.0,
      "loss": 0.2261,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.46875,
      "step": 4800
    },
    {
      "epoch": 0.34721720926875044,
      "grad_norm": 14.302948848960554,
      "learning_rate": 4.1257388710968533e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.953125,
      "logps/chosen": -496.0,
      "logps/rejected": -832.0,
      "loss": 0.2135,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.625,
      "step": 4810
    },
    {
      "epoch": 0.3479390745686855,
      "grad_norm": 13.662773710796282,
      "learning_rate": 4.120947861292757e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -3.078125,
      "logps/chosen": -490.0,
      "logps/rejected": -792.0,
      "loss": 0.2099,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.1875,
      "rewards/rejected": -6.1875,
      "step": 4820
    },
    {
      "epoch": 0.3486609398686205,
      "grad_norm": 13.85449746585761,
      "learning_rate": 4.11614655842539e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.921875,
      "logps/chosen": -506.0,
      "logps/rejected": -784.0,
      "loss": 0.2278,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.03125,
      "rewards/rejected": -6.0,
      "step": 4830
    },
    {
      "epoch": 0.3493828051685555,
      "grad_norm": 15.021261384561745,
      "learning_rate": 4.111334992983156e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.890625,
      "logps/chosen": -480.0,
      "logps/rejected": -808.0,
      "loss": 0.214,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.90625,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.3125,
      "step": 4840
    },
    {
      "epoch": 0.3501046704684906,
      "grad_norm": 12.63540399346357,
      "learning_rate": 4.1065131955196294e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.109375,
      "logps/chosen": -480.0,
      "logps/rejected": -836.0,
      "loss": 0.2231,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.609375,
      "rewards/rejected": -6.59375,
      "step": 4850
    },
    {
      "epoch": 0.3508265357684256,
      "grad_norm": 11.22880016919188,
      "learning_rate": 4.1016811966533536e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.53125,
      "logps/chosen": -524.0,
      "logps/rejected": -888.0,
      "loss": 0.193,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.65625,
      "rewards/rejected": -7.15625,
      "step": 4860
    },
    {
      "epoch": 0.35154840106836066,
      "grad_norm": 14.880036287798237,
      "learning_rate": 4.096839027067656e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.84375,
      "logps/chosen": -500.0,
      "logps/rejected": -868.0,
      "loss": 0.2326,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.75,
      "rewards/rejected": -6.90625,
      "step": 4870
    },
    {
      "epoch": 0.35227026636829567,
      "grad_norm": 12.2019692344996,
      "learning_rate": 4.0919867175104435e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.9375,
      "logps/chosen": -480.0,
      "logps/rejected": -752.0,
      "loss": 0.2426,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.84375,
      "rewards/margins": 2.96875,
      "rewards/rejected": -5.8125,
      "step": 4880
    },
    {
      "epoch": 0.35299213166823074,
      "grad_norm": 15.703638351023915,
      "learning_rate": 4.0871242987940155e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -3.0,
      "logps/chosen": -450.0,
      "logps/rejected": -712.0,
      "loss": 0.231,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.65625,
      "rewards/margins": 2.796875,
      "rewards/rejected": -5.4375,
      "step": 4890
    },
    {
      "epoch": 0.35371399696816574,
      "grad_norm": 12.640672925100505,
      "learning_rate": 4.082251801794865e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.90625,
      "logps/chosen": -496.0,
      "logps/rejected": -840.0,
      "loss": 0.2057,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.4375,
      "step": 4900
    },
    {
      "epoch": 0.35443586226810075,
      "grad_norm": 10.448070820157008,
      "learning_rate": 4.0773692574534795e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.0,
      "logps/chosen": -482.0,
      "logps/rejected": -748.0,
      "loss": 0.2188,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.9375,
      "rewards/margins": 2.890625,
      "rewards/rejected": -5.8125,
      "step": 4910
    },
    {
      "epoch": 0.3551577275680358,
      "grad_norm": 10.23740185542739,
      "learning_rate": 4.072476696774151e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.765625,
      "logps/chosen": -486.0,
      "logps/rejected": -832.0,
      "loss": 0.216,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.40625,
      "step": 4920
    },
    {
      "epoch": 0.35587959286797083,
      "grad_norm": 12.619623909307741,
      "learning_rate": 4.0675741508247715e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.796875,
      "logps/chosen": -488.0,
      "logps/rejected": -864.0,
      "loss": 0.1933,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.09375,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.03125,
      "step": 4930
    },
    {
      "epoch": 0.3566014581679059,
      "grad_norm": 13.535767047638867,
      "learning_rate": 4.062661650736643e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.046875,
      "logps/chosen": -536.0,
      "logps/rejected": -876.0,
      "loss": 0.2182,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.625,
      "rewards/rejected": -7.0,
      "step": 4940
    },
    {
      "epoch": 0.3573233234678409,
      "grad_norm": 8.543446965748016,
      "learning_rate": 4.0577392277042766e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.984375,
      "logps/chosen": -500.0,
      "logps/rejected": -804.0,
      "loss": 0.2311,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.21875,
      "rewards/rejected": -6.3125,
      "step": 4950
    },
    {
      "epoch": 0.3580451887677759,
      "grad_norm": 12.102451163706249,
      "learning_rate": 4.0528069129851914e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -482.0,
      "logps/rejected": -812.0,
      "loss": 0.2086,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.21875,
      "step": 4960
    },
    {
      "epoch": 0.358767054067711,
      "grad_norm": 14.334996737164225,
      "learning_rate": 4.0478647378997223e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.765625,
      "logps/chosen": -508.0,
      "logps/rejected": -772.0,
      "loss": 0.2308,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.125,
      "rewards/margins": 2.890625,
      "rewards/rejected": -6.0,
      "step": 4970
    },
    {
      "epoch": 0.359488919367646,
      "grad_norm": 15.262829956566275,
      "learning_rate": 4.0429127338308154e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.9375,
      "logps/chosen": -498.0,
      "logps/rejected": -788.0,
      "loss": 0.2362,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.140625,
      "rewards/rejected": -6.1875,
      "step": 4980
    },
    {
      "epoch": 0.36021078466758105,
      "grad_norm": 10.563519061501934,
      "learning_rate": 4.037950932223832e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.75,
      "logps/chosen": -464.0,
      "logps/rejected": -800.0,
      "loss": 0.2194,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.234375,
      "rewards/rejected": -6.0625,
      "step": 4990
    },
    {
      "epoch": 0.36093264996751606,
      "grad_norm": 11.218661281499253,
      "learning_rate": 4.032979364586349e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.640625,
      "logps/chosen": -508.0,
      "logps/rejected": -832.0,
      "loss": 0.2247,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.21875,
      "rewards/margins": 3.265625,
      "rewards/rejected": -6.5,
      "step": 5000
    },
    {
      "epoch": 0.36165451526745107,
      "grad_norm": 13.269802215719091,
      "learning_rate": 4.027998062487955e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.984375,
      "logps/chosen": -496.0,
      "logps/rejected": -800.0,
      "loss": 0.2124,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.171875,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.40625,
      "step": 5010
    },
    {
      "epoch": 0.36237638056738614,
      "grad_norm": 12.229402474197753,
      "learning_rate": 4.023007057560057e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.953125,
      "logps/chosen": -524.0,
      "logps/rejected": -816.0,
      "loss": 0.2189,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.234375,
      "rewards/rejected": -6.28125,
      "step": 5020
    },
    {
      "epoch": 0.36309824586732115,
      "grad_norm": 12.754420462398162,
      "learning_rate": 4.018006381495671e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.03125,
      "logps/chosen": -502.0,
      "logps/rejected": -856.0,
      "loss": 0.196,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.71875,
      "step": 5030
    },
    {
      "epoch": 0.3638201111672562,
      "grad_norm": 10.504285924043495,
      "learning_rate": 4.012996066049229e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.796875,
      "logps/chosen": -512.0,
      "logps/rejected": -836.0,
      "loss": 0.2263,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.46875,
      "step": 5040
    },
    {
      "epoch": 0.3645419764671912,
      "grad_norm": 11.800507566664859,
      "learning_rate": 4.0079761430363676e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.609375,
      "logps/chosen": -472.0,
      "logps/rejected": -820.0,
      "loss": 0.215,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.390625,
      "rewards/rejected": -6.25,
      "step": 5050
    },
    {
      "epoch": 0.36526384176712623,
      "grad_norm": 13.958762539808074,
      "learning_rate": 4.002946644333739e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -536.0,
      "logps/rejected": -852.0,
      "loss": 0.2266,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 3.21875,
      "rewards/rejected": -6.71875,
      "step": 5060
    },
    {
      "epoch": 0.3659857070670613,
      "grad_norm": 12.380507356564978,
      "learning_rate": 3.997907601878796e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.65625,
      "logps/chosen": -552.0,
      "logps/rejected": -872.0,
      "loss": 0.211,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.625,
      "rewards/margins": 2.984375,
      "rewards/rejected": -6.625,
      "step": 5070
    },
    {
      "epoch": 0.3667075723669963,
      "grad_norm": 11.115032567540778,
      "learning_rate": 3.992859047669596e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.640625,
      "logps/chosen": -560.0,
      "logps/rejected": -828.0,
      "loss": 0.2114,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.609375,
      "rewards/margins": 2.78125,
      "rewards/rejected": -6.375,
      "step": 5080
    },
    {
      "epoch": 0.36742943766693137,
      "grad_norm": 15.555376281112473,
      "learning_rate": 3.987801013764596e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.8125,
      "logps/chosen": -532.0,
      "logps/rejected": -860.0,
      "loss": 0.1847,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.6875,
      "step": 5090
    },
    {
      "epoch": 0.3681513029668664,
      "grad_norm": 12.587838530288725,
      "learning_rate": 3.9827335322824496e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.890625,
      "logps/chosen": -544.0,
      "logps/rejected": -836.0,
      "loss": 0.222,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.6875,
      "step": 5100
    },
    {
      "epoch": 0.3688731682668014,
      "grad_norm": 12.71094661225021,
      "learning_rate": 3.977656635401805e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.8125,
      "logps/chosen": -512.0,
      "logps/rejected": -848.0,
      "loss": 0.2165,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.265625,
      "rewards/rejected": -6.65625,
      "step": 5110
    },
    {
      "epoch": 0.36959503356673645,
      "grad_norm": 12.815687420784371,
      "learning_rate": 3.972570355361093e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.90625,
      "logps/chosen": -520.0,
      "logps/rejected": -804.0,
      "loss": 0.2062,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.1875,
      "rewards/margins": 3.078125,
      "rewards/rejected": -6.25,
      "step": 5120
    },
    {
      "epoch": 0.37031689886667146,
      "grad_norm": 16.886482521916776,
      "learning_rate": 3.967474724458334e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.953125,
      "logps/chosen": -516.0,
      "logps/rejected": -804.0,
      "loss": 0.2176,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.140625,
      "rewards/rejected": -6.28125,
      "step": 5130
    },
    {
      "epoch": 0.3710387641666065,
      "grad_norm": 18.465978692620126,
      "learning_rate": 3.962369775050922e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -3.0,
      "logps/chosen": -506.0,
      "logps/rejected": -812.0,
      "loss": 0.2038,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.21875,
      "rewards/margins": 3.046875,
      "rewards/rejected": -6.28125,
      "step": 5140
    },
    {
      "epoch": 0.37176062946654154,
      "grad_norm": 9.946848999985843,
      "learning_rate": 3.957255539555426e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.9375,
      "logps/chosen": -496.0,
      "logps/rejected": -840.0,
      "loss": 0.2126,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.5,
      "step": 5150
    },
    {
      "epoch": 0.3724824947664766,
      "grad_norm": 8.869909885975988,
      "learning_rate": 3.9521320504473804e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.96875,
      "logps/chosen": -506.0,
      "logps/rejected": -840.0,
      "loss": 0.2107,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.171875,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.65625,
      "step": 5160
    },
    {
      "epoch": 0.3732043600664116,
      "grad_norm": 11.05241305262312,
      "learning_rate": 3.9469993402610823e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.90625,
      "logps/chosen": -524.0,
      "logps/rejected": -824.0,
      "loss": 0.2227,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.171875,
      "rewards/rejected": -6.4375,
      "step": 5170
    },
    {
      "epoch": 0.3739262253663466,
      "grad_norm": 11.809424538501196,
      "learning_rate": 3.941857441589382e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.96875,
      "logps/chosen": -520.0,
      "logps/rejected": -816.0,
      "loss": 0.209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.234375,
      "rewards/rejected": -6.5,
      "step": 5180
    },
    {
      "epoch": 0.3746480906662817,
      "grad_norm": 12.095044873862754,
      "learning_rate": 3.936706387083476e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.90625,
      "logps/chosen": -516.0,
      "logps/rejected": -828.0,
      "loss": 0.2225,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.5625,
      "step": 5190
    },
    {
      "epoch": 0.3753699559662167,
      "grad_norm": 10.610047069125807,
      "learning_rate": 3.9315462094527006e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.75,
      "logps/chosen": -520.0,
      "logps/rejected": -828.0,
      "loss": 0.2256,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.21875,
      "rewards/rejected": -6.59375,
      "step": 5200
    },
    {
      "epoch": 0.37609182126615176,
      "grad_norm": 11.905113713976712,
      "learning_rate": 3.9263769414643273e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.046875,
      "logps/chosen": -540.0,
      "logps/rejected": -832.0,
      "loss": 0.2065,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.140625,
      "rewards/rejected": -6.5,
      "step": 5210
    },
    {
      "epoch": 0.37681368656608677,
      "grad_norm": 10.265134471762115,
      "learning_rate": 3.9211986159433477e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.359375,
      "logps/chosen": -512.0,
      "logps/rejected": -832.0,
      "loss": 0.2038,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.34375,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.75,
      "step": 5220
    },
    {
      "epoch": 0.3775355518660218,
      "grad_norm": 12.629345681611609,
      "learning_rate": 3.91601126577227e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.578125,
      "logps/chosen": -520.0,
      "logps/rejected": -884.0,
      "loss": 0.2239,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.625,
      "rewards/rejected": -6.9375,
      "step": 5230
    },
    {
      "epoch": 0.37825741716595684,
      "grad_norm": 10.237442226956594,
      "learning_rate": 3.910814923890911e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.8125,
      "logps/chosen": -484.0,
      "logps/rejected": -832.0,
      "loss": 0.1958,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.53125,
      "step": 5240
    },
    {
      "epoch": 0.37897928246589185,
      "grad_norm": 13.798181414490367,
      "learning_rate": 3.9056096232961834e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.84375,
      "logps/chosen": -492.0,
      "logps/rejected": -812.0,
      "loss": 0.229,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.21875,
      "rewards/rejected": -6.3125,
      "step": 5250
    },
    {
      "epoch": 0.3797011477658269,
      "grad_norm": 10.21745707384059,
      "learning_rate": 3.9003953970418877e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.65625,
      "logps/chosen": -494.0,
      "logps/rejected": -792.0,
      "loss": 0.21,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.015625,
      "rewards/rejected": -6.09375,
      "step": 5260
    },
    {
      "epoch": 0.38042301306576193,
      "grad_norm": 11.393393719065127,
      "learning_rate": 3.8951722782385046e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.75,
      "logps/chosen": -510.0,
      "logps/rejected": -848.0,
      "loss": 0.2201,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.296875,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.78125,
      "step": 5270
    },
    {
      "epoch": 0.38114487836569694,
      "grad_norm": 10.548097987504747,
      "learning_rate": 3.8899403000529803e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.9375,
      "logps/chosen": -500.0,
      "logps/rejected": -780.0,
      "loss": 0.222,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.15625,
      "rewards/rejected": -6.1875,
      "step": 5280
    },
    {
      "epoch": 0.381866743665632,
      "grad_norm": 12.867088487027445,
      "learning_rate": 3.8846994957085194e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.71875,
      "logps/chosen": -500.0,
      "logps/rejected": -804.0,
      "loss": 0.21,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.15625,
      "rewards/rejected": -6.15625,
      "step": 5290
    },
    {
      "epoch": 0.382588608965567,
      "grad_norm": 12.286148720748878,
      "learning_rate": 3.879449898484374e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.8125,
      "logps/chosen": -462.0,
      "logps/rejected": -784.0,
      "loss": 0.208,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.09375,
      "rewards/rejected": -5.9375,
      "step": 5300
    },
    {
      "epoch": 0.3833104742655021,
      "grad_norm": 13.093734711877175,
      "learning_rate": 3.874191541715629e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.734375,
      "logps/chosen": -536.0,
      "logps/rejected": -872.0,
      "loss": 0.204,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.03125,
      "rewards/margins": 3.671875,
      "rewards/rejected": -6.6875,
      "step": 5310
    },
    {
      "epoch": 0.3840323395654371,
      "grad_norm": 10.988549200871912,
      "learning_rate": 3.868924458792994e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.9375,
      "logps/chosen": -532.0,
      "logps/rejected": -772.0,
      "loss": 0.2199,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.21875,
      "rewards/margins": 2.84375,
      "rewards/rejected": -6.0625,
      "step": 5320
    },
    {
      "epoch": 0.3847542048653721,
      "grad_norm": 16.568980130849713,
      "learning_rate": 3.86364868316259e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.734375,
      "logps/chosen": -492.0,
      "logps/rejected": -880.0,
      "loss": 0.2117,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.859375,
      "rewards/rejected": -6.75,
      "step": 5330
    },
    {
      "epoch": 0.38547607016530716,
      "grad_norm": 10.765260053780459,
      "learning_rate": 3.8583642483257374e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.828125,
      "logps/chosen": -528.0,
      "logps/rejected": -860.0,
      "loss": 0.1971,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.75,
      "step": 5340
    },
    {
      "epoch": 0.38619793546524217,
      "grad_norm": 12.039170231237595,
      "learning_rate": 3.85307118783874e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -3.0,
      "logps/chosen": -494.0,
      "logps/rejected": -824.0,
      "loss": 0.2112,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.5,
      "step": 5350
    },
    {
      "epoch": 0.38691980076517724,
      "grad_norm": 9.784054493209107,
      "learning_rate": 3.8477695353126785e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.75,
      "logps/chosen": -504.0,
      "logps/rejected": -792.0,
      "loss": 0.2006,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.015625,
      "rewards/rejected": -6.09375,
      "step": 5360
    },
    {
      "epoch": 0.38764166606511224,
      "grad_norm": 12.377794242288434,
      "learning_rate": 3.8424593244131897e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.78125,
      "logps/chosen": -506.0,
      "logps/rejected": -824.0,
      "loss": 0.2167,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.375,
      "step": 5370
    },
    {
      "epoch": 0.3883635313650473,
      "grad_norm": 9.873815426342125,
      "learning_rate": 3.83714058886026e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.890625,
      "logps/chosen": -468.0,
      "logps/rejected": -772.0,
      "loss": 0.1961,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.171875,
      "rewards/rejected": -6.03125,
      "step": 5380
    },
    {
      "epoch": 0.3890853966649823,
      "grad_norm": 9.106777248596611,
      "learning_rate": 3.8318133624280046e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.1875,
      "logps/chosen": -462.0,
      "logps/rejected": -768.0,
      "loss": 0.2067,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.234375,
      "rewards/rejected": -6.09375,
      "step": 5390
    },
    {
      "epoch": 0.38980726196491733,
      "grad_norm": 14.131110638895713,
      "learning_rate": 3.826477678944457e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.109375,
      "logps/chosen": -458.0,
      "logps/rejected": -788.0,
      "loss": 0.2183,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.3125,
      "step": 5400
    },
    {
      "epoch": 0.3905291272648524,
      "grad_norm": 11.435392914807423,
      "learning_rate": 3.821133572291354e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.34375,
      "logps/chosen": -488.0,
      "logps/rejected": -768.0,
      "loss": 0.2131,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.09375,
      "rewards/margins": 2.9375,
      "rewards/rejected": -6.03125,
      "step": 5410
    },
    {
      "epoch": 0.3912509925647874,
      "grad_norm": 11.023660535889212,
      "learning_rate": 3.8157810764039187e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.078125,
      "logps/chosen": -464.0,
      "logps/rejected": -780.0,
      "loss": 0.2146,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.296875,
      "rewards/rejected": -6.0625,
      "step": 5420
    },
    {
      "epoch": 0.39197285786472247,
      "grad_norm": 10.258000804825487,
      "learning_rate": 3.810420225270647e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.171875,
      "logps/chosen": -498.0,
      "logps/rejected": -812.0,
      "loss": 0.2181,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.15625,
      "rewards/rejected": -6.28125,
      "step": 5430
    },
    {
      "epoch": 0.3926947231646575,
      "grad_norm": 9.947944794291038,
      "learning_rate": 3.80505105293309e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.15625,
      "logps/chosen": -496.0,
      "logps/rejected": -808.0,
      "loss": 0.2085,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.21875,
      "rewards/rejected": -6.3125,
      "step": 5440
    },
    {
      "epoch": 0.3934165884645925,
      "grad_norm": 12.401940144388783,
      "learning_rate": 3.799673593485638e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.78125,
      "logps/chosen": -532.0,
      "logps/rejected": -876.0,
      "loss": 0.1936,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.421875,
      "rewards/margins": 3.390625,
      "rewards/rejected": -6.8125,
      "step": 5450
    },
    {
      "epoch": 0.39413845376452755,
      "grad_norm": 15.018109846546098,
      "learning_rate": 3.794287881075307e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.21875,
      "logps/chosen": -536.0,
      "logps/rejected": -868.0,
      "loss": 0.1941,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.296875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.875,
      "step": 5460
    },
    {
      "epoch": 0.39486031906446256,
      "grad_norm": 8.998670584561829,
      "learning_rate": 3.788893949901517e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.859375,
      "logps/chosen": -516.0,
      "logps/rejected": -860.0,
      "loss": 0.214,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.390625,
      "rewards/rejected": -6.53125,
      "step": 5470
    },
    {
      "epoch": 0.3955821843643976,
      "grad_norm": 10.840763621901418,
      "learning_rate": 3.783491834215879e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.953125,
      "logps/chosen": -502.0,
      "logps/rejected": -816.0,
      "loss": 0.2109,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.34375,
      "step": 5480
    },
    {
      "epoch": 0.39630404966433264,
      "grad_norm": 10.819083335965448,
      "learning_rate": 3.778081568321976e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.90625,
      "logps/chosen": -500.0,
      "logps/rejected": -860.0,
      "loss": 0.2125,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.203125,
      "rewards/margins": 3.671875,
      "rewards/rejected": -6.875,
      "step": 5490
    },
    {
      "epoch": 0.39702591496426765,
      "grad_norm": 12.55817491076989,
      "learning_rate": 3.772663186575143e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.75,
      "logps/chosen": -472.0,
      "logps/rejected": -796.0,
      "loss": 0.2125,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.125,
      "step": 5500
    },
    {
      "epoch": 0.3977477802642027,
      "grad_norm": 14.558203022380027,
      "learning_rate": 3.7672367233822537e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.90625,
      "logps/chosen": -480.0,
      "logps/rejected": -792.0,
      "loss": 0.2013,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.375,
      "rewards/rejected": -6.21875,
      "step": 5510
    },
    {
      "epoch": 0.3984696455641377,
      "grad_norm": 7.868322918792799,
      "learning_rate": 3.761802213201498e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.65625,
      "logps/chosen": -472.0,
      "logps/rejected": -880.0,
      "loss": 0.183,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.796875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -6.875,
      "step": 5520
    },
    {
      "epoch": 0.3991915108640728,
      "grad_norm": 10.425096764755386,
      "learning_rate": 3.756359690542163e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.78125,
      "logps/chosen": -516.0,
      "logps/rejected": -836.0,
      "loss": 0.1992,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.1875,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.6875,
      "step": 5530
    },
    {
      "epoch": 0.3999133761640078,
      "grad_norm": 12.203744458714734,
      "learning_rate": 3.750909189964417e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.71875,
      "logps/chosen": -460.0,
      "logps/rejected": -792.0,
      "loss": 0.2178,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.78125,
      "rewards/margins": 3.28125,
      "rewards/rejected": -6.0625,
      "step": 5540
    },
    {
      "epoch": 0.4006352414639428,
      "grad_norm": 10.670566216965154,
      "learning_rate": 3.7454507460790893e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.953125,
      "logps/chosen": -474.0,
      "logps/rejected": -776.0,
      "loss": 0.2144,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.0625,
      "rewards/rejected": -5.96875,
      "step": 5550
    },
    {
      "epoch": 0.40135710676387787,
      "grad_norm": 10.213756908552064,
      "learning_rate": 3.739984393547446e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.84375,
      "logps/chosen": -510.0,
      "logps/rejected": -852.0,
      "loss": 0.1854,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.296875,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.75,
      "step": 5560
    },
    {
      "epoch": 0.4020789720638129,
      "grad_norm": 11.319478848859855,
      "learning_rate": 3.7345101670809755e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.046875,
      "logps/chosen": -556.0,
      "logps/rejected": -860.0,
      "loss": 0.2146,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.5625,
      "rewards/margins": 3.390625,
      "rewards/rejected": -6.96875,
      "step": 5570
    },
    {
      "epoch": 0.40280083736374794,
      "grad_norm": 9.710905230048406,
      "learning_rate": 3.7290281014411655e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.6875,
      "logps/chosen": -490.0,
      "logps/rejected": -844.0,
      "loss": 0.1944,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.625,
      "rewards/rejected": -6.71875,
      "step": 5580
    },
    {
      "epoch": 0.40352270266368295,
      "grad_norm": 12.87721234406461,
      "learning_rate": 3.723538231439284e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.828125,
      "logps/chosen": -528.0,
      "logps/rejected": -824.0,
      "loss": 0.2206,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.203125,
      "rewards/rejected": -6.46875,
      "step": 5590
    },
    {
      "epoch": 0.40424456796361796,
      "grad_norm": 12.436521533356522,
      "learning_rate": 3.7180405919361547e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.71875,
      "logps/chosen": -444.0,
      "logps/rejected": -772.0,
      "loss": 0.2071,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.234375,
      "rewards/rejected": -5.96875,
      "step": 5600
    },
    {
      "epoch": 0.404966433263553,
      "grad_norm": 14.691244712293978,
      "learning_rate": 3.7125352178419396e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.953125,
      "logps/chosen": -472.0,
      "logps/rejected": -764.0,
      "loss": 0.2314,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.15625,
      "rewards/rejected": -6.03125,
      "step": 5610
    },
    {
      "epoch": 0.40568829856348804,
      "grad_norm": 9.328884808464844,
      "learning_rate": 3.707022144115914e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.859375,
      "logps/chosen": -472.0,
      "logps/rejected": -788.0,
      "loss": 0.2242,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.25,
      "step": 5620
    },
    {
      "epoch": 0.4064101638634231,
      "grad_norm": 10.569996322993527,
      "learning_rate": 3.701501405766248e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.703125,
      "logps/chosen": -462.0,
      "logps/rejected": -780.0,
      "loss": 0.2041,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.21875,
      "step": 5630
    },
    {
      "epoch": 0.4071320291633581,
      "grad_norm": 11.414876419040468,
      "learning_rate": 3.69597303784978e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.125,
      "logps/chosen": -506.0,
      "logps/rejected": -840.0,
      "loss": 0.1775,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.5625,
      "step": 5640
    },
    {
      "epoch": 0.4078538944632932,
      "grad_norm": 12.324792066522699,
      "learning_rate": 3.690437075471797e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.953125,
      "logps/chosen": -510.0,
      "logps/rejected": -872.0,
      "loss": 0.1947,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.765625,
      "rewards/rejected": -6.875,
      "step": 5650
    },
    {
      "epoch": 0.4085757597632282,
      "grad_norm": 12.112356187543522,
      "learning_rate": 3.684893553785815e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.984375,
      "logps/chosen": -512.0,
      "logps/rejected": -852.0,
      "loss": 0.2023,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.515625,
      "rewards/rejected": -6.8125,
      "step": 5660
    },
    {
      "epoch": 0.4092976250631632,
      "grad_norm": 11.277196995269652,
      "learning_rate": 3.6793425079933446e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.875,
      "logps/chosen": -494.0,
      "logps/rejected": -808.0,
      "loss": 0.2206,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.171875,
      "rewards/rejected": -6.21875,
      "step": 5670
    },
    {
      "epoch": 0.41001949036309826,
      "grad_norm": 18.310688097082878,
      "learning_rate": 3.67378397334368e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.8125,
      "logps/chosen": -492.0,
      "logps/rejected": -792.0,
      "loss": 0.209,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.171875,
      "rewards/rejected": -6.125,
      "step": 5680
    },
    {
      "epoch": 0.41074135566303327,
      "grad_norm": 13.423980936871521,
      "learning_rate": 3.668217985133668e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.984375,
      "logps/chosen": -486.0,
      "logps/rejected": -816.0,
      "loss": 0.1865,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.09375,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.40625,
      "step": 5690
    },
    {
      "epoch": 0.41146322096296833,
      "grad_norm": 13.492020445634386,
      "learning_rate": 3.662644578707486e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.796875,
      "logps/chosen": -500.0,
      "logps/rejected": -828.0,
      "loss": 0.2063,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.5,
      "step": 5700
    },
    {
      "epoch": 0.41218508626290334,
      "grad_norm": 12.371508392995993,
      "learning_rate": 3.657063789456418e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.921875,
      "logps/chosen": -504.0,
      "logps/rejected": -852.0,
      "loss": 0.1964,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.234375,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.78125,
      "step": 5710
    },
    {
      "epoch": 0.41290695156283835,
      "grad_norm": 12.631965011270024,
      "learning_rate": 3.6514756528186283e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.859375,
      "logps/chosen": -512.0,
      "logps/rejected": -916.0,
      "loss": 0.1867,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.234375,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.34375,
      "step": 5720
    },
    {
      "epoch": 0.4136288168627734,
      "grad_norm": 9.834736129561511,
      "learning_rate": 3.645880204278936e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.96875,
      "logps/chosen": -516.0,
      "logps/rejected": -812.0,
      "loss": 0.2302,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.203125,
      "rewards/rejected": -6.3125,
      "step": 5730
    },
    {
      "epoch": 0.41435068216270843,
      "grad_norm": 10.15571880138521,
      "learning_rate": 3.640277479368594e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.828125,
      "logps/chosen": -482.0,
      "logps/rejected": -748.0,
      "loss": 0.218,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.90625,
      "rewards/margins": 2.796875,
      "rewards/rejected": -5.71875,
      "step": 5740
    },
    {
      "epoch": 0.4150725474626435,
      "grad_norm": 9.06684694314198,
      "learning_rate": 3.6346675136650595e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.71875,
      "logps/chosen": -464.0,
      "logps/rejected": -776.0,
      "loss": 0.1819,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.671875,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.03125,
      "step": 5750
    },
    {
      "epoch": 0.4157944127625785,
      "grad_norm": 11.739302128987262,
      "learning_rate": 3.629050342791766e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.796875,
      "logps/chosen": -502.0,
      "logps/rejected": -808.0,
      "loss": 0.2298,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.375,
      "step": 5760
    },
    {
      "epoch": 0.4165162780625135,
      "grad_norm": 9.964994685099832,
      "learning_rate": 3.6234260024179033e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.953125,
      "logps/chosen": -488.0,
      "logps/rejected": -828.0,
      "loss": 0.1928,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.515625,
      "rewards/rejected": -6.5,
      "step": 5770
    },
    {
      "epoch": 0.4172381433624486,
      "grad_norm": 10.113769973607042,
      "learning_rate": 3.6177945282581866e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.03125,
      "logps/chosen": -468.0,
      "logps/rejected": -808.0,
      "loss": 0.1992,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.40625,
      "step": 5780
    },
    {
      "epoch": 0.4179600086623836,
      "grad_norm": 10.763893681567684,
      "learning_rate": 3.6121559560726313e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.625,
      "logps/chosen": -496.0,
      "logps/rejected": -876.0,
      "loss": 0.1913,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.75,
      "rewards/rejected": -6.75,
      "step": 5790
    },
    {
      "epoch": 0.41868187396231865,
      "grad_norm": 15.238875588837477,
      "learning_rate": 3.6065103216663237e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.0,
      "logps/chosen": -532.0,
      "logps/rejected": -880.0,
      "loss": 0.222,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.90625,
      "step": 5800
    },
    {
      "epoch": 0.41940373926225366,
      "grad_norm": 10.978531361486608,
      "learning_rate": 3.600857660889199e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.90625,
      "logps/chosen": -472.0,
      "logps/rejected": -808.0,
      "loss": 0.1873,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.25,
      "step": 5810
    },
    {
      "epoch": 0.42012560456218867,
      "grad_norm": 12.579798047254075,
      "learning_rate": 3.5951980096358055e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.9375,
      "logps/chosen": -470.0,
      "logps/rejected": -780.0,
      "loss": 0.2153,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.90625,
      "rewards/margins": 3.1875,
      "rewards/rejected": -6.09375,
      "step": 5820
    },
    {
      "epoch": 0.42084746986212374,
      "grad_norm": 8.999361143548848,
      "learning_rate": 3.589531403845085e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.90625,
      "logps/chosen": -480.0,
      "logps/rejected": -792.0,
      "loss": 0.2022,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.03125,
      "rewards/margins": 3.203125,
      "rewards/rejected": -6.21875,
      "step": 5830
    },
    {
      "epoch": 0.42156933516205874,
      "grad_norm": 14.995597595576596,
      "learning_rate": 3.5838578795001393e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.796875,
      "logps/chosen": -528.0,
      "logps/rejected": -928.0,
      "loss": 0.1944,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.15625,
      "step": 5840
    },
    {
      "epoch": 0.4222912004619938,
      "grad_norm": 10.35974050407089,
      "learning_rate": 3.578177472628002e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.875,
      "logps/chosen": -532.0,
      "logps/rejected": -872.0,
      "loss": 0.207,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.84375,
      "step": 5850
    },
    {
      "epoch": 0.4230130657619288,
      "grad_norm": 13.38342118602672,
      "learning_rate": 3.572490219299414e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.8125,
      "logps/chosen": -532.0,
      "logps/rejected": -864.0,
      "loss": 0.2185,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.96875,
      "step": 5860
    },
    {
      "epoch": 0.42373493106186383,
      "grad_norm": 12.654435485678842,
      "learning_rate": 3.5667961556285876e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.921875,
      "logps/chosen": -520.0,
      "logps/rejected": -840.0,
      "loss": 0.1989,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.296875,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.65625,
      "step": 5870
    },
    {
      "epoch": 0.4244567963617989,
      "grad_norm": 14.499148980207245,
      "learning_rate": 3.561095317772984e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.953125,
      "logps/chosen": -528.0,
      "logps/rejected": -872.0,
      "loss": 0.2177,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.84375,
      "step": 5880
    },
    {
      "epoch": 0.4251786616617339,
      "grad_norm": 15.212124294208948,
      "learning_rate": 3.5553877419330797e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.6875,
      "logps/chosen": -512.0,
      "logps/rejected": -860.0,
      "loss": 0.2003,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.640625,
      "rewards/rejected": -6.65625,
      "step": 5890
    },
    {
      "epoch": 0.42590052696166897,
      "grad_norm": 11.854659535758758,
      "learning_rate": 3.5496734643521364e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.5625,
      "logps/chosen": -496.0,
      "logps/rejected": -844.0,
      "loss": 0.2172,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.59375,
      "step": 5900
    },
    {
      "epoch": 0.426622392261604,
      "grad_norm": 13.725670530550158,
      "learning_rate": 3.543952521315975e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.921875,
      "logps/chosen": -474.0,
      "logps/rejected": -808.0,
      "loss": 0.1997,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.390625,
      "rewards/rejected": -6.3125,
      "step": 5910
    },
    {
      "epoch": 0.42734425756153904,
      "grad_norm": 12.069282328667542,
      "learning_rate": 3.538224949152738e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.765625,
      "logps/chosen": -498.0,
      "logps/rejected": -856.0,
      "loss": 0.2074,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.75,
      "rewards/rejected": -6.84375,
      "step": 5920
    },
    {
      "epoch": 0.42806612286147405,
      "grad_norm": 11.60358571693507,
      "learning_rate": 3.5324907842326676e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.078125,
      "logps/chosen": -502.0,
      "logps/rejected": -808.0,
      "loss": 0.191,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.46875,
      "step": 5930
    },
    {
      "epoch": 0.42878798816140906,
      "grad_norm": 11.87784074274127,
      "learning_rate": 3.526750062967866e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.15625,
      "logps/chosen": -516.0,
      "logps/rejected": -840.0,
      "loss": 0.2039,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.21875,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.6875,
      "step": 5940
    },
    {
      "epoch": 0.4295098534613441,
      "grad_norm": 11.356699849333223,
      "learning_rate": 3.5210028218120714e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.6875,
      "logps/chosen": -506.0,
      "logps/rejected": -880.0,
      "loss": 0.1804,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.09375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -6.90625,
      "step": 5950
    },
    {
      "epoch": 0.43023171876127914,
      "grad_norm": 13.756690255359302,
      "learning_rate": 3.515249097260422e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.03125,
      "logps/chosen": -488.0,
      "logps/rejected": -796.0,
      "loss": 0.2183,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.3125,
      "step": 5960
    },
    {
      "epoch": 0.4309535840612142,
      "grad_norm": 11.527883922366224,
      "learning_rate": 3.509488925849227e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.9375,
      "logps/chosen": -500.0,
      "logps/rejected": -856.0,
      "loss": 0.2044,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.203125,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.6875,
      "step": 5970
    },
    {
      "epoch": 0.4316754493611492,
      "grad_norm": 10.187490602603694,
      "learning_rate": 3.503722344155731e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.75,
      "logps/chosen": -502.0,
      "logps/rejected": -800.0,
      "loss": 0.1911,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.109375,
      "rewards/rejected": -6.125,
      "step": 5980
    },
    {
      "epoch": 0.4323973146610842,
      "grad_norm": 13.954773620685325,
      "learning_rate": 3.4979493887978864e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.296875,
      "logps/chosen": -490.0,
      "logps/rejected": -824.0,
      "loss": 0.2106,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.6875,
      "step": 5990
    },
    {
      "epoch": 0.4331191799610193,
      "grad_norm": 15.079095762832088,
      "learning_rate": 3.4921700964341173e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.09375,
      "logps/chosen": -512.0,
      "logps/rejected": -852.0,
      "loss": 0.2019,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.640625,
      "rewards/rejected": -6.71875,
      "step": 6000
    },
    {
      "epoch": 0.4338410452609543,
      "grad_norm": 15.273149752824992,
      "learning_rate": 3.486384503763089e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.453125,
      "logps/chosen": -516.0,
      "logps/rejected": -832.0,
      "loss": 0.1727,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.34375,
      "rewards/margins": 3.515625,
      "rewards/rejected": -6.875,
      "step": 6010
    },
    {
      "epoch": 0.43456291056088936,
      "grad_norm": 11.508047282440224,
      "learning_rate": 3.480592647523472e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.25,
      "logps/chosen": -498.0,
      "logps/rejected": -828.0,
      "loss": 0.1811,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.53125,
      "rewards/rejected": -6.5625,
      "step": 6020
    },
    {
      "epoch": 0.43528477586082437,
      "grad_norm": 12.751055472893686,
      "learning_rate": 3.4747945644937133e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.015625,
      "logps/chosen": -470.0,
      "logps/rejected": -868.0,
      "loss": 0.2064,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.015625,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.0,
      "step": 6030
    },
    {
      "epoch": 0.4360066411607594,
      "grad_norm": 13.513388967005664,
      "learning_rate": 3.468990291491799e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.046875,
      "logps/chosen": -472.0,
      "logps/rejected": -792.0,
      "loss": 0.2004,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.28125,
      "rewards/rejected": -6.21875,
      "step": 6040
    },
    {
      "epoch": 0.43672850646069444,
      "grad_norm": 10.387151451133871,
      "learning_rate": 3.46317986537502e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.75,
      "logps/chosen": -504.0,
      "logps/rejected": -936.0,
      "loss": 0.1948,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.15625,
      "rewards/margins": 4.25,
      "rewards/rejected": -7.40625,
      "step": 6050
    },
    {
      "epoch": 0.43745037176062945,
      "grad_norm": 8.796497268690288,
      "learning_rate": 3.4573633230397414e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.921875,
      "logps/chosen": -524.0,
      "logps/rejected": -888.0,
      "loss": 0.2073,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.09375,
      "step": 6060
    },
    {
      "epoch": 0.4381722370605645,
      "grad_norm": 11.290644095910826,
      "learning_rate": 3.451540701421167e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -3.03125,
      "logps/chosen": -494.0,
      "logps/rejected": -804.0,
      "loss": 0.2023,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.3125,
      "step": 6070
    },
    {
      "epoch": 0.4388941023604995,
      "grad_norm": 12.723042373928061,
      "learning_rate": 3.445712037493104e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.875,
      "logps/chosen": -508.0,
      "logps/rejected": -892.0,
      "loss": 0.1968,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.0,
      "step": 6080
    },
    {
      "epoch": 0.43961596766043454,
      "grad_norm": 11.213668608337127,
      "learning_rate": 3.439877368267725e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.703125,
      "logps/chosen": -502.0,
      "logps/rejected": -896.0,
      "loss": 0.2131,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.25,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.25,
      "step": 6090
    },
    {
      "epoch": 0.4403378329603696,
      "grad_norm": 8.151790599635312,
      "learning_rate": 3.434036730795342e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -3.046875,
      "logps/chosen": -498.0,
      "logps/rejected": -848.0,
      "loss": 0.1768,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -6.875,
      "step": 6100
    },
    {
      "epoch": 0.4410596982603046,
      "grad_norm": 14.136832037036582,
      "learning_rate": 3.428190162164162e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.0,
      "logps/chosen": -488.0,
      "logps/rejected": -880.0,
      "loss": 0.1698,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.109375,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.09375,
      "step": 6110
    },
    {
      "epoch": 0.4417815635602397,
      "grad_norm": 14.306447564407692,
      "learning_rate": 3.422337699500056e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.015625,
      "logps/chosen": -520.0,
      "logps/rejected": -856.0,
      "loss": 0.2156,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.8125,
      "step": 6120
    },
    {
      "epoch": 0.4425034288601747,
      "grad_norm": 13.282163586374544,
      "learning_rate": 3.4164793799663207e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.640625,
      "logps/chosen": -472.0,
      "logps/rejected": -828.0,
      "loss": 0.2163,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.828125,
      "rewards/margins": 3.625,
      "rewards/rejected": -6.46875,
      "step": 6130
    },
    {
      "epoch": 0.44322529416010975,
      "grad_norm": 21.04713233577766,
      "learning_rate": 3.410615240763446e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.171875,
      "logps/chosen": -474.0,
      "logps/rejected": -824.0,
      "loss": 0.222,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.90625,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.46875,
      "step": 6140
    },
    {
      "epoch": 0.44394715946004476,
      "grad_norm": 12.677512181521996,
      "learning_rate": 3.4047453191288775e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.703125,
      "logps/chosen": -500.0,
      "logps/rejected": -848.0,
      "loss": 0.2061,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.65625,
      "step": 6150
    },
    {
      "epoch": 0.44466902475997977,
      "grad_norm": 8.758567001037813,
      "learning_rate": 3.3988696523367777e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.703125,
      "logps/chosen": -464.0,
      "logps/rejected": -820.0,
      "loss": 0.1856,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.3125,
      "step": 6160
    },
    {
      "epoch": 0.44539089005991483,
      "grad_norm": 12.427278771127583,
      "learning_rate": 3.3929882776977905e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.03125,
      "logps/chosen": -506.0,
      "logps/rejected": -812.0,
      "loss": 0.2142,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.296875,
      "rewards/rejected": -6.4375,
      "step": 6170
    },
    {
      "epoch": 0.44611275535984984,
      "grad_norm": 9.902708277277068,
      "learning_rate": 3.387101232558807e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.75,
      "logps/chosen": -484.0,
      "logps/rejected": -832.0,
      "loss": 0.2071,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.53125,
      "step": 6180
    },
    {
      "epoch": 0.4468346206597849,
      "grad_norm": 14.835961454334548,
      "learning_rate": 3.381208554302723e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.875,
      "logps/chosen": -512.0,
      "logps/rejected": -852.0,
      "loss": 0.2071,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.234375,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.71875,
      "step": 6190
    },
    {
      "epoch": 0.4475564859597199,
      "grad_norm": 11.81721985400321,
      "learning_rate": 3.3753102803482094e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.828125,
      "logps/chosen": -504.0,
      "logps/rejected": -820.0,
      "loss": 0.1805,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.390625,
      "rewards/rejected": -6.4375,
      "step": 6200
    },
    {
      "epoch": 0.44827835125965493,
      "grad_norm": 13.8942280333494,
      "learning_rate": 3.3694064481494655e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.1875,
      "logps/chosen": -528.0,
      "logps/rejected": -904.0,
      "loss": 0.2253,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.25,
      "step": 6210
    },
    {
      "epoch": 0.44900021655959,
      "grad_norm": 13.539630755424914,
      "learning_rate": 3.3634970951959867e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.890625,
      "logps/chosen": -516.0,
      "logps/rejected": -824.0,
      "loss": 0.2019,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.171875,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.53125,
      "step": 6220
    },
    {
      "epoch": 0.449722081859525,
      "grad_norm": 12.146604763485005,
      "learning_rate": 3.357582259012327e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.140625,
      "logps/chosen": -512.0,
      "logps/rejected": -816.0,
      "loss": 0.1992,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.21875,
      "rewards/rejected": -6.46875,
      "step": 6230
    },
    {
      "epoch": 0.45044394715946007,
      "grad_norm": 11.192268903416704,
      "learning_rate": 3.351661977157859e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.75,
      "logps/chosen": -494.0,
      "logps/rejected": -896.0,
      "loss": 0.2093,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.9375,
      "rewards/rejected": -7.0625,
      "step": 6240
    },
    {
      "epoch": 0.4511658124593951,
      "grad_norm": 9.707973697237056,
      "learning_rate": 3.3457362872265324e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.96875,
      "logps/chosen": -516.0,
      "logps/rejected": -804.0,
      "loss": 0.1835,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.09375,
      "rewards/rejected": -6.4375,
      "step": 6250
    },
    {
      "epoch": 0.4518876777593301,
      "grad_norm": 13.397903995479625,
      "learning_rate": 3.339805226846642e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.03125,
      "logps/chosen": -516.0,
      "logps/rejected": -864.0,
      "loss": 0.2085,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.6875,
      "rewards/rejected": -6.9375,
      "step": 6260
    },
    {
      "epoch": 0.45260954305926515,
      "grad_norm": 10.055117415732347,
      "learning_rate": 3.3338688336805823e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.921875,
      "logps/chosen": -528.0,
      "logps/rejected": -848.0,
      "loss": 0.1752,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.515625,
      "rewards/rejected": -6.8125,
      "step": 6270
    },
    {
      "epoch": 0.45333140835920016,
      "grad_norm": 10.657525879965881,
      "learning_rate": 3.327927145424613e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.90625,
      "logps/chosen": -512.0,
      "logps/rejected": -852.0,
      "loss": 0.2059,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.75,
      "rewards/rejected": -6.8125,
      "step": 6280
    },
    {
      "epoch": 0.4540532736591352,
      "grad_norm": 13.0556075301802,
      "learning_rate": 3.321980199808616e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -3.125,
      "logps/chosen": -472.0,
      "logps/rejected": -796.0,
      "loss": 0.2072,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.34375,
      "step": 6290
    },
    {
      "epoch": 0.45477513895907024,
      "grad_norm": 13.833213803327755,
      "learning_rate": 3.316028034595861e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.140625,
      "logps/chosen": -478.0,
      "logps/rejected": -836.0,
      "loss": 0.1834,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.625,
      "rewards/rejected": -6.5625,
      "step": 6300
    },
    {
      "epoch": 0.45549700425900524,
      "grad_norm": 14.618968175924426,
      "learning_rate": 3.3100706875827576e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.75,
      "logps/chosen": -532.0,
      "logps/rejected": -884.0,
      "loss": 0.1997,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.234375,
      "rewards/margins": 3.609375,
      "rewards/rejected": -6.84375,
      "step": 6310
    },
    {
      "epoch": 0.4562188695589403,
      "grad_norm": 13.626769137496037,
      "learning_rate": 3.304108196598624e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.109375,
      "logps/chosen": -512.0,
      "logps/rejected": -812.0,
      "loss": 0.2001,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.34375,
      "rewards/margins": 3.171875,
      "rewards/rejected": -6.5,
      "step": 6320
    },
    {
      "epoch": 0.4569407348588753,
      "grad_norm": 10.697102550118315,
      "learning_rate": 3.29814059950544e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.875,
      "logps/chosen": -536.0,
      "logps/rejected": -900.0,
      "loss": 0.1912,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 3.671875,
      "rewards/rejected": -7.15625,
      "step": 6330
    },
    {
      "epoch": 0.4576626001588104,
      "grad_norm": 11.198300453442561,
      "learning_rate": 3.2921679341976104e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -3.03125,
      "logps/chosen": -536.0,
      "logps/rejected": -852.0,
      "loss": 0.2069,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.5625,
      "rewards/margins": 3.234375,
      "rewards/rejected": -6.78125,
      "step": 6340
    },
    {
      "epoch": 0.4583844654587454,
      "grad_norm": 13.19395749618235,
      "learning_rate": 3.286190238601725e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.78125,
      "logps/chosen": -482.0,
      "logps/rejected": -828.0,
      "loss": 0.1902,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.65625,
      "rewards/rejected": -6.59375,
      "step": 6350
    },
    {
      "epoch": 0.4591063307586804,
      "grad_norm": 19.133376155075073,
      "learning_rate": 3.280207550676312e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.953125,
      "logps/chosen": -504.0,
      "logps/rejected": -844.0,
      "loss": 0.2007,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.09375,
      "rewards/margins": 3.515625,
      "rewards/rejected": -6.59375,
      "step": 6360
    },
    {
      "epoch": 0.45982819605861547,
      "grad_norm": 14.760358940244657,
      "learning_rate": 3.274219908411605e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.765625,
      "logps/chosen": -512.0,
      "logps/rejected": -896.0,
      "loss": 0.2045,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.1875,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.09375,
      "step": 6370
    },
    {
      "epoch": 0.4605500613585505,
      "grad_norm": 12.521784369440143,
      "learning_rate": 3.2682273498292957e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.78125,
      "logps/chosen": -512.0,
      "logps/rejected": -880.0,
      "loss": 0.1975,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -6.90625,
      "step": 6380
    },
    {
      "epoch": 0.46127192665848554,
      "grad_norm": 14.126322134135277,
      "learning_rate": 3.262229912982295e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.046875,
      "logps/chosen": -480.0,
      "logps/rejected": -848.0,
      "loss": 0.1931,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.890625,
      "rewards/margins": 3.765625,
      "rewards/rejected": -6.65625,
      "step": 6390
    },
    {
      "epoch": 0.46199379195842055,
      "grad_norm": 13.732680291554699,
      "learning_rate": 3.2562276359544917e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.03125,
      "logps/chosen": -540.0,
      "logps/rejected": -888.0,
      "loss": 0.1952,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.234375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.09375,
      "step": 6400
    },
    {
      "epoch": 0.4627156572583556,
      "grad_norm": 9.017759505379763,
      "learning_rate": 3.2502205568605087e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.0625,
      "logps/chosen": -520.0,
      "logps/rejected": -824.0,
      "loss": 0.1927,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.625,
      "step": 6410
    },
    {
      "epoch": 0.4634375225582906,
      "grad_norm": 13.48596822807741,
      "learning_rate": 3.244208713845461e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.84375,
      "logps/chosen": -498.0,
      "logps/rejected": -852.0,
      "loss": 0.1991,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.9375,
      "rewards/rejected": -6.78125,
      "step": 6420
    },
    {
      "epoch": 0.46415938785822564,
      "grad_norm": 12.203264154954496,
      "learning_rate": 3.2381921450847187e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.875,
      "logps/chosen": -478.0,
      "logps/rejected": -828.0,
      "loss": 0.1889,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.609375,
      "rewards/rejected": -6.59375,
      "step": 6430
    },
    {
      "epoch": 0.4648812531581607,
      "grad_norm": 15.592235255434165,
      "learning_rate": 3.2321708887836573e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.96875,
      "logps/chosen": -512.0,
      "logps/rejected": -852.0,
      "loss": 0.2095,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.78125,
      "step": 6440
    },
    {
      "epoch": 0.4656031184580957,
      "grad_norm": 12.253310313100313,
      "learning_rate": 3.226144983177419e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.90625,
      "logps/chosen": -484.0,
      "logps/rejected": -796.0,
      "loss": 0.2069,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.9375,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.25,
      "step": 6450
    },
    {
      "epoch": 0.4663249837580308,
      "grad_norm": 10.878585895275169,
      "learning_rate": 3.220114466530668e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.8125,
      "logps/chosen": -492.0,
      "logps/rejected": -804.0,
      "loss": 0.1976,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 3.1875,
      "rewards/rejected": -6.28125,
      "step": 6460
    },
    {
      "epoch": 0.4670468490579658,
      "grad_norm": 13.504290139362213,
      "learning_rate": 3.214079377137352e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.9375,
      "logps/chosen": -508.0,
      "logps/rejected": -860.0,
      "loss": 0.1877,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.921875,
      "rewards/rejected": -6.9375,
      "step": 6470
    },
    {
      "epoch": 0.4677687143579008,
      "grad_norm": 9.13753660742345,
      "learning_rate": 3.2080397533204526e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -3.015625,
      "logps/chosen": -502.0,
      "logps/rejected": -836.0,
      "loss": 0.1717,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.171875,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.65625,
      "step": 6480
    },
    {
      "epoch": 0.46849057965783586,
      "grad_norm": 17.306909383872124,
      "learning_rate": 3.201995633431747e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.21875,
      "logps/chosen": -512.0,
      "logps/rejected": -824.0,
      "loss": 0.2095,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.6875,
      "step": 6490
    },
    {
      "epoch": 0.46921244495777087,
      "grad_norm": 12.077252528347664,
      "learning_rate": 3.1959470558515613e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -3.0625,
      "logps/chosen": -532.0,
      "logps/rejected": -816.0,
      "loss": 0.2089,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.125,
      "rewards/rejected": -6.5,
      "step": 6500
    },
    {
      "epoch": 0.46993431025770593,
      "grad_norm": 17.372805335349508,
      "learning_rate": 3.189894058988528e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.96875,
      "logps/chosen": -470.0,
      "logps/rejected": -824.0,
      "loss": 0.1933,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.46875,
      "step": 6510
    },
    {
      "epoch": 0.47065617555764094,
      "grad_norm": 14.398445656737117,
      "learning_rate": 3.183836681279344e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.96875,
      "logps/chosen": -480.0,
      "logps/rejected": -792.0,
      "loss": 0.2121,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.21875,
      "step": 6520
    },
    {
      "epoch": 0.47137804085757595,
      "grad_norm": 14.380506467247084,
      "learning_rate": 3.177774961188524e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.859375,
      "logps/chosen": -476.0,
      "logps/rejected": -872.0,
      "loss": 0.2262,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.875,
      "rewards/margins": 3.9375,
      "rewards/rejected": -6.8125,
      "step": 6530
    },
    {
      "epoch": 0.472099906157511,
      "grad_norm": 8.415048328930093,
      "learning_rate": 3.171708937208154e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.71875,
      "logps/chosen": -456.0,
      "logps/rejected": -824.0,
      "loss": 0.1978,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.75,
      "rewards/margins": 3.640625,
      "rewards/rejected": -6.40625,
      "step": 6540
    },
    {
      "epoch": 0.472821771457446,
      "grad_norm": 10.84464807285243,
      "learning_rate": 3.1656386478576553e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.140625,
      "logps/chosen": -528.0,
      "logps/rejected": -880.0,
      "loss": 0.1756,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.34375,
      "rewards/margins": 3.671875,
      "rewards/rejected": -7.03125,
      "step": 6550
    },
    {
      "epoch": 0.4735436367573811,
      "grad_norm": 12.682801254195484,
      "learning_rate": 3.159564131683529e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.046875,
      "logps/chosen": -532.0,
      "logps/rejected": -896.0,
      "loss": 0.1863,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.609375,
      "rewards/rejected": -7.125,
      "step": 6560
    },
    {
      "epoch": 0.4742655020573161,
      "grad_norm": 13.630045573582526,
      "learning_rate": 3.15348542725912e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.28125,
      "logps/chosen": -524.0,
      "logps/rejected": -896.0,
      "loss": 0.1871,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.265625,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.28125,
      "step": 6570
    },
    {
      "epoch": 0.4749873673572511,
      "grad_norm": 11.30596445239808,
      "learning_rate": 3.1474025731843675e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.96875,
      "logps/chosen": -504.0,
      "logps/rejected": -868.0,
      "loss": 0.1837,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.8125,
      "rewards/rejected": -6.84375,
      "step": 6580
    },
    {
      "epoch": 0.4757092326571862,
      "grad_norm": 10.788030773410654,
      "learning_rate": 3.141315608085561e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -3.0625,
      "logps/chosen": -516.0,
      "logps/rejected": -868.0,
      "loss": 0.1793,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.203125,
      "rewards/margins": 3.765625,
      "rewards/rejected": -6.96875,
      "step": 6590
    },
    {
      "epoch": 0.4764310979571212,
      "grad_norm": 14.767303382710391,
      "learning_rate": 3.1352245706150966e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -3.15625,
      "logps/chosen": -544.0,
      "logps/rejected": -872.0,
      "loss": 0.1891,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.53125,
      "rewards/rejected": -7.03125,
      "step": 6600
    },
    {
      "epoch": 0.47715296325705625,
      "grad_norm": 11.069092948755834,
      "learning_rate": 3.129129499451229e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.875,
      "logps/chosen": -556.0,
      "logps/rejected": -920.0,
      "loss": 0.1874,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.53125,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.3125,
      "step": 6610
    },
    {
      "epoch": 0.47787482855699126,
      "grad_norm": 9.484279905356596,
      "learning_rate": 3.123030433297823e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -3.03125,
      "logps/chosen": -496.0,
      "logps/rejected": -828.0,
      "loss": 0.2036,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.234375,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.5625,
      "step": 6620
    },
    {
      "epoch": 0.4785966938569263,
      "grad_norm": 14.241836376520746,
      "learning_rate": 3.11692741088412e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.671875,
      "logps/chosen": -482.0,
      "logps/rejected": -876.0,
      "loss": 0.235,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.890625,
      "rewards/rejected": -6.9375,
      "step": 6630
    },
    {
      "epoch": 0.47931855915686133,
      "grad_norm": 16.027966450135057,
      "learning_rate": 3.110820470964479e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.015625,
      "logps/chosen": -468.0,
      "logps/rejected": -828.0,
      "loss": 0.1976,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.859375,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.4375,
      "step": 6640
    },
    {
      "epoch": 0.48004042445679634,
      "grad_norm": 11.576067733762232,
      "learning_rate": 3.1047096523181334e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -3.21875,
      "logps/chosen": -524.0,
      "logps/rejected": -844.0,
      "loss": 0.1715,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.21875,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.78125,
      "step": 6650
    },
    {
      "epoch": 0.4807622897567314,
      "grad_norm": 16.882663245338957,
      "learning_rate": 3.098594993748949e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.0,
      "logps/chosen": -502.0,
      "logps/rejected": -876.0,
      "loss": 0.2039,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.09375,
      "step": 6660
    },
    {
      "epoch": 0.4814841550566664,
      "grad_norm": 12.617104805947024,
      "learning_rate": 3.092476534085177e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.859375,
      "logps/chosen": -508.0,
      "logps/rejected": -864.0,
      "loss": 0.1825,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.828125,
      "rewards/rejected": -6.90625,
      "step": 6670
    },
    {
      "epoch": 0.4822060203566015,
      "grad_norm": 13.894492770780278,
      "learning_rate": 3.0863543121792023e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.09375,
      "logps/chosen": -508.0,
      "logps/rejected": -872.0,
      "loss": 0.195,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.203125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.90625,
      "step": 6680
    },
    {
      "epoch": 0.4829278856565365,
      "grad_norm": 12.389701437099122,
      "learning_rate": 3.080228366907302e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.3125,
      "logps/chosen": -536.0,
      "logps/rejected": -872.0,
      "loss": 0.1708,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.578125,
      "rewards/rejected": -7.0,
      "step": 6690
    },
    {
      "epoch": 0.4836497509564715,
      "grad_norm": 13.314395162520189,
      "learning_rate": 3.074098737169395e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.40625,
      "logps/chosen": -548.0,
      "logps/rejected": -904.0,
      "loss": 0.2038,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.609375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.375,
      "step": 6700
    },
    {
      "epoch": 0.48437161625640657,
      "grad_norm": 9.727862895218676,
      "learning_rate": 3.0679654618887997e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.859375,
      "logps/chosen": -500.0,
      "logps/rejected": -828.0,
      "loss": 0.1968,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.5625,
      "step": 6710
    },
    {
      "epoch": 0.4850934815563416,
      "grad_norm": 18.478600121201747,
      "learning_rate": 3.0618285800119807e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.09375,
      "logps/chosen": -516.0,
      "logps/rejected": -876.0,
      "loss": 0.2017,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.625,
      "rewards/rejected": -6.875,
      "step": 6720
    },
    {
      "epoch": 0.48581534685627664,
      "grad_norm": 11.72239129198747,
      "learning_rate": 3.0556881305083064e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.84375,
      "logps/chosen": -492.0,
      "logps/rejected": -820.0,
      "loss": 0.2055,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.515625,
      "rewards/rejected": -6.5,
      "step": 6730
    },
    {
      "epoch": 0.48653721215621165,
      "grad_norm": 14.46807417377367,
      "learning_rate": 3.049544152369798e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -3.0,
      "logps/chosen": -528.0,
      "logps/rejected": -824.0,
      "loss": 0.1961,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.140625,
      "rewards/rejected": -6.40625,
      "step": 6740
    },
    {
      "epoch": 0.48725907745614666,
      "grad_norm": 10.403322841819048,
      "learning_rate": 3.043396684610887e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.890625,
      "logps/chosen": -528.0,
      "logps/rejected": -888.0,
      "loss": 0.1877,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.640625,
      "rewards/rejected": -6.90625,
      "step": 6750
    },
    {
      "epoch": 0.4879809427560817,
      "grad_norm": 12.61753116197978,
      "learning_rate": 3.0372457662681597e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.8125,
      "logps/chosen": -488.0,
      "logps/rejected": -852.0,
      "loss": 0.1959,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.71875,
      "step": 6760
    },
    {
      "epoch": 0.48870280805601674,
      "grad_norm": 10.757361268458817,
      "learning_rate": 3.031091436400117e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.015625,
      "logps/chosen": -508.0,
      "logps/rejected": -824.0,
      "loss": 0.2014,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.1875,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.625,
      "step": 6770
    },
    {
      "epoch": 0.4894246733559518,
      "grad_norm": 11.482520167696046,
      "learning_rate": 3.024933734086922e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.9375,
      "logps/chosen": -456.0,
      "logps/rejected": -816.0,
      "loss": 0.1959,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.84375,
      "rewards/margins": 3.625,
      "rewards/rejected": -6.46875,
      "step": 6780
    },
    {
      "epoch": 0.4901465386558868,
      "grad_norm": 11.834174161237362,
      "learning_rate": 3.018772698430152e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5,
      "logps/chosen": -484.0,
      "logps/rejected": -772.0,
      "loss": 0.1923,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.765625,
      "rewards/margins": 3.0625,
      "rewards/rejected": -5.8125,
      "step": 6790
    },
    {
      "epoch": 0.4908684039558218,
      "grad_norm": 12.486701951314425,
      "learning_rate": 3.0126083685525526e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.125,
      "logps/chosen": -490.0,
      "logps/rejected": -808.0,
      "loss": 0.1947,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.375,
      "rewards/rejected": -6.5,
      "step": 6800
    },
    {
      "epoch": 0.4915902692557569,
      "grad_norm": 12.530214748720425,
      "learning_rate": 3.006440783597787e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.078125,
      "logps/chosen": -516.0,
      "logps/rejected": -840.0,
      "loss": 0.1879,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.6875,
      "step": 6810
    },
    {
      "epoch": 0.4923121345556919,
      "grad_norm": 16.054591096686195,
      "learning_rate": 3.000269982730189e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.8125,
      "logps/chosen": -490.0,
      "logps/rejected": -832.0,
      "loss": 0.2002,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.1875,
      "rewards/margins": 3.390625,
      "rewards/rejected": -6.5625,
      "step": 6820
    },
    {
      "epoch": 0.49303399985562696,
      "grad_norm": 10.017766590785808,
      "learning_rate": 2.9940960051345135e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.171875,
      "logps/chosen": -502.0,
      "logps/rejected": -836.0,
      "loss": 0.2105,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.625,
      "step": 6830
    },
    {
      "epoch": 0.49375586515556197,
      "grad_norm": 17.867620325166374,
      "learning_rate": 2.987918890015685e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.78125,
      "logps/chosen": -512.0,
      "logps/rejected": -828.0,
      "loss": 0.1862,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.5625,
      "step": 6840
    },
    {
      "epoch": 0.494477730455497,
      "grad_norm": 13.234693911297601,
      "learning_rate": 2.981738676598555e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.9375,
      "logps/chosen": -502.0,
      "logps/rejected": -852.0,
      "loss": 0.195,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.203125,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.65625,
      "step": 6850
    },
    {
      "epoch": 0.49519959575543204,
      "grad_norm": 10.91568091066035,
      "learning_rate": 2.9755554041276465e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.140625,
      "logps/chosen": -452.0,
      "logps/rejected": -812.0,
      "loss": 0.2012,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.515625,
      "rewards/rejected": -6.4375,
      "step": 6860
    },
    {
      "epoch": 0.49592146105536705,
      "grad_norm": 10.129687126420459,
      "learning_rate": 2.9693691118669117e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.890625,
      "logps/chosen": -532.0,
      "logps/rejected": -864.0,
      "loss": 0.186,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.96875,
      "step": 6870
    },
    {
      "epoch": 0.4966433263553021,
      "grad_norm": 15.345418146246224,
      "learning_rate": 2.9631798390994726e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.25,
      "logps/chosen": -536.0,
      "logps/rejected": -896.0,
      "loss": 0.1876,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.71875,
      "rewards/rejected": -7.21875,
      "step": 6880
    },
    {
      "epoch": 0.4973651916552371,
      "grad_norm": 11.5517506279307,
      "learning_rate": 2.9569876251273826e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.4375,
      "logps/chosen": -540.0,
      "logps/rejected": -880.0,
      "loss": 0.2079,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.625,
      "rewards/rejected": -7.0625,
      "step": 6890
    },
    {
      "epoch": 0.4980870569551722,
      "grad_norm": 10.305253234304468,
      "learning_rate": 2.9507925092713685e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.234375,
      "logps/chosen": -508.0,
      "logps/rejected": -872.0,
      "loss": 0.1767,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -6.875,
      "step": 6900
    },
    {
      "epoch": 0.4988089222551072,
      "grad_norm": 11.897649462127623,
      "learning_rate": 2.9445945308705866e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.09375,
      "logps/chosen": -500.0,
      "logps/rejected": -848.0,
      "loss": 0.2162,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.65625,
      "step": 6910
    },
    {
      "epoch": 0.4995307875550422,
      "grad_norm": 11.567662785888109,
      "learning_rate": 2.938393729282369e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.828125,
      "logps/chosen": -506.0,
      "logps/rejected": -852.0,
      "loss": 0.1854,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.75,
      "step": 6920
    },
    {
      "epoch": 0.5002526528549772,
      "grad_norm": 13.838376356595091,
      "learning_rate": 2.932190143881976e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.140625,
      "logps/chosen": -508.0,
      "logps/rejected": -884.0,
      "loss": 0.1847,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.09375,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.03125,
      "step": 6930
    },
    {
      "epoch": 0.5009745181549123,
      "grad_norm": 10.287657850221445,
      "learning_rate": 2.925983814062342e-07,
      "logits/chosen": -3.125,
      "logits/rejected": -3.1875,
      "logps/chosen": -508.0,
      "logps/rejected": -880.0,
      "loss": 0.1864,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.171875,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.15625,
      "step": 6940
    },
    {
      "epoch": 0.5016963834548473,
      "grad_norm": 9.273870007955368,
      "learning_rate": 2.9197747792338344e-07,
      "logits/chosen": -3.140625,
      "logits/rejected": -3.34375,
      "logps/chosen": -502.0,
      "logps/rejected": -848.0,
      "loss": 0.186,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.78125,
      "step": 6950
    },
    {
      "epoch": 0.5024182487547824,
      "grad_norm": 13.765789333814018,
      "learning_rate": 2.9135630788239936e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.0625,
      "logps/chosen": -512.0,
      "logps/rejected": -832.0,
      "loss": 0.2016,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.5625,
      "step": 6960
    },
    {
      "epoch": 0.5031401140547174,
      "grad_norm": 14.606379064987632,
      "learning_rate": 2.907348752277286e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.078125,
      "logps/chosen": -524.0,
      "logps/rejected": -896.0,
      "loss": 0.2075,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.171875,
      "rewards/margins": 3.78125,
      "rewards/rejected": -6.96875,
      "step": 6970
    },
    {
      "epoch": 0.5038619793546524,
      "grad_norm": 13.437786216176978,
      "learning_rate": 2.901131839054856e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.015625,
      "logps/chosen": -516.0,
      "logps/rejected": -848.0,
      "loss": 0.1857,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.234375,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.75,
      "step": 6980
    },
    {
      "epoch": 0.5045838446545875,
      "grad_norm": 13.654924590150399,
      "learning_rate": 2.894912378634272e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.09375,
      "logps/chosen": -510.0,
      "logps/rejected": -864.0,
      "loss": 0.2,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.625,
      "rewards/rejected": -6.78125,
      "step": 6990
    },
    {
      "epoch": 0.5053057099545225,
      "grad_norm": 10.069751304391762,
      "learning_rate": 2.888690410509278e-07,
      "logits/chosen": -3.140625,
      "logits/rejected": -3.125,
      "logps/chosen": -504.0,
      "logps/rejected": -856.0,
      "loss": 0.1843,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.75,
      "step": 7000
    },
    {
      "epoch": 0.5060275752544575,
      "grad_norm": 11.655983078709607,
      "learning_rate": 2.8824659741895424e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -3.0625,
      "logps/chosen": -568.0,
      "logps/rejected": -856.0,
      "loss": 0.1904,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.5625,
      "rewards/margins": 3.1875,
      "rewards/rejected": -6.75,
      "step": 7010
    },
    {
      "epoch": 0.5067494405543925,
      "grad_norm": 12.917396868058894,
      "learning_rate": 2.8762391092004054e-07,
      "logits/chosen": -3.25,
      "logits/rejected": -3.34375,
      "logps/chosen": -512.0,
      "logps/rejected": -896.0,
      "loss": 0.2015,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.9375,
      "rewards/rejected": -7.28125,
      "step": 7020
    },
    {
      "epoch": 0.5074713058543275,
      "grad_norm": 17.58698053446558,
      "learning_rate": 2.8700098550826317e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -3.03125,
      "logps/chosen": -520.0,
      "logps/rejected": -804.0,
      "loss": 0.1945,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.0625,
      "rewards/rejected": -6.3125,
      "step": 7030
    },
    {
      "epoch": 0.5081931711542627,
      "grad_norm": 8.973334537563886,
      "learning_rate": 2.8637782513921537e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.796875,
      "logps/chosen": -488.0,
      "logps/rejected": -868.0,
      "loss": 0.169,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.9375,
      "rewards/margins": 4.0,
      "rewards/rejected": -6.9375,
      "step": 7040
    },
    {
      "epoch": 0.5089150364541977,
      "grad_norm": 15.787339394387272,
      "learning_rate": 2.857544337699826e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.9375,
      "logps/chosen": -496.0,
      "logps/rejected": -808.0,
      "loss": 0.2076,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.359375,
      "rewards/rejected": -6.4375,
      "step": 7050
    },
    {
      "epoch": 0.5096369017541327,
      "grad_norm": 12.888288799171846,
      "learning_rate": 2.85130815359117e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.0625,
      "logps/chosen": -528.0,
      "logps/rejected": -872.0,
      "loss": 0.1819,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.53125,
      "rewards/rejected": -6.96875,
      "step": 7060
    },
    {
      "epoch": 0.5103587670540677,
      "grad_norm": 15.921187968612813,
      "learning_rate": 2.845069738666128e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.078125,
      "logps/chosen": -536.0,
      "logps/rejected": -852.0,
      "loss": 0.1959,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.421875,
      "rewards/margins": 3.375,
      "rewards/rejected": -6.8125,
      "step": 7070
    },
    {
      "epoch": 0.5110806323540027,
      "grad_norm": 10.096087628655205,
      "learning_rate": 2.8388291325388024e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.9375,
      "logps/chosen": -496.0,
      "logps/rejected": -840.0,
      "loss": 0.1896,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.09375,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.65625,
      "step": 7080
    },
    {
      "epoch": 0.5118024976539378,
      "grad_norm": 13.539171325620412,
      "learning_rate": 2.832586374837215e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.125,
      "logps/chosen": -520.0,
      "logps/rejected": -836.0,
      "loss": 0.2097,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.625,
      "step": 7090
    },
    {
      "epoch": 0.5125243629538728,
      "grad_norm": 11.142024498657356,
      "learning_rate": 2.826341505203047e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.96875,
      "logps/chosen": -508.0,
      "logps/rejected": -828.0,
      "loss": 0.2008,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.171875,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.625,
      "step": 7100
    },
    {
      "epoch": 0.5132462282538078,
      "grad_norm": 11.062816515068299,
      "learning_rate": 2.8200945632913917e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -3.109375,
      "logps/chosen": -516.0,
      "logps/rejected": -828.0,
      "loss": 0.1779,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.5,
      "step": 7110
    },
    {
      "epoch": 0.5139680935537428,
      "grad_norm": 10.444518132011344,
      "learning_rate": 2.8138455887705017e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.84375,
      "logps/chosen": -540.0,
      "logps/rejected": -952.0,
      "loss": 0.1967,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.46875,
      "step": 7120
    },
    {
      "epoch": 0.5146899588536779,
      "grad_norm": 11.481596370883336,
      "learning_rate": 2.807594621321536e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.28125,
      "logps/chosen": -512.0,
      "logps/rejected": -832.0,
      "loss": 0.1953,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.75,
      "step": 7130
    },
    {
      "epoch": 0.515411824153613,
      "grad_norm": 9.753886495553875,
      "learning_rate": 2.801341700638307e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.0,
      "logps/chosen": -524.0,
      "logps/rejected": -912.0,
      "loss": 0.1965,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.296875,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.125,
      "step": 7140
    },
    {
      "epoch": 0.516133689453548,
      "grad_norm": 14.545395388593484,
      "learning_rate": 2.7950868664270355e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.15625,
      "logps/chosen": -486.0,
      "logps/rejected": -832.0,
      "loss": 0.1929,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.625,
      "rewards/rejected": -6.59375,
      "step": 7150
    },
    {
      "epoch": 0.516855554753483,
      "grad_norm": 9.237708777622595,
      "learning_rate": 2.788830158406088e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.953125,
      "logps/chosen": -520.0,
      "logps/rejected": -844.0,
      "loss": 0.1939,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.71875,
      "step": 7160
    },
    {
      "epoch": 0.517577420053418,
      "grad_norm": 11.233445817897985,
      "learning_rate": 2.782571616305731e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.828125,
      "logps/chosen": -494.0,
      "logps/rejected": -880.0,
      "loss": 0.1808,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.0,
      "step": 7170
    },
    {
      "epoch": 0.518299285353353,
      "grad_norm": 8.141977506993294,
      "learning_rate": 2.7763112798678787e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.296875,
      "logps/chosen": -466.0,
      "logps/rejected": -848.0,
      "loss": 0.1879,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.875,
      "rewards/rejected": -6.8125,
      "step": 7180
    },
    {
      "epoch": 0.5190211506532881,
      "grad_norm": 13.037802047586771,
      "learning_rate": 2.77004918884584e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.796875,
      "logps/chosen": -502.0,
      "logps/rejected": -856.0,
      "loss": 0.201,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.75,
      "step": 7190
    },
    {
      "epoch": 0.5197430159532231,
      "grad_norm": 14.133906902975497,
      "learning_rate": 2.763785383004063e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.75,
      "logps/chosen": -552.0,
      "logps/rejected": -876.0,
      "loss": 0.1765,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.65625,
      "rewards/rejected": -7.03125,
      "step": 7200
    },
    {
      "epoch": 0.5204648812531582,
      "grad_norm": 12.153644300642805,
      "learning_rate": 2.7575199021178855e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.15625,
      "logps/chosen": -528.0,
      "logps/rejected": -912.0,
      "loss": 0.1794,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.34375,
      "step": 7210
    },
    {
      "epoch": 0.5211867465530932,
      "grad_norm": 15.293128649387842,
      "learning_rate": 2.751252785973284e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.1875,
      "logps/chosen": -564.0,
      "logps/rejected": -920.0,
      "loss": 0.1823,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.75,
      "rewards/margins": 3.734375,
      "rewards/rejected": -7.5,
      "step": 7220
    },
    {
      "epoch": 0.5219086118530282,
      "grad_norm": 12.67629150141062,
      "learning_rate": 2.744984074366617e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.15625,
      "logps/chosen": -584.0,
      "logps/rejected": -916.0,
      "loss": 0.1847,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.90625,
      "rewards/margins": 3.609375,
      "rewards/rejected": -7.5,
      "step": 7230
    },
    {
      "epoch": 0.5226304771529633,
      "grad_norm": 15.105711037205923,
      "learning_rate": 2.738713807104375e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.859375,
      "logps/chosen": -568.0,
      "logps/rejected": -904.0,
      "loss": 0.198,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.546875,
      "rewards/margins": 3.453125,
      "rewards/rejected": -7.0,
      "step": 7240
    },
    {
      "epoch": 0.5233523424528983,
      "grad_norm": 12.09768436759906,
      "learning_rate": 2.732442024002926e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.984375,
      "logps/chosen": -500.0,
      "logps/rejected": -872.0,
      "loss": 0.1844,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.0,
      "step": 7250
    },
    {
      "epoch": 0.5240742077528333,
      "grad_norm": 8.545375861835133,
      "learning_rate": 2.726168764888264e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.875,
      "logps/chosen": -520.0,
      "logps/rejected": -876.0,
      "loss": 0.1864,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.84375,
      "step": 7260
    },
    {
      "epoch": 0.5247960730527683,
      "grad_norm": 14.361483972643855,
      "learning_rate": 2.7198940695957573e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.0625,
      "logps/chosen": -512.0,
      "logps/rejected": -892.0,
      "loss": 0.198,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.296875,
      "rewards/margins": 3.953125,
      "rewards/rejected": -7.25,
      "step": 7270
    },
    {
      "epoch": 0.5255179383527034,
      "grad_norm": 12.801511076375247,
      "learning_rate": 2.713617977969892e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -3.078125,
      "logps/chosen": -528.0,
      "logps/rejected": -864.0,
      "loss": 0.2188,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.34375,
      "rewards/margins": 3.609375,
      "rewards/rejected": -6.96875,
      "step": 7280
    },
    {
      "epoch": 0.5262398036526384,
      "grad_norm": 12.206462164663073,
      "learning_rate": 2.707340529864022e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.90625,
      "logps/chosen": -544.0,
      "logps/rejected": -860.0,
      "loss": 0.2073,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.53125,
      "rewards/margins": 3.15625,
      "rewards/rejected": -6.6875,
      "step": 7290
    },
    {
      "epoch": 0.5269616689525735,
      "grad_norm": 11.888075315735234,
      "learning_rate": 2.7010617651401133e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.15625,
      "logps/chosen": -524.0,
      "logps/rejected": -828.0,
      "loss": 0.1734,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.265625,
      "rewards/rejected": -6.75,
      "step": 7300
    },
    {
      "epoch": 0.5276835342525085,
      "grad_norm": 14.428511430380174,
      "learning_rate": 2.694781723668495e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.984375,
      "logps/chosen": -492.0,
      "logps/rejected": -844.0,
      "loss": 0.1805,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.5625,
      "step": 7310
    },
    {
      "epoch": 0.5284053995524435,
      "grad_norm": 16.00861134730293,
      "learning_rate": 2.6885004453276014e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.15625,
      "logps/chosen": -540.0,
      "logps/rejected": -936.0,
      "loss": 0.1919,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.484375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.53125,
      "step": 7320
    },
    {
      "epoch": 0.5291272648523786,
      "grad_norm": 11.427671700876187,
      "learning_rate": 2.682217970003724e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.921875,
      "logps/chosen": -504.0,
      "logps/rejected": -884.0,
      "loss": 0.19,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.234375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -7.03125,
      "step": 7330
    },
    {
      "epoch": 0.5298491301523136,
      "grad_norm": 9.741941087303589,
      "learning_rate": 2.6759343375907497e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.15625,
      "logps/chosen": -468.0,
      "logps/rejected": -836.0,
      "loss": 0.1848,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.921875,
      "rewards/margins": 3.8125,
      "rewards/rejected": -6.71875,
      "step": 7340
    },
    {
      "epoch": 0.5305709954522486,
      "grad_norm": 11.570878555173877,
      "learning_rate": 2.669649587989918e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.015625,
      "logps/chosen": -532.0,
      "logps/rejected": -888.0,
      "loss": 0.1914,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.125,
      "step": 7350
    },
    {
      "epoch": 0.5312928607521836,
      "grad_norm": 13.370937400579859,
      "learning_rate": 2.6633637611095615e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.125,
      "logps/chosen": -510.0,
      "logps/rejected": -848.0,
      "loss": 0.186,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.625,
      "step": 7360
    },
    {
      "epoch": 0.5320147260521186,
      "grad_norm": 9.26084511545428,
      "learning_rate": 2.6570768968648515e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.828125,
      "logps/chosen": -520.0,
      "logps/rejected": -848.0,
      "loss": 0.1842,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.78125,
      "step": 7370
    },
    {
      "epoch": 0.5327365913520538,
      "grad_norm": 11.874013382287073,
      "learning_rate": 2.6507890351775485e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.953125,
      "logps/chosen": -482.0,
      "logps/rejected": -844.0,
      "loss": 0.1749,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.53125,
      "step": 7380
    },
    {
      "epoch": 0.5334584566519888,
      "grad_norm": 9.97209880781557,
      "learning_rate": 2.644500215975747e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.6875,
      "logps/chosen": -528.0,
      "logps/rejected": -832.0,
      "loss": 0.1938,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.5625,
      "step": 7390
    },
    {
      "epoch": 0.5341803219519238,
      "grad_norm": 9.7508494192091,
      "learning_rate": 2.63821047919362e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.265625,
      "logps/chosen": -532.0,
      "logps/rejected": -888.0,
      "loss": 0.1898,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.1875,
      "step": 7400
    },
    {
      "epoch": 0.5349021872518588,
      "grad_norm": 12.866473104370337,
      "learning_rate": 2.631919864771168e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.96875,
      "logps/chosen": -508.0,
      "logps/rejected": -884.0,
      "loss": 0.1811,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.0625,
      "step": 7410
    },
    {
      "epoch": 0.5356240525517938,
      "grad_norm": 11.140404066410284,
      "learning_rate": 2.6256284126539657e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.0625,
      "logps/chosen": -516.0,
      "logps/rejected": -860.0,
      "loss": 0.1949,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.78125,
      "step": 7420
    },
    {
      "epoch": 0.5363459178517289,
      "grad_norm": 11.0786281528918,
      "learning_rate": 2.619336162792905e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.90625,
      "logps/chosen": -524.0,
      "logps/rejected": -888.0,
      "loss": 0.1676,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.21875,
      "step": 7430
    },
    {
      "epoch": 0.5370677831516639,
      "grad_norm": 15.506222958883363,
      "learning_rate": 2.6130431551439456e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.1875,
      "logps/chosen": -544.0,
      "logps/rejected": -952.0,
      "loss": 0.1693,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.5625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.71875,
      "step": 7440
    },
    {
      "epoch": 0.5377896484515989,
      "grad_norm": 13.463297911328912,
      "learning_rate": 2.606749429667859e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.328125,
      "logps/chosen": -568.0,
      "logps/rejected": -944.0,
      "loss": 0.1742,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.796875,
      "rewards/margins": 3.875,
      "rewards/rejected": -7.65625,
      "step": 7450
    },
    {
      "epoch": 0.5385115137515339,
      "grad_norm": 12.577800445202923,
      "learning_rate": 2.6004550263299724e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.6875,
      "logps/chosen": -516.0,
      "logps/rejected": -880.0,
      "loss": 0.1835,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.296875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -7.0,
      "step": 7460
    },
    {
      "epoch": 0.539233379051469,
      "grad_norm": 11.460691333024593,
      "learning_rate": 2.5941599850999194e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.015625,
      "logps/chosen": -532.0,
      "logps/rejected": -852.0,
      "loss": 0.1856,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.78125,
      "step": 7470
    },
    {
      "epoch": 0.5399552443514041,
      "grad_norm": 14.858572241125202,
      "learning_rate": 2.5878643459513857e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.84375,
      "logps/chosen": -528.0,
      "logps/rejected": -940.0,
      "loss": 0.187,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.59375,
      "step": 7480
    },
    {
      "epoch": 0.5406771096513391,
      "grad_norm": 16.410033622938368,
      "learning_rate": 2.58156814886185e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.5,
      "logps/chosen": -544.0,
      "logps/rejected": -868.0,
      "loss": 0.1997,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.609375,
      "rewards/margins": 3.453125,
      "rewards/rejected": -7.0625,
      "step": 7490
    },
    {
      "epoch": 0.5413989749512741,
      "grad_norm": 13.027073484035425,
      "learning_rate": 2.575271433812338e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.1875,
      "logps/chosen": -512.0,
      "logps/rejected": -880.0,
      "loss": 0.1861,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.75,
      "rewards/rejected": -7.0,
      "step": 7500
    },
    {
      "epoch": 0.5421208402512091,
      "grad_norm": 11.186461944090576,
      "learning_rate": 2.5689742407871615e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.15625,
      "logps/chosen": -492.0,
      "logps/rejected": -824.0,
      "loss": 0.1782,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.5,
      "step": 7510
    },
    {
      "epoch": 0.5428427055511441,
      "grad_norm": 11.248817729962756,
      "learning_rate": 2.562676609773669e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.234375,
      "logps/chosen": -528.0,
      "logps/rejected": -916.0,
      "loss": 0.1792,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.375,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.375,
      "step": 7520
    },
    {
      "epoch": 0.5435645708510792,
      "grad_norm": 11.881680952161064,
      "learning_rate": 2.556378580761989e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -2.96875,
      "logps/chosen": -520.0,
      "logps/rejected": -908.0,
      "loss": 0.1871,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.40625,
      "step": 7530
    },
    {
      "epoch": 0.5442864361510142,
      "grad_norm": 11.765920765599722,
      "learning_rate": 2.5500801937447794e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.25,
      "logps/chosen": -548.0,
      "logps/rejected": -864.0,
      "loss": 0.1856,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.59375,
      "rewards/margins": 3.453125,
      "rewards/rejected": -7.0625,
      "step": 7540
    },
    {
      "epoch": 0.5450083014509493,
      "grad_norm": 10.705656464510058,
      "learning_rate": 2.5437814887169684e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.828125,
      "logps/chosen": -488.0,
      "logps/rejected": -872.0,
      "loss": 0.1739,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.0,
      "rewards/margins": 3.84375,
      "rewards/rejected": -6.84375,
      "step": 7550
    },
    {
      "epoch": 0.5457301667508843,
      "grad_norm": 11.526996419577394,
      "learning_rate": 2.5374825056755073e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.03125,
      "logps/chosen": -502.0,
      "logps/rejected": -856.0,
      "loss": 0.213,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.875,
      "step": 7560
    },
    {
      "epoch": 0.5464520320508193,
      "grad_norm": 14.673145287172161,
      "learning_rate": 2.531183284619109e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.78125,
      "logps/chosen": -494.0,
      "logps/rejected": -848.0,
      "loss": 0.2054,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.4375,
      "rewards/rejected": -6.5625,
      "step": 7570
    },
    {
      "epoch": 0.5471738973507544,
      "grad_norm": 12.3021195529523,
      "learning_rate": 2.524883865548e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.84375,
      "logps/chosen": -502.0,
      "logps/rejected": -808.0,
      "loss": 0.1787,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.25,
      "rewards/rejected": -6.40625,
      "step": 7580
    },
    {
      "epoch": 0.5478957626506894,
      "grad_norm": 13.135082770486216,
      "learning_rate": 2.5185842884636633e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.96875,
      "logps/chosen": -532.0,
      "logps/rejected": -848.0,
      "loss": 0.1947,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.34375,
      "rewards/margins": 3.28125,
      "rewards/rejected": -6.625,
      "step": 7590
    },
    {
      "epoch": 0.5486176279506244,
      "grad_norm": 11.034407249591222,
      "learning_rate": 2.5122845933685873e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.296875,
      "logps/chosen": -486.0,
      "logps/rejected": -900.0,
      "loss": 0.1631,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.09375,
      "rewards/margins": 4.3125,
      "rewards/rejected": -7.40625,
      "step": 7600
    },
    {
      "epoch": 0.5493394932505594,
      "grad_norm": 11.333580525175115,
      "learning_rate": 2.505984820266007e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.28125,
      "logps/chosen": -532.0,
      "logps/rejected": -892.0,
      "loss": 0.1881,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.671875,
      "rewards/rejected": -7.125,
      "step": 7610
    },
    {
      "epoch": 0.5500613585504944,
      "grad_norm": 14.796276495360805,
      "learning_rate": 2.4996850091596535e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.96875,
      "logps/chosen": -492.0,
      "logps/rejected": -884.0,
      "loss": 0.1865,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.1875,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.09375,
      "step": 7620
    },
    {
      "epoch": 0.5507832238504295,
      "grad_norm": 15.831903288164375,
      "learning_rate": 2.493385200053502e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.203125,
      "logps/chosen": -520.0,
      "logps/rejected": -856.0,
      "loss": 0.1969,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.875,
      "step": 7630
    },
    {
      "epoch": 0.5515050891503646,
      "grad_norm": 12.289453507131972,
      "learning_rate": 2.48708543295151e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.203125,
      "logps/chosen": -510.0,
      "logps/rejected": -920.0,
      "loss": 0.1819,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.296875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.375,
      "step": 7640
    },
    {
      "epoch": 0.5522269544502996,
      "grad_norm": 12.385521936624869,
      "learning_rate": 2.480785747857372e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.28125,
      "logps/chosen": -524.0,
      "logps/rejected": -868.0,
      "loss": 0.1678,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.578125,
      "rewards/rejected": -6.9375,
      "step": 7650
    },
    {
      "epoch": 0.5529488197502346,
      "grad_norm": 15.015070988269633,
      "learning_rate": 2.474486184774262e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.03125,
      "logps/chosen": -528.0,
      "logps/rejected": -896.0,
      "loss": 0.1779,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -7.15625,
      "step": 7660
    },
    {
      "epoch": 0.5536706850501696,
      "grad_norm": 14.53649305247226,
      "learning_rate": 2.468186783704575e-07,
      "logits/chosen": -3.171875,
      "logits/rejected": -3.4375,
      "logps/chosen": -532.0,
      "logps/rejected": -912.0,
      "loss": 0.1969,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.34375,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.34375,
      "step": 7670
    },
    {
      "epoch": 0.5543925503501047,
      "grad_norm": 15.955775187674798,
      "learning_rate": 2.461887584649684e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.40625,
      "logps/chosen": -552.0,
      "logps/rejected": -888.0,
      "loss": 0.1969,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.625,
      "rewards/margins": 3.46875,
      "rewards/rejected": -7.09375,
      "step": 7680
    },
    {
      "epoch": 0.5551144156500397,
      "grad_norm": 16.713504026846465,
      "learning_rate": 2.4555886276096713e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.3125,
      "logps/chosen": -524.0,
      "logps/rejected": -872.0,
      "loss": 0.1872,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.6875,
      "rewards/rejected": -6.9375,
      "step": 7690
    },
    {
      "epoch": 0.5558362809499747,
      "grad_norm": 8.05376619910709,
      "learning_rate": 2.4492899525830896e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -3.265625,
      "logps/chosen": -498.0,
      "logps/rejected": -840.0,
      "loss": 0.1882,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.6875,
      "step": 7700
    },
    {
      "epoch": 0.5565581462499097,
      "grad_norm": 11.096062763104255,
      "learning_rate": 2.4429915995666967e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.09375,
      "logps/chosen": -524.0,
      "logps/rejected": -880.0,
      "loss": 0.1854,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.84375,
      "step": 7710
    },
    {
      "epoch": 0.5572800115498447,
      "grad_norm": 7.94016711065329,
      "learning_rate": 2.4366936085552055e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.0625,
      "logps/chosen": -524.0,
      "logps/rejected": -864.0,
      "loss": 0.1727,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.625,
      "rewards/rejected": -7.0,
      "step": 7720
    },
    {
      "epoch": 0.5580018768497799,
      "grad_norm": 10.336185440694942,
      "learning_rate": 2.430396019541032e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.09375,
      "logps/chosen": -524.0,
      "logps/rejected": -944.0,
      "loss": 0.1786,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.40625,
      "step": 7730
    },
    {
      "epoch": 0.5587237421497149,
      "grad_norm": 12.903258749145895,
      "learning_rate": 2.42409887251404e-07,
      "logits/chosen": -3.171875,
      "logits/rejected": -3.390625,
      "logps/chosen": -480.0,
      "logps/rejected": -880.0,
      "loss": 0.1824,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.0,
      "step": 7740
    },
    {
      "epoch": 0.5594456074496499,
      "grad_norm": 17.685627171392795,
      "learning_rate": 2.4178022074612834e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.203125,
      "logps/chosen": -516.0,
      "logps/rejected": -848.0,
      "loss": 0.1784,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.6875,
      "step": 7750
    },
    {
      "epoch": 0.5601674727495849,
      "grad_norm": 9.81534029197449,
      "learning_rate": 2.4115060643667596e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.109375,
      "logps/chosen": -510.0,
      "logps/rejected": -900.0,
      "loss": 0.1738,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.21875,
      "rewards/margins": 3.9375,
      "rewards/rejected": -7.15625,
      "step": 7760
    },
    {
      "epoch": 0.56088933804952,
      "grad_norm": 12.495410743124921,
      "learning_rate": 2.4052104832111477e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.5,
      "logps/chosen": -536.0,
      "logps/rejected": -912.0,
      "loss": 0.1799,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.984375,
      "rewards/rejected": -7.40625,
      "step": 7770
    },
    {
      "epoch": 0.561611203349455,
      "grad_norm": 15.33841888328186,
      "learning_rate": 2.39891550397156e-07,
      "logits/chosen": -3.171875,
      "logits/rejected": -3.140625,
      "logps/chosen": -524.0,
      "logps/rejected": -908.0,
      "loss": 0.1915,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.53125,
      "rewards/margins": 3.828125,
      "rewards/rejected": -7.34375,
      "step": 7780
    },
    {
      "epoch": 0.56233306864939,
      "grad_norm": 12.445378232378138,
      "learning_rate": 2.392621166621289e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.109375,
      "logps/chosen": -516.0,
      "logps/rejected": -924.0,
      "loss": 0.1935,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.46875,
      "step": 7790
    },
    {
      "epoch": 0.563054933949325,
      "grad_norm": 8.784716164451078,
      "learning_rate": 2.386327511129548e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.859375,
      "logps/chosen": -532.0,
      "logps/rejected": -880.0,
      "loss": 0.2093,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.96875,
      "step": 7800
    },
    {
      "epoch": 0.56377679924926,
      "grad_norm": 10.319440836977469,
      "learning_rate": 2.380034577461221e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.859375,
      "logps/chosen": -508.0,
      "logps/rejected": -872.0,
      "loss": 0.1732,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.84375,
      "step": 7810
    },
    {
      "epoch": 0.5644986645491952,
      "grad_norm": 15.940360664948143,
      "learning_rate": 2.373742405576611e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.21875,
      "logps/chosen": -532.0,
      "logps/rejected": -836.0,
      "loss": 0.1891,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.3125,
      "rewards/rejected": -6.65625,
      "step": 7820
    },
    {
      "epoch": 0.5652205298491302,
      "grad_norm": 16.82664598018782,
      "learning_rate": 2.3674510354311787e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.59375,
      "logps/chosen": -508.0,
      "logps/rejected": -880.0,
      "loss": 0.1932,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.765625,
      "rewards/rejected": -6.90625,
      "step": 7830
    },
    {
      "epoch": 0.5659423951490652,
      "grad_norm": 9.931656743981264,
      "learning_rate": 2.3611605069752988e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.09375,
      "logps/chosen": -498.0,
      "logps/rejected": -840.0,
      "loss": 0.1834,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.625,
      "rewards/rejected": -6.75,
      "step": 7840
    },
    {
      "epoch": 0.5666642604490002,
      "grad_norm": 11.999942287475443,
      "learning_rate": 2.3548708601539982e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.0625,
      "logps/chosen": -516.0,
      "logps/rejected": -856.0,
      "loss": 0.1689,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.8125,
      "step": 7850
    },
    {
      "epoch": 0.5673861257489352,
      "grad_norm": 15.619300105337258,
      "learning_rate": 2.3485821349067062e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.53125,
      "logps/chosen": -524.0,
      "logps/rejected": -920.0,
      "loss": 0.1913,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.4375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.53125,
      "step": 7860
    },
    {
      "epoch": 0.5681079910488703,
      "grad_norm": 13.401758330538527,
      "learning_rate": 2.3422943711670015e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.1875,
      "logps/chosen": -552.0,
      "logps/rejected": -916.0,
      "loss": 0.1904,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.578125,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.4375,
      "step": 7870
    },
    {
      "epoch": 0.5688298563488053,
      "grad_norm": 12.551899266975843,
      "learning_rate": 2.3360076088623544e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.34375,
      "logps/chosen": -536.0,
      "logps/rejected": -856.0,
      "loss": 0.2012,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.9375,
      "step": 7880
    },
    {
      "epoch": 0.5695517216487404,
      "grad_norm": 12.392869447668025,
      "learning_rate": 2.3297218879138775e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.125,
      "logps/chosen": -528.0,
      "logps/rejected": -872.0,
      "loss": 0.177,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.96875,
      "step": 7890
    },
    {
      "epoch": 0.5702735869486754,
      "grad_norm": 7.97335519233871,
      "learning_rate": 2.3234372482360726e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.0,
      "logps/chosen": -502.0,
      "logps/rejected": -840.0,
      "loss": 0.1767,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.484375,
      "rewards/rejected": -6.75,
      "step": 7900
    },
    {
      "epoch": 0.5709954522486104,
      "grad_norm": 14.189184545361437,
      "learning_rate": 2.3171537297365718e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.1875,
      "logps/chosen": -528.0,
      "logps/rejected": -896.0,
      "loss": 0.1767,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.296875,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.21875,
      "step": 7910
    },
    {
      "epoch": 0.5717173175485455,
      "grad_norm": 13.615803632379027,
      "learning_rate": 2.3108713723158908e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.015625,
      "logps/chosen": -508.0,
      "logps/rejected": -900.0,
      "loss": 0.1771,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.25,
      "step": 7920
    },
    {
      "epoch": 0.5724391828484805,
      "grad_norm": 18.15085008868228,
      "learning_rate": 2.304590215867169e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.125,
      "logps/chosen": -540.0,
      "logps/rejected": -904.0,
      "loss": 0.2044,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.25,
      "step": 7930
    },
    {
      "epoch": 0.5731610481484155,
      "grad_norm": 11.73979081994675,
      "learning_rate": 2.298310300275924e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.203125,
      "logps/chosen": -524.0,
      "logps/rejected": -888.0,
      "loss": 0.1761,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.734375,
      "rewards/rejected": -7.125,
      "step": 7940
    },
    {
      "epoch": 0.5738829134483505,
      "grad_norm": 11.774269111918269,
      "learning_rate": 2.2920316654197912e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.90625,
      "logps/chosen": -544.0,
      "logps/rejected": -880.0,
      "loss": 0.171,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.71875,
      "rewards/rejected": -7.15625,
      "step": 7950
    },
    {
      "epoch": 0.5746047787482855,
      "grad_norm": 10.38267859949208,
      "learning_rate": 2.2857543511682721e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -3.03125,
      "logps/chosen": -532.0,
      "logps/rejected": -856.0,
      "loss": 0.1604,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.390625,
      "rewards/rejected": -6.75,
      "step": 7960
    },
    {
      "epoch": 0.5753266440482206,
      "grad_norm": 20.36771507722865,
      "learning_rate": 2.279478397382486e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.890625,
      "logps/chosen": -556.0,
      "logps/rejected": -936.0,
      "loss": 0.2033,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.65625,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.5625,
      "step": 7970
    },
    {
      "epoch": 0.5760485093481557,
      "grad_norm": 10.49134512024263,
      "learning_rate": 2.273203843914911e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.953125,
      "logps/chosen": -544.0,
      "logps/rejected": -900.0,
      "loss": 0.1689,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.421875,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.3125,
      "step": 7980
    },
    {
      "epoch": 0.5767703746480907,
      "grad_norm": 13.47761308652524,
      "learning_rate": 2.266930730609132e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.015625,
      "logps/chosen": -548.0,
      "logps/rejected": -880.0,
      "loss": 0.169,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.515625,
      "rewards/margins": 3.515625,
      "rewards/rejected": -7.03125,
      "step": 7990
    },
    {
      "epoch": 0.5774922399480257,
      "grad_norm": 12.224951986625072,
      "learning_rate": 2.260659097299591e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.203125,
      "logps/chosen": -520.0,
      "logps/rejected": -892.0,
      "loss": 0.1915,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.21875,
      "step": 8000
    },
    {
      "epoch": 0.5782141052479607,
      "grad_norm": 8.896730833919746,
      "learning_rate": 2.2543889838113297e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.28125,
      "logps/chosen": -490.0,
      "logps/rejected": -872.0,
      "loss": 0.1851,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.984375,
      "rewards/rejected": -7.0,
      "step": 8010
    },
    {
      "epoch": 0.5789359705478958,
      "grad_norm": 16.802434184957853,
      "learning_rate": 2.2481204299597422e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.09375,
      "logps/chosen": -498.0,
      "logps/rejected": -832.0,
      "loss": 0.1889,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.515625,
      "rewards/rejected": -6.59375,
      "step": 8020
    },
    {
      "epoch": 0.5796578358478308,
      "grad_norm": 11.959374364001132,
      "learning_rate": 2.2418534755503169e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.0625,
      "logps/chosen": -496.0,
      "logps/rejected": -872.0,
      "loss": 0.1814,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.078125,
      "rewards/margins": 3.75,
      "rewards/rejected": -6.8125,
      "step": 8030
    },
    {
      "epoch": 0.5803797011477658,
      "grad_norm": 16.67472958146222,
      "learning_rate": 2.235588160378384e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.765625,
      "logps/chosen": -506.0,
      "logps/rejected": -840.0,
      "loss": 0.1934,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.1875,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.59375,
      "step": 8040
    },
    {
      "epoch": 0.5811015664477008,
      "grad_norm": 10.535607757575622,
      "learning_rate": 2.229324524228868e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.0625,
      "logps/chosen": -524.0,
      "logps/rejected": -856.0,
      "loss": 0.1921,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.84375,
      "step": 8050
    },
    {
      "epoch": 0.5818234317476358,
      "grad_norm": 12.590175617620153,
      "learning_rate": 2.2230626068760297e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.0625,
      "logps/chosen": -508.0,
      "logps/rejected": -896.0,
      "loss": 0.1847,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.125,
      "step": 8060
    },
    {
      "epoch": 0.582545297047571,
      "grad_norm": 6.19990918500515,
      "learning_rate": 2.216802448083214e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.15625,
      "logps/chosen": -500.0,
      "logps/rejected": -864.0,
      "loss": 0.1801,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.875,
      "step": 8070
    },
    {
      "epoch": 0.583267162347506,
      "grad_norm": 6.65168295725874,
      "learning_rate": 2.2105440876026026e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.0625,
      "logps/chosen": -548.0,
      "logps/rejected": -944.0,
      "loss": 0.1569,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 4.25,
      "rewards/rejected": -7.53125,
      "step": 8080
    },
    {
      "epoch": 0.583989027647441,
      "grad_norm": 13.41817755559622,
      "learning_rate": 2.2042875651749548e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.140625,
      "logps/chosen": -524.0,
      "logps/rejected": -904.0,
      "loss": 0.1833,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.171875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.25,
      "step": 8090
    },
    {
      "epoch": 0.584710892947376,
      "grad_norm": 12.232656939458046,
      "learning_rate": 2.1980329205293585e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.28125,
      "logps/chosen": -488.0,
      "logps/rejected": -824.0,
      "loss": 0.1842,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.65625,
      "step": 8100
    },
    {
      "epoch": 0.585432758247311,
      "grad_norm": 15.670146944940559,
      "learning_rate": 2.1917801933829807e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.25,
      "logps/chosen": -508.0,
      "logps/rejected": -856.0,
      "loss": 0.1824,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.203125,
      "rewards/margins": 3.65625,
      "rewards/rejected": -6.84375,
      "step": 8110
    },
    {
      "epoch": 0.5861546235472461,
      "grad_norm": 12.889291855179298,
      "learning_rate": 2.1855294234408068e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.96875,
      "logps/chosen": -528.0,
      "logps/rejected": -872.0,
      "loss": 0.1693,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.640625,
      "rewards/rejected": -7.0,
      "step": 8120
    },
    {
      "epoch": 0.5868764888471811,
      "grad_norm": 15.336485078796073,
      "learning_rate": 2.1792806503953986e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.109375,
      "logps/chosen": -496.0,
      "logps/rejected": -892.0,
      "loss": 0.1759,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.15625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.21875,
      "step": 8130
    },
    {
      "epoch": 0.5875983541471161,
      "grad_norm": 15.148752018930544,
      "learning_rate": 2.1730339139266368e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.296875,
      "logps/chosen": -498.0,
      "logps/rejected": -884.0,
      "loss": 0.1782,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.078125,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.125,
      "step": 8140
    },
    {
      "epoch": 0.5883202194470512,
      "grad_norm": 14.056612263217595,
      "learning_rate": 2.1667892537014664e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.03125,
      "logps/chosen": -520.0,
      "logps/rejected": -876.0,
      "loss": 0.1921,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.671875,
      "rewards/rejected": -6.9375,
      "step": 8150
    },
    {
      "epoch": 0.5890420847469862,
      "grad_norm": 14.417498533558616,
      "learning_rate": 2.1605467093736534e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.140625,
      "logps/chosen": -504.0,
      "logps/rejected": -908.0,
      "loss": 0.1787,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.21875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.28125,
      "step": 8160
    },
    {
      "epoch": 0.5897639500469213,
      "grad_norm": 13.963861595373876,
      "learning_rate": 2.1543063205835232e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.28125,
      "logps/chosen": -492.0,
      "logps/rejected": -900.0,
      "loss": 0.1825,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.046875,
      "rewards/margins": 4.25,
      "rewards/rejected": -7.28125,
      "step": 8170
    },
    {
      "epoch": 0.5904858153468563,
      "grad_norm": 14.230131260698164,
      "learning_rate": 2.1480681269577156e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -3.09375,
      "logps/chosen": -540.0,
      "logps/rejected": -848.0,
      "loss": 0.1785,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.296875,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.625,
      "step": 8180
    },
    {
      "epoch": 0.5912076806467913,
      "grad_norm": 15.767474534501417,
      "learning_rate": 2.141832168108932e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.34375,
      "logps/chosen": -508.0,
      "logps/rejected": -840.0,
      "loss": 0.1953,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.625,
      "step": 8190
    },
    {
      "epoch": 0.5919295459467263,
      "grad_norm": 13.816351894615584,
      "learning_rate": 2.1355984836356805e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.28125,
      "logps/chosen": -494.0,
      "logps/rejected": -844.0,
      "loss": 0.1726,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.140625,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.71875,
      "step": 8200
    },
    {
      "epoch": 0.5926514112466613,
      "grad_norm": 13.456125979818355,
      "learning_rate": 2.1293671131220277e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.890625,
      "logps/chosen": -508.0,
      "logps/rejected": -924.0,
      "loss": 0.1598,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.34375,
      "rewards/margins": 3.875,
      "rewards/rejected": -7.21875,
      "step": 8210
    },
    {
      "epoch": 0.5933732765465964,
      "grad_norm": 18.638941102403493,
      "learning_rate": 2.123138096137349e-07,
      "logits/chosen": -3.21875,
      "logits/rejected": -3.296875,
      "logps/chosen": -540.0,
      "logps/rejected": -908.0,
      "loss": 0.1895,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.546875,
      "rewards/margins": 3.578125,
      "rewards/rejected": -7.125,
      "step": 8220
    },
    {
      "epoch": 0.5940951418465314,
      "grad_norm": 8.686922517173084,
      "learning_rate": 2.1169114722360708e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.921875,
      "logps/chosen": -492.0,
      "logps/rejected": -928.0,
      "loss": 0.1608,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 4.46875,
      "rewards/rejected": -7.40625,
      "step": 8230
    },
    {
      "epoch": 0.5948170071464665,
      "grad_norm": 12.235444259129421,
      "learning_rate": 2.1106872809574266e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.546875,
      "logps/chosen": -508.0,
      "logps/rejected": -864.0,
      "loss": 0.1899,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.203125,
      "rewards/margins": 3.875,
      "rewards/rejected": -7.09375,
      "step": 8240
    },
    {
      "epoch": 0.5955388724464015,
      "grad_norm": 14.679756758037282,
      "learning_rate": 2.1044655618251994e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.234375,
      "logps/chosen": -580.0,
      "logps/rejected": -928.0,
      "loss": 0.1852,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.546875,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.46875,
      "step": 8250
    },
    {
      "epoch": 0.5962607377463366,
      "grad_norm": 11.821046377611859,
      "learning_rate": 2.098246354347477e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.328125,
      "logps/chosen": -540.0,
      "logps/rejected": -868.0,
      "loss": 0.1776,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.640625,
      "rewards/rejected": -7.0625,
      "step": 8260
    },
    {
      "epoch": 0.5969826030462716,
      "grad_norm": 15.401241096412328,
      "learning_rate": 2.0920296980163975e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.96875,
      "logps/chosen": -524.0,
      "logps/rejected": -892.0,
      "loss": 0.2193,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.5625,
      "rewards/rejected": -6.90625,
      "step": 8270
    },
    {
      "epoch": 0.5977044683462066,
      "grad_norm": 15.093199247560605,
      "learning_rate": 2.0858156323078986e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.0625,
      "logps/chosen": -510.0,
      "logps/rejected": -852.0,
      "loss": 0.174,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.6875,
      "step": 8280
    },
    {
      "epoch": 0.5984263336461416,
      "grad_norm": 11.639122869855335,
      "learning_rate": 2.0796041966814658e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.28125,
      "logps/chosen": -540.0,
      "logps/rejected": -876.0,
      "loss": 0.1834,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.78125,
      "step": 8290
    },
    {
      "epoch": 0.5991481989460766,
      "grad_norm": 14.907843300335792,
      "learning_rate": 2.0733954305798884e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.015625,
      "logps/chosen": -490.0,
      "logps/rejected": -848.0,
      "loss": 0.1875,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.0625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.78125,
      "step": 8300
    },
    {
      "epoch": 0.5998700642460117,
      "grad_norm": 11.379937012664104,
      "learning_rate": 2.0671893734289984e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.21875,
      "logps/chosen": -494.0,
      "logps/rejected": -832.0,
      "loss": 0.1839,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.15625,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.625,
      "step": 8310
    },
    {
      "epoch": 0.6005919295459468,
      "grad_norm": 11.219530062427582,
      "learning_rate": 2.0609860646374305e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.34375,
      "logps/chosen": -556.0,
      "logps/rejected": -868.0,
      "loss": 0.1828,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.90625,
      "step": 8320
    },
    {
      "epoch": 0.6013137948458818,
      "grad_norm": 11.798932906506304,
      "learning_rate": 2.054785543596363e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.125,
      "logps/chosen": -488.0,
      "logps/rejected": -912.0,
      "loss": 0.174,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.1875,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.3125,
      "step": 8330
    },
    {
      "epoch": 0.6020356601458168,
      "grad_norm": 9.618797817552972,
      "learning_rate": 2.0485878496792758e-07,
      "logits/chosen": -3.171875,
      "logits/rejected": -3.296875,
      "logps/chosen": -506.0,
      "logps/rejected": -892.0,
      "loss": 0.1724,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.234375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.03125,
      "step": 8340
    },
    {
      "epoch": 0.6027575254457518,
      "grad_norm": 10.728859514205965,
      "learning_rate": 2.042393022241695e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.0625,
      "logps/chosen": -508.0,
      "logps/rejected": -876.0,
      "loss": 0.1633,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.234375,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.9375,
      "step": 8350
    },
    {
      "epoch": 0.6034793907456869,
      "grad_norm": 14.698239592082293,
      "learning_rate": 2.0362011006209429e-07,
      "logits/chosen": -3.140625,
      "logits/rejected": -3.109375,
      "logps/chosen": -490.0,
      "logps/rejected": -900.0,
      "loss": 0.1838,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.15625,
      "step": 8360
    },
    {
      "epoch": 0.6042012560456219,
      "grad_norm": 14.51648676283174,
      "learning_rate": 2.0300121241358925e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.34375,
      "logps/chosen": -540.0,
      "logps/rejected": -904.0,
      "loss": 0.1829,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.53125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -7.3125,
      "step": 8370
    },
    {
      "epoch": 0.6049231213455569,
      "grad_norm": 9.48780699491775,
      "learning_rate": 2.023826132086714e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.109375,
      "logps/chosen": -536.0,
      "logps/rejected": -896.0,
      "loss": 0.1921,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.421875,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.25,
      "step": 8380
    },
    {
      "epoch": 0.6056449866454919,
      "grad_norm": 13.54032471904167,
      "learning_rate": 2.0176431637546243e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.328125,
      "logps/chosen": -520.0,
      "logps/rejected": -860.0,
      "loss": 0.1753,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.875,
      "rewards/rejected": -7.0,
      "step": 8390
    },
    {
      "epoch": 0.6063668519454269,
      "grad_norm": 12.47759241761334,
      "learning_rate": 2.0114632584016444e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.28125,
      "logps/chosen": -552.0,
      "logps/rejected": -920.0,
      "loss": 0.1614,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.671875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -7.375,
      "step": 8400
    },
    {
      "epoch": 0.6070887172453621,
      "grad_norm": 13.187923816606775,
      "learning_rate": 2.0052864552703392e-07,
      "logits/chosen": -3.125,
      "logits/rejected": -2.953125,
      "logps/chosen": -524.0,
      "logps/rejected": -936.0,
      "loss": 0.1855,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.3125,
      "step": 8410
    },
    {
      "epoch": 0.6078105825452971,
      "grad_norm": 11.498030392139778,
      "learning_rate": 1.9991127935835796e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.140625,
      "logps/chosen": -474.0,
      "logps/rejected": -852.0,
      "loss": 0.1676,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.96875,
      "rewards/margins": 3.828125,
      "rewards/rejected": -6.8125,
      "step": 8420
    },
    {
      "epoch": 0.6085324478452321,
      "grad_norm": 15.333757423043899,
      "learning_rate": 1.9929423125442866e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.265625,
      "logps/chosen": -516.0,
      "logps/rejected": -896.0,
      "loss": 0.1933,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.296875,
      "rewards/margins": 3.828125,
      "rewards/rejected": -7.125,
      "step": 8430
    },
    {
      "epoch": 0.6092543131451671,
      "grad_norm": 13.785919958777182,
      "learning_rate": 1.9867750513351813e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.375,
      "logps/chosen": -556.0,
      "logps/rejected": -888.0,
      "loss": 0.1589,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.546875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -7.09375,
      "step": 8440
    },
    {
      "epoch": 0.6099761784451021,
      "grad_norm": 9.926895348592163,
      "learning_rate": 1.9806110491185434e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.046875,
      "logps/chosen": -552.0,
      "logps/rejected": -936.0,
      "loss": 0.1604,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.53125,
      "rewards/margins": 3.828125,
      "rewards/rejected": -7.375,
      "step": 8450
    },
    {
      "epoch": 0.6106980437450372,
      "grad_norm": 12.683126222269557,
      "learning_rate": 1.9744503450359555e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.21875,
      "logps/chosen": -528.0,
      "logps/rejected": -880.0,
      "loss": 0.1768,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.6875,
      "rewards/rejected": -7.0,
      "step": 8460
    },
    {
      "epoch": 0.6114199090449722,
      "grad_norm": 10.47211411002499,
      "learning_rate": 1.968292978208055e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.171875,
      "logps/chosen": -520.0,
      "logps/rejected": -884.0,
      "loss": 0.1622,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.296875,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.21875,
      "step": 8470
    },
    {
      "epoch": 0.6121417743449072,
      "grad_norm": 9.270375708074656,
      "learning_rate": 1.9621389877342926e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.140625,
      "logps/chosen": -520.0,
      "logps/rejected": -908.0,
      "loss": 0.1735,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.421875,
      "rewards/margins": 3.9375,
      "rewards/rejected": -7.375,
      "step": 8480
    },
    {
      "epoch": 0.6128636396448423,
      "grad_norm": 10.270216134904993,
      "learning_rate": 1.9559884126926742e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.21875,
      "logps/chosen": -552.0,
      "logps/rejected": -904.0,
      "loss": 0.1827,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.640625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -7.34375,
      "step": 8490
    },
    {
      "epoch": 0.6135855049447773,
      "grad_norm": 9.518912127418925,
      "learning_rate": 1.9498412921395196e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.0625,
      "logps/chosen": -544.0,
      "logps/rejected": -936.0,
      "loss": 0.1642,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.59375,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.59375,
      "step": 8500
    },
    {
      "epoch": 0.6143073702447124,
      "grad_norm": 9.671738155095253,
      "learning_rate": 1.9436976651092142e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.671875,
      "logps/chosen": -512.0,
      "logps/rejected": -964.0,
      "loss": 0.1613,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.3125,
      "rewards/margins": 4.4375,
      "rewards/rejected": -7.75,
      "step": 8510
    },
    {
      "epoch": 0.6150292355446474,
      "grad_norm": 9.064376360826051,
      "learning_rate": 1.9375575706139557e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.40625,
      "logps/chosen": -536.0,
      "logps/rejected": -908.0,
      "loss": 0.1782,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.46875,
      "step": 8520
    },
    {
      "epoch": 0.6157511008445824,
      "grad_norm": 10.81950151915535,
      "learning_rate": 1.931421047643513e-07,
      "logits/chosen": -3.140625,
      "logits/rejected": -3.53125,
      "logps/chosen": -532.0,
      "logps/rejected": -920.0,
      "loss": 0.1694,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.34375,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.46875,
      "step": 8530
    },
    {
      "epoch": 0.6164729661445174,
      "grad_norm": 8.630415342936741,
      "learning_rate": 1.9252881351649764e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.34375,
      "logps/chosen": -510.0,
      "logps/rejected": -964.0,
      "loss": 0.1634,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.203125,
      "rewards/margins": 4.65625,
      "rewards/rejected": -7.84375,
      "step": 8540
    },
    {
      "epoch": 0.6171948314444524,
      "grad_norm": 12.614156200351186,
      "learning_rate": 1.919158872122505e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.375,
      "logps/chosen": -510.0,
      "logps/rejected": -948.0,
      "loss": 0.1762,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.28125,
      "rewards/margins": 4.40625,
      "rewards/rejected": -7.65625,
      "step": 8550
    },
    {
      "epoch": 0.6179166967443875,
      "grad_norm": 11.00131232394739,
      "learning_rate": 1.9130332974370888e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.21875,
      "logps/chosen": -536.0,
      "logps/rejected": -936.0,
      "loss": 0.1964,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.609375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.71875,
      "step": 8560
    },
    {
      "epoch": 0.6186385620443225,
      "grad_norm": 11.504288314895426,
      "learning_rate": 1.9069114500062944e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.15625,
      "logps/chosen": -516.0,
      "logps/rejected": -884.0,
      "loss": 0.1758,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.234375,
      "rewards/margins": 3.765625,
      "rewards/rejected": -7.0,
      "step": 8570
    },
    {
      "epoch": 0.6193604273442576,
      "grad_norm": 10.287311634128432,
      "learning_rate": 1.9007933687040192e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.265625,
      "logps/chosen": -480.0,
      "logps/rejected": -832.0,
      "loss": 0.1873,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 3.609375,
      "rewards/rejected": -6.6875,
      "step": 8580
    },
    {
      "epoch": 0.6200822926441926,
      "grad_norm": 12.748133734426029,
      "learning_rate": 1.8946790923802491e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.140625,
      "logps/chosen": -528.0,
      "logps/rejected": -900.0,
      "loss": 0.1936,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.34375,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.34375,
      "step": 8590
    },
    {
      "epoch": 0.6208041579441276,
      "grad_norm": 11.075760161214234,
      "learning_rate": 1.8885686598608044e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.921875,
      "logps/chosen": -512.0,
      "logps/rejected": -900.0,
      "loss": 0.1849,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.1875,
      "step": 8600
    },
    {
      "epoch": 0.6215260232440627,
      "grad_norm": 11.849005974471151,
      "learning_rate": 1.8824621099470986e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.34375,
      "logps/chosen": -540.0,
      "logps/rejected": -920.0,
      "loss": 0.1671,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.53125,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.4375,
      "step": 8610
    },
    {
      "epoch": 0.6222478885439977,
      "grad_norm": 9.904401949191602,
      "learning_rate": 1.8763594814158926e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.859375,
      "logps/chosen": -532.0,
      "logps/rejected": -916.0,
      "loss": 0.1679,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.359375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.375,
      "step": 8620
    },
    {
      "epoch": 0.6229697538439327,
      "grad_norm": 26.51334399545837,
      "learning_rate": 1.8702608130190427e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.078125,
      "logps/chosen": -560.0,
      "logps/rejected": -884.0,
      "loss": 0.1769,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.765625,
      "rewards/margins": 3.4375,
      "rewards/rejected": -7.1875,
      "step": 8630
    },
    {
      "epoch": 0.6236916191438677,
      "grad_norm": 14.189059450783944,
      "learning_rate": 1.8641661434832615e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.140625,
      "logps/chosen": -552.0,
      "logps/rejected": -912.0,
      "loss": 0.1724,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.546875,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.34375,
      "step": 8640
    },
    {
      "epoch": 0.6244134844438027,
      "grad_norm": 13.954446631732921,
      "learning_rate": 1.858075511509865e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.5,
      "logps/chosen": -520.0,
      "logps/rejected": -872.0,
      "loss": 0.1743,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.234375,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.03125,
      "step": 8650
    },
    {
      "epoch": 0.6251353497437379,
      "grad_norm": 10.979631722636864,
      "learning_rate": 1.8519889557745355e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.3125,
      "logps/chosen": -524.0,
      "logps/rejected": -900.0,
      "loss": 0.1751,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -7.21875,
      "step": 8660
    },
    {
      "epoch": 0.6258572150436729,
      "grad_norm": 13.559083992858115,
      "learning_rate": 1.8459065149270675e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.046875,
      "logps/chosen": -524.0,
      "logps/rejected": -844.0,
      "loss": 0.2029,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.40625,
      "rewards/rejected": -6.8125,
      "step": 8670
    },
    {
      "epoch": 0.6265790803436079,
      "grad_norm": 10.562568040272208,
      "learning_rate": 1.8398282275911265e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.234375,
      "logps/chosen": -540.0,
      "logps/rejected": -904.0,
      "loss": 0.1619,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.25,
      "step": 8680
    },
    {
      "epoch": 0.6273009456435429,
      "grad_norm": 15.213795707278102,
      "learning_rate": 1.833754132364003e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.078125,
      "logps/chosen": -498.0,
      "logps/rejected": -856.0,
      "loss": 0.2035,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -6.96875,
      "step": 8690
    },
    {
      "epoch": 0.6280228109434779,
      "grad_norm": 14.245455285055707,
      "learning_rate": 1.8276842678163694e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.0,
      "logps/chosen": -524.0,
      "logps/rejected": -916.0,
      "loss": 0.1785,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.3125,
      "step": 8700
    },
    {
      "epoch": 0.628744676243413,
      "grad_norm": 8.60823781040084,
      "learning_rate": 1.82161867249203e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.90625,
      "logps/chosen": -528.0,
      "logps/rejected": -888.0,
      "loss": 0.1772,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.34375,
      "rewards/margins": 3.609375,
      "rewards/rejected": -6.96875,
      "step": 8710
    },
    {
      "epoch": 0.629466541543348,
      "grad_norm": 14.22394995719737,
      "learning_rate": 1.8155573849076829e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.0625,
      "logps/chosen": -506.0,
      "logps/rejected": -864.0,
      "loss": 0.1974,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -7.0,
      "step": 8720
    },
    {
      "epoch": 0.630188406843283,
      "grad_norm": 11.175839489382133,
      "learning_rate": 1.8095004435526673e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.03125,
      "logps/chosen": -520.0,
      "logps/rejected": -908.0,
      "loss": 0.1762,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.125,
      "step": 8730
    },
    {
      "epoch": 0.630910272143218,
      "grad_norm": 13.348965383607348,
      "learning_rate": 1.8034478868887293e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -3.25,
      "logps/chosen": -520.0,
      "logps/rejected": -836.0,
      "loss": 0.176,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.390625,
      "rewards/rejected": -6.6875,
      "step": 8740
    },
    {
      "epoch": 0.631632137443153,
      "grad_norm": 13.995637804039818,
      "learning_rate": 1.7973997533497684e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.140625,
      "logps/chosen": -512.0,
      "logps/rejected": -892.0,
      "loss": 0.175,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.15625,
      "step": 8750
    },
    {
      "epoch": 0.6323540027430882,
      "grad_norm": 13.240851744061366,
      "learning_rate": 1.791356081341597e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.265625,
      "logps/chosen": -580.0,
      "logps/rejected": -928.0,
      "loss": 0.1855,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.796875,
      "rewards/margins": 3.625,
      "rewards/rejected": -7.40625,
      "step": 8760
    },
    {
      "epoch": 0.6330758680430232,
      "grad_norm": 12.727640092001586,
      "learning_rate": 1.7853169092416992e-07,
      "logits/chosen": -3.125,
      "logits/rejected": -3.40625,
      "logps/chosen": -512.0,
      "logps/rejected": -872.0,
      "loss": 0.1725,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.34375,
      "rewards/margins": 3.734375,
      "rewards/rejected": -7.0625,
      "step": 8770
    },
    {
      "epoch": 0.6337977333429582,
      "grad_norm": 14.00833683845359,
      "learning_rate": 1.779282275398983e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.234375,
      "logps/chosen": -568.0,
      "logps/rejected": -936.0,
      "loss": 0.1806,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.703125,
      "rewards/margins": 3.734375,
      "rewards/rejected": -7.4375,
      "step": 8780
    },
    {
      "epoch": 0.6345195986428932,
      "grad_norm": 14.829141185937326,
      "learning_rate": 1.773252218133537e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.28125,
      "logps/chosen": -532.0,
      "logps/rejected": -892.0,
      "loss": 0.1811,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.765625,
      "rewards/rejected": -7.09375,
      "step": 8790
    },
    {
      "epoch": 0.6352414639428283,
      "grad_norm": 15.033731924721785,
      "learning_rate": 1.7672267757363917e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.328125,
      "logps/chosen": -520.0,
      "logps/rejected": -844.0,
      "loss": 0.1824,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.421875,
      "rewards/margins": 3.328125,
      "rewards/rejected": -6.75,
      "step": 8800
    },
    {
      "epoch": 0.6359633292427633,
      "grad_norm": 9.64580188695773,
      "learning_rate": 1.7612059864692696e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.984375,
      "logps/chosen": -520.0,
      "logps/rejected": -892.0,
      "loss": 0.1643,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.34375,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.15625,
      "step": 8810
    },
    {
      "epoch": 0.6366851945426983,
      "grad_norm": 11.59441569874909,
      "learning_rate": 1.7551898885643478e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.9375,
      "logps/chosen": -488.0,
      "logps/rejected": -880.0,
      "loss": 0.1606,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.0,
      "step": 8820
    },
    {
      "epoch": 0.6374070598426334,
      "grad_norm": 12.713478878306795,
      "learning_rate": 1.749178520224014e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.046875,
      "logps/chosen": -496.0,
      "logps/rejected": -876.0,
      "loss": 0.1809,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.03125,
      "step": 8830
    },
    {
      "epoch": 0.6381289251425684,
      "grad_norm": 12.34376808306809,
      "learning_rate": 1.7431719196206186e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.359375,
      "logps/chosen": -486.0,
      "logps/rejected": -856.0,
      "loss": 0.1761,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -6.90625,
      "step": 8840
    },
    {
      "epoch": 0.6388507904425035,
      "grad_norm": 14.554096855268138,
      "learning_rate": 1.7371701248962426e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.046875,
      "logps/chosen": -500.0,
      "logps/rejected": -828.0,
      "loss": 0.1975,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.296875,
      "rewards/rejected": -6.59375,
      "step": 8850
    },
    {
      "epoch": 0.6395726557424385,
      "grad_norm": 10.027189965356278,
      "learning_rate": 1.7311731741624458e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.109375,
      "logps/chosen": -536.0,
      "logps/rejected": -920.0,
      "loss": 0.1519,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.359375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.4375,
      "step": 8860
    },
    {
      "epoch": 0.6402945210423735,
      "grad_norm": 14.98330809731259,
      "learning_rate": 1.7251811055000283e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.28125,
      "logps/chosen": -512.0,
      "logps/rejected": -924.0,
      "loss": 0.1743,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.1875,
      "rewards/margins": 4.25,
      "rewards/rejected": -7.4375,
      "step": 8870
    },
    {
      "epoch": 0.6410163863423085,
      "grad_norm": 8.691396365587716,
      "learning_rate": 1.7191939569587923e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.125,
      "logps/chosen": -524.0,
      "logps/rejected": -924.0,
      "loss": 0.1593,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.875,
      "rewards/rejected": -7.25,
      "step": 8880
    },
    {
      "epoch": 0.6417382516422435,
      "grad_norm": 13.896946382393645,
      "learning_rate": 1.7132117665572932e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.171875,
      "logps/chosen": -544.0,
      "logps/rejected": -936.0,
      "loss": 0.168,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.34375,
      "step": 8890
    },
    {
      "epoch": 0.6424601169421786,
      "grad_norm": 8.835859102076444,
      "learning_rate": 1.7072345722826035e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.140625,
      "logps/chosen": -548.0,
      "logps/rejected": -932.0,
      "loss": 0.1543,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.984375,
      "rewards/rejected": -7.4375,
      "step": 8900
    },
    {
      "epoch": 0.6431819822421136,
      "grad_norm": 13.31113126102166,
      "learning_rate": 1.701262412090072e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.140625,
      "logps/chosen": -524.0,
      "logps/rejected": -912.0,
      "loss": 0.197,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.9375,
      "rewards/rejected": -7.40625,
      "step": 8910
    },
    {
      "epoch": 0.6439038475420487,
      "grad_norm": 12.469637252975557,
      "learning_rate": 1.6952953239030779e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.875,
      "logps/chosen": -540.0,
      "logps/rejected": -908.0,
      "loss": 0.1616,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.953125,
      "rewards/rejected": -7.375,
      "step": 8920
    },
    {
      "epoch": 0.6446257128419837,
      "grad_norm": 14.513033348232137,
      "learning_rate": 1.6893333456127946e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.109375,
      "logps/chosen": -560.0,
      "logps/rejected": -976.0,
      "loss": 0.1573,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 4.375,
      "rewards/rejected": -7.90625,
      "step": 8930
    },
    {
      "epoch": 0.6453475781419187,
      "grad_norm": 13.319806064509727,
      "learning_rate": 1.6833765150779484e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.421875,
      "logps/chosen": -536.0,
      "logps/rejected": -888.0,
      "loss": 0.168,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 3.734375,
      "rewards/rejected": -7.25,
      "step": 8940
    },
    {
      "epoch": 0.6460694434418538,
      "grad_norm": 13.182029456288902,
      "learning_rate": 1.677424870124575e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.296875,
      "logps/chosen": -508.0,
      "logps/rejected": -900.0,
      "loss": 0.1676,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.953125,
      "rewards/rejected": -7.1875,
      "step": 8950
    },
    {
      "epoch": 0.6467913087417888,
      "grad_norm": 10.635716280326807,
      "learning_rate": 1.671478448545784e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.140625,
      "logps/chosen": -532.0,
      "logps/rejected": -940.0,
      "loss": 0.1848,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.53125,
      "step": 8960
    },
    {
      "epoch": 0.6475131740417238,
      "grad_norm": 16.09749119174247,
      "learning_rate": 1.665537288101514e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.234375,
      "logps/chosen": -540.0,
      "logps/rejected": -872.0,
      "loss": 0.1859,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.59375,
      "rewards/rejected": -7.0625,
      "step": 8970
    },
    {
      "epoch": 0.6482350393416588,
      "grad_norm": 11.566981105735882,
      "learning_rate": 1.6596014265182957e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.203125,
      "logps/chosen": -536.0,
      "logps/rejected": -888.0,
      "loss": 0.1911,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -7.21875,
      "step": 8980
    },
    {
      "epoch": 0.6489569046415938,
      "grad_norm": 11.964758920862586,
      "learning_rate": 1.6536709014890147e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -3.1875,
      "logps/chosen": -528.0,
      "logps/rejected": -876.0,
      "loss": 0.1667,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.734375,
      "rewards/rejected": -7.0625,
      "step": 8990
    },
    {
      "epoch": 0.649678769941529,
      "grad_norm": 14.526877439058453,
      "learning_rate": 1.6477457506726655e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.96875,
      "logps/chosen": -548.0,
      "logps/rejected": -924.0,
      "loss": 0.1624,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.5625,
      "rewards/margins": 3.765625,
      "rewards/rejected": -7.3125,
      "step": 9000
    },
    {
      "epoch": 0.650400635241464,
      "grad_norm": 18.575485009821474,
      "learning_rate": 1.641826011694119e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.0625,
      "logps/chosen": -560.0,
      "logps/rejected": -908.0,
      "loss": 0.191,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 3.59375,
      "rewards/rejected": -7.09375,
      "step": 9010
    },
    {
      "epoch": 0.651122500541399,
      "grad_norm": 14.455586249645847,
      "learning_rate": 1.6359117221438784e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.875,
      "logps/chosen": -544.0,
      "logps/rejected": -944.0,
      "loss": 0.1706,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.65625,
      "step": 9020
    },
    {
      "epoch": 0.651844365841334,
      "grad_norm": 13.847285540350875,
      "learning_rate": 1.6300029195778453e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.09375,
      "logps/chosen": -506.0,
      "logps/rejected": -900.0,
      "loss": 0.1781,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.28125,
      "step": 9030
    },
    {
      "epoch": 0.652566231141269,
      "grad_norm": 13.83435767481529,
      "learning_rate": 1.624099641517078e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.171875,
      "logps/chosen": -540.0,
      "logps/rejected": -952.0,
      "loss": 0.1741,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.453125,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.65625,
      "step": 9040
    },
    {
      "epoch": 0.6532880964412041,
      "grad_norm": 8.856166097204618,
      "learning_rate": 1.6182019254475514e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.21875,
      "logps/chosen": -512.0,
      "logps/rejected": -908.0,
      "loss": 0.1624,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.875,
      "rewards/rejected": -7.25,
      "step": 9050
    },
    {
      "epoch": 0.6540099617411391,
      "grad_norm": 9.09013525473115,
      "learning_rate": 1.6123098088199267e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.359375,
      "logps/chosen": -510.0,
      "logps/rejected": -888.0,
      "loss": 0.1747,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.9375,
      "rewards/rejected": -7.1875,
      "step": 9060
    },
    {
      "epoch": 0.6547318270410741,
      "grad_norm": 11.290725209117234,
      "learning_rate": 1.6064233290493048e-07,
      "logits/chosen": -3.171875,
      "logits/rejected": -3.140625,
      "logps/chosen": -528.0,
      "logps/rejected": -892.0,
      "loss": 0.1715,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -7.25,
      "step": 9070
    },
    {
      "epoch": 0.6554536923410091,
      "grad_norm": 14.477702076981982,
      "learning_rate": 1.600542523514992e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.15625,
      "logps/chosen": -532.0,
      "logps/rejected": -920.0,
      "loss": 0.1695,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.34375,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.34375,
      "step": 9080
    },
    {
      "epoch": 0.6561755576409442,
      "grad_norm": 9.249377666633862,
      "learning_rate": 1.5946674295602668e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.21875,
      "logps/chosen": -510.0,
      "logps/rejected": -864.0,
      "loss": 0.1884,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.09375,
      "step": 9090
    },
    {
      "epoch": 0.6568974229408793,
      "grad_norm": 13.384906611346702,
      "learning_rate": 1.5887980844921338e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.359375,
      "logps/chosen": -524.0,
      "logps/rejected": -884.0,
      "loss": 0.1802,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.21875,
      "step": 9100
    },
    {
      "epoch": 0.6576192882408143,
      "grad_norm": 11.67582450149049,
      "learning_rate": 1.5829345255810966e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.140625,
      "logps/chosen": -524.0,
      "logps/rejected": -860.0,
      "loss": 0.1649,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.640625,
      "rewards/rejected": -6.96875,
      "step": 9110
    },
    {
      "epoch": 0.6583411535407493,
      "grad_norm": 15.599163069162278,
      "learning_rate": 1.5770767900609144e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.203125,
      "logps/chosen": -502.0,
      "logps/rejected": -888.0,
      "loss": 0.1502,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.03125,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.1875,
      "step": 9120
    },
    {
      "epoch": 0.6590630188406843,
      "grad_norm": 12.810725656637196,
      "learning_rate": 1.5712249151283674e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.109375,
      "logps/chosen": -490.0,
      "logps/rejected": -920.0,
      "loss": 0.1705,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.09375,
      "rewards/margins": 4.3125,
      "rewards/rejected": -7.40625,
      "step": 9130
    },
    {
      "epoch": 0.6597848841406193,
      "grad_norm": 24.394244029906687,
      "learning_rate": 1.565378937943022e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.65625,
      "logps/chosen": -540.0,
      "logps/rejected": -900.0,
      "loss": 0.1771,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.421875,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.34375,
      "step": 9140
    },
    {
      "epoch": 0.6605067494405544,
      "grad_norm": 12.837227520621255,
      "learning_rate": 1.559538895626994e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.28125,
      "logps/chosen": -524.0,
      "logps/rejected": -896.0,
      "loss": 0.1716,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.421875,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.28125,
      "step": 9150
    },
    {
      "epoch": 0.6612286147404894,
      "grad_norm": 16.108378395476873,
      "learning_rate": 1.5537048252647102e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -3.171875,
      "logps/chosen": -532.0,
      "logps/rejected": -916.0,
      "loss": 0.1708,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.453125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.53125,
      "step": 9160
    },
    {
      "epoch": 0.6619504800404244,
      "grad_norm": 14.101215157530886,
      "learning_rate": 1.547876763902679e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.25,
      "logps/chosen": -528.0,
      "logps/rejected": -912.0,
      "loss": 0.1857,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.375,
      "step": 9170
    },
    {
      "epoch": 0.6626723453403595,
      "grad_norm": 11.414661725445344,
      "learning_rate": 1.5420547485492483e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.40625,
      "logps/chosen": -520.0,
      "logps/rejected": -932.0,
      "loss": 0.1664,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.203125,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.375,
      "step": 9180
    },
    {
      "epoch": 0.6633942106402945,
      "grad_norm": 9.691066471839772,
      "learning_rate": 1.5362388161743743e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.140625,
      "logps/chosen": -548.0,
      "logps/rejected": -948.0,
      "loss": 0.175,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.71875,
      "step": 9190
    },
    {
      "epoch": 0.6641160759402296,
      "grad_norm": 14.365319490379342,
      "learning_rate": 1.5304290037093887e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.171875,
      "logps/chosen": -528.0,
      "logps/rejected": -904.0,
      "loss": 0.1598,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.25,
      "step": 9200
    },
    {
      "epoch": 0.6648379412401646,
      "grad_norm": 11.665029992113674,
      "learning_rate": 1.524625348046758e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.078125,
      "logps/chosen": -512.0,
      "logps/rejected": -896.0,
      "loss": 0.1543,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.21875,
      "step": 9210
    },
    {
      "epoch": 0.6655598065400996,
      "grad_norm": 14.238448378406478,
      "learning_rate": 1.5188278860398543e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.84375,
      "logps/chosen": -552.0,
      "logps/rejected": -988.0,
      "loss": 0.1635,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.453125,
      "rewards/margins": 4.53125,
      "rewards/rejected": -8.0,
      "step": 9220
    },
    {
      "epoch": 0.6662816718400346,
      "grad_norm": 16.248658893719846,
      "learning_rate": 1.5130366545027215e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.875,
      "logps/chosen": -560.0,
      "logps/rejected": -1000.0,
      "loss": 0.1667,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.515625,
      "rewards/margins": 4.375,
      "rewards/rejected": -7.875,
      "step": 9230
    },
    {
      "epoch": 0.6670035371399696,
      "grad_norm": 11.479891225082076,
      "learning_rate": 1.507251690209837e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.21875,
      "logps/chosen": -540.0,
      "logps/rejected": -928.0,
      "loss": 0.1855,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.9375,
      "rewards/rejected": -7.375,
      "step": 9240
    },
    {
      "epoch": 0.6677254024399047,
      "grad_norm": 14.38489819710241,
      "learning_rate": 1.5014730298958834e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -3.171875,
      "logps/chosen": -524.0,
      "logps/rejected": -892.0,
      "loss": 0.1696,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.25,
      "step": 9250
    },
    {
      "epoch": 0.6684472677398398,
      "grad_norm": 16.652287797931404,
      "learning_rate": 1.4957007102555101e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.28125,
      "logps/chosen": -536.0,
      "logps/rejected": -936.0,
      "loss": 0.1785,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.53125,
      "step": 9260
    },
    {
      "epoch": 0.6691691330397748,
      "grad_norm": 16.191601092731727,
      "learning_rate": 1.489934767943106e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.265625,
      "logps/chosen": -528.0,
      "logps/rejected": -900.0,
      "loss": 0.166,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.3125,
      "step": 9270
    },
    {
      "epoch": 0.6698909983397098,
      "grad_norm": 8.964797886256093,
      "learning_rate": 1.484175239572563e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.296875,
      "logps/chosen": -560.0,
      "logps/rejected": -972.0,
      "loss": 0.1595,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.65625,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.84375,
      "step": 9280
    },
    {
      "epoch": 0.6706128636396449,
      "grad_norm": 16.667852169459124,
      "learning_rate": 1.4784221617170427e-07,
      "logits/chosen": -3.125,
      "logits/rejected": -3.421875,
      "logps/chosen": -510.0,
      "logps/rejected": -908.0,
      "loss": 0.1688,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.3125,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.4375,
      "step": 9290
    },
    {
      "epoch": 0.6713347289395799,
      "grad_norm": 11.470361549627574,
      "learning_rate": 1.4726755709087467e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.125,
      "logps/chosen": -540.0,
      "logps/rejected": -924.0,
      "loss": 0.1588,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.546875,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.375,
      "step": 9300
    },
    {
      "epoch": 0.6720565942395149,
      "grad_norm": 13.403027221320416,
      "learning_rate": 1.466935503638686e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.96875,
      "logps/chosen": -536.0,
      "logps/rejected": -956.0,
      "loss": 0.1693,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 4.3125,
      "rewards/rejected": -7.75,
      "step": 9310
    },
    {
      "epoch": 0.6727784595394499,
      "grad_norm": 11.976074878273202,
      "learning_rate": 1.4612019963564425e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.40625,
      "logps/chosen": -548.0,
      "logps/rejected": -912.0,
      "loss": 0.1919,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.375,
      "step": 9320
    },
    {
      "epoch": 0.6735003248393849,
      "grad_norm": 11.705207199332705,
      "learning_rate": 1.4554750854699454e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.28125,
      "logps/chosen": -524.0,
      "logps/rejected": -888.0,
      "loss": 0.1542,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.15625,
      "step": 9330
    },
    {
      "epoch": 0.67422219013932,
      "grad_norm": 16.955256841022532,
      "learning_rate": 1.4497548073452353e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.265625,
      "logps/chosen": -540.0,
      "logps/rejected": -912.0,
      "loss": 0.1649,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.40625,
      "step": 9340
    },
    {
      "epoch": 0.6749440554392551,
      "grad_norm": 11.60723369401256,
      "learning_rate": 1.444041198306235e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.125,
      "logps/chosen": -500.0,
      "logps/rejected": -856.0,
      "loss": 0.169,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.1875,
      "rewards/margins": 3.75,
      "rewards/rejected": -6.96875,
      "step": 9350
    },
    {
      "epoch": 0.6756659207391901,
      "grad_norm": 11.203681633630948,
      "learning_rate": 1.4383342946345178e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.03125,
      "logps/chosen": -552.0,
      "logps/rejected": -956.0,
      "loss": 0.158,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.46875,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.6875,
      "step": 9360
    },
    {
      "epoch": 0.6763877860391251,
      "grad_norm": 15.64732744797141,
      "learning_rate": 1.432634132569079e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.125,
      "logps/chosen": -560.0,
      "logps/rejected": -972.0,
      "loss": 0.1772,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.734375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.96875,
      "step": 9370
    },
    {
      "epoch": 0.6771096513390601,
      "grad_norm": 12.998118903172957,
      "learning_rate": 1.4269407483061025e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.765625,
      "logps/chosen": -548.0,
      "logps/rejected": -960.0,
      "loss": 0.1694,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.6875,
      "step": 9380
    },
    {
      "epoch": 0.6778315166389952,
      "grad_norm": 12.669153188092436,
      "learning_rate": 1.4212541779987358e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -3.109375,
      "logps/chosen": -548.0,
      "logps/rejected": -884.0,
      "loss": 0.1843,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -7.125,
      "step": 9390
    },
    {
      "epoch": 0.6785533819389302,
      "grad_norm": 8.842758876016001,
      "learning_rate": 1.415574457756853e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.421875,
      "logps/chosen": -548.0,
      "logps/rejected": -908.0,
      "loss": 0.1556,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.546875,
      "rewards/margins": 3.796875,
      "rewards/rejected": -7.34375,
      "step": 9400
    },
    {
      "epoch": 0.6792752472388652,
      "grad_norm": 9.202568677261356,
      "learning_rate": 1.409901623646837e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.09375,
      "logps/chosen": -532.0,
      "logps/rejected": -908.0,
      "loss": 0.1743,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.25,
      "step": 9410
    },
    {
      "epoch": 0.6799971125388002,
      "grad_norm": 13.652053469414295,
      "learning_rate": 1.4042357116913365e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.078125,
      "logps/chosen": -516.0,
      "logps/rejected": -896.0,
      "loss": 0.1652,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.21875,
      "step": 9420
    },
    {
      "epoch": 0.6807189778387353,
      "grad_norm": 18.902818884153398,
      "learning_rate": 1.3985767578690474e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.125,
      "logps/chosen": -536.0,
      "logps/rejected": -864.0,
      "loss": 0.1928,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.453125,
      "rewards/rejected": -6.96875,
      "step": 9430
    },
    {
      "epoch": 0.6814408431386704,
      "grad_norm": 11.064371866656955,
      "learning_rate": 1.3929247981144845e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.203125,
      "logps/chosen": -524.0,
      "logps/rejected": -924.0,
      "loss": 0.154,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.421875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.5625,
      "step": 9440
    },
    {
      "epoch": 0.6821627084386054,
      "grad_norm": 14.277439810034117,
      "learning_rate": 1.387279868317744e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.859375,
      "logps/chosen": -540.0,
      "logps/rejected": -988.0,
      "loss": 0.1608,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.453125,
      "rewards/margins": 4.4375,
      "rewards/rejected": -7.875,
      "step": 9450
    },
    {
      "epoch": 0.6828845737385404,
      "grad_norm": 13.619416810692789,
      "learning_rate": 1.3816420043242843e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.140625,
      "logps/chosen": -504.0,
      "logps/rejected": -868.0,
      "loss": 0.1727,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.296875,
      "rewards/margins": 3.59375,
      "rewards/rejected": -6.90625,
      "step": 9460
    },
    {
      "epoch": 0.6836064390384754,
      "grad_norm": 12.681012641582917,
      "learning_rate": 1.3760112419346986e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.15625,
      "logps/chosen": -524.0,
      "logps/rejected": -904.0,
      "loss": 0.1683,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.3125,
      "step": 9470
    },
    {
      "epoch": 0.6843283043384104,
      "grad_norm": 11.849978255106802,
      "learning_rate": 1.3703876169044802e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.96875,
      "logps/chosen": -544.0,
      "logps/rejected": -884.0,
      "loss": 0.1706,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.640625,
      "rewards/rejected": -7.125,
      "step": 9480
    },
    {
      "epoch": 0.6850501696383455,
      "grad_norm": 13.093820996112642,
      "learning_rate": 1.364771164943802e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.0625,
      "logps/chosen": -524.0,
      "logps/rejected": -872.0,
      "loss": 0.1769,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -7.0625,
      "step": 9490
    },
    {
      "epoch": 0.6857720349382805,
      "grad_norm": 12.896247213001585,
      "learning_rate": 1.3591619217172883e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -2.96875,
      "logps/chosen": -532.0,
      "logps/rejected": -948.0,
      "loss": 0.1591,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.265625,
      "rewards/margins": 4.40625,
      "rewards/rejected": -7.6875,
      "step": 9500
    },
    {
      "epoch": 0.6864939002382155,
      "grad_norm": 10.868482403843391,
      "learning_rate": 1.3535599228437867e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.9375,
      "logps/chosen": -524.0,
      "logps/rejected": -932.0,
      "loss": 0.163,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.46875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.625,
      "step": 9510
    },
    {
      "epoch": 0.6872157655381506,
      "grad_norm": 18.448674497902317,
      "learning_rate": 1.3479652038961432e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.15625,
      "logps/chosen": -560.0,
      "logps/rejected": -912.0,
      "loss": 0.1802,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.546875,
      "rewards/margins": 3.796875,
      "rewards/rejected": -7.34375,
      "step": 9520
    },
    {
      "epoch": 0.6879376308380856,
      "grad_norm": 9.514314881827286,
      "learning_rate": 1.3423778004009762e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -3.0625,
      "logps/chosen": -544.0,
      "logps/rejected": -944.0,
      "loss": 0.1539,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.53125,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.71875,
      "step": 9530
    },
    {
      "epoch": 0.6886594961380207,
      "grad_norm": 10.019588383805546,
      "learning_rate": 1.3367977478384513e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.171875,
      "logps/chosen": -528.0,
      "logps/rejected": -932.0,
      "loss": 0.1702,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.5625,
      "step": 9540
    },
    {
      "epoch": 0.6893813614379557,
      "grad_norm": 16.86491244768804,
      "learning_rate": 1.3312250816420542e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.953125,
      "logps/chosen": -516.0,
      "logps/rejected": -904.0,
      "loss": 0.1808,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.3125,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.3125,
      "step": 9550
    },
    {
      "epoch": 0.6901032267378907,
      "grad_norm": 8.943845719589888,
      "learning_rate": 1.3256598371983686e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.953125,
      "logps/chosen": -520.0,
      "logps/rejected": -932.0,
      "loss": 0.1487,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.359375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.5,
      "step": 9560
    },
    {
      "epoch": 0.6908250920378257,
      "grad_norm": 14.452255719780107,
      "learning_rate": 1.320102049846849e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.125,
      "logps/chosen": -524.0,
      "logps/rejected": -916.0,
      "loss": 0.1561,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.296875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.375,
      "step": 9570
    },
    {
      "epoch": 0.6915469573377607,
      "grad_norm": 10.015917276168755,
      "learning_rate": 1.314551754879595e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.140625,
      "logps/chosen": -532.0,
      "logps/rejected": -912.0,
      "loss": 0.1567,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -7.25,
      "step": 9580
    },
    {
      "epoch": 0.6922688226376958,
      "grad_norm": 14.787051092004921,
      "learning_rate": 1.3090089875411337e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.953125,
      "logps/chosen": -544.0,
      "logps/rejected": -964.0,
      "loss": 0.1632,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.59375,
      "step": 9590
    },
    {
      "epoch": 0.6929906879376309,
      "grad_norm": 12.82194722015672,
      "learning_rate": 1.30347378302819e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.203125,
      "logps/chosen": -528.0,
      "logps/rejected": -912.0,
      "loss": 0.1817,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.984375,
      "rewards/rejected": -7.46875,
      "step": 9600
    },
    {
      "epoch": 0.6937125532375659,
      "grad_norm": 13.181020933581339,
      "learning_rate": 1.2979461764894608e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -3.0,
      "logps/chosen": -540.0,
      "logps/rejected": -896.0,
      "loss": 0.1646,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.1875,
      "step": 9610
    },
    {
      "epoch": 0.6944344185375009,
      "grad_norm": 13.154400288009723,
      "learning_rate": 1.2924262030254022e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.046875,
      "logps/chosen": -500.0,
      "logps/rejected": -876.0,
      "loss": 0.1905,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.125,
      "rewards/margins": 3.9375,
      "rewards/rejected": -7.0625,
      "step": 9620
    },
    {
      "epoch": 0.6951562838374359,
      "grad_norm": 13.7601292151576,
      "learning_rate": 1.2869138976879961e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.015625,
      "logps/chosen": -540.0,
      "logps/rejected": -880.0,
      "loss": 0.1742,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.5625,
      "rewards/margins": 3.5,
      "rewards/rejected": -7.0625,
      "step": 9630
    },
    {
      "epoch": 0.695878149137371,
      "grad_norm": 14.21645132894316,
      "learning_rate": 1.2814092954805294e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.1875,
      "logps/chosen": -524.0,
      "logps/rejected": -896.0,
      "loss": 0.1755,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.515625,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.3125,
      "step": 9640
    },
    {
      "epoch": 0.696600014437306,
      "grad_norm": 13.196279645318443,
      "learning_rate": 1.275912431357381e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.34375,
      "logps/chosen": -560.0,
      "logps/rejected": -912.0,
      "loss": 0.1829,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.578125,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.4375,
      "step": 9650
    },
    {
      "epoch": 0.697321879737241,
      "grad_norm": 9.70626351046916,
      "learning_rate": 1.2704233402237858e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.828125,
      "logps/chosen": -532.0,
      "logps/rejected": -896.0,
      "loss": 0.1764,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.21875,
      "step": 9660
    },
    {
      "epoch": 0.698043745037176,
      "grad_norm": 12.504923641553384,
      "learning_rate": 1.2649420569356217e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.953125,
      "logps/chosen": -524.0,
      "logps/rejected": -916.0,
      "loss": 0.1728,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.25,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.46875,
      "step": 9670
    },
    {
      "epoch": 0.698765610337111,
      "grad_norm": 15.792709304104829,
      "learning_rate": 1.2594686162991914e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.140625,
      "logps/chosen": -512.0,
      "logps/rejected": -880.0,
      "loss": 0.1557,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.15625,
      "step": 9680
    },
    {
      "epoch": 0.6994874756370462,
      "grad_norm": 12.508474000382183,
      "learning_rate": 1.2540030530709888e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.3125,
      "logps/chosen": -528.0,
      "logps/rejected": -876.0,
      "loss": 0.1707,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.671875,
      "rewards/rejected": -7.0625,
      "step": 9690
    },
    {
      "epoch": 0.7002093409369812,
      "grad_norm": 11.79123615131554,
      "learning_rate": 1.248545401957492e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.90625,
      "logps/chosen": -510.0,
      "logps/rejected": -872.0,
      "loss": 0.1757,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.234375,
      "rewards/margins": 3.75,
      "rewards/rejected": -7.0,
      "step": 9700
    },
    {
      "epoch": 0.7009312062369162,
      "grad_norm": 12.074354295077159,
      "learning_rate": 1.2430956976149345e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.890625,
      "logps/chosen": -494.0,
      "logps/rejected": -904.0,
      "loss": 0.1588,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.1875,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.3125,
      "step": 9710
    },
    {
      "epoch": 0.7016530715368512,
      "grad_norm": 10.292679338298079,
      "learning_rate": 1.2376539746490878e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.109375,
      "logps/chosen": -532.0,
      "logps/rejected": -920.0,
      "loss": 0.1714,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.359375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.5,
      "step": 9720
    },
    {
      "epoch": 0.7023749368367862,
      "grad_norm": 14.666786151585931,
      "learning_rate": 1.2322202676150417e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.421875,
      "logps/chosen": -520.0,
      "logps/rejected": -920.0,
      "loss": 0.1775,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.46875,
      "step": 9730
    },
    {
      "epoch": 0.7030968021367213,
      "grad_norm": 12.359960679267141,
      "learning_rate": 1.2267946110169847e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.25,
      "logps/chosen": -532.0,
      "logps/rejected": -884.0,
      "loss": 0.1593,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.65625,
      "rewards/rejected": -7.125,
      "step": 9740
    },
    {
      "epoch": 0.7038186674366563,
      "grad_norm": 10.24405875899712,
      "learning_rate": 1.2213770393079846e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.09375,
      "logps/chosen": -564.0,
      "logps/rejected": -936.0,
      "loss": 0.1561,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.65625,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.5625,
      "step": 9750
    },
    {
      "epoch": 0.7045405327365913,
      "grad_norm": 11.19036322432956,
      "learning_rate": 1.21596758688977e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.828125,
      "logps/chosen": -528.0,
      "logps/rejected": -932.0,
      "loss": 0.1642,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.46875,
      "step": 9760
    },
    {
      "epoch": 0.7052623980365264,
      "grad_norm": 14.709229789322158,
      "learning_rate": 1.2105662881125112e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -2.859375,
      "logps/chosen": -536.0,
      "logps/rejected": -924.0,
      "loss": 0.1761,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.5625,
      "rewards/margins": 3.828125,
      "rewards/rejected": -7.40625,
      "step": 9770
    },
    {
      "epoch": 0.7059842633364615,
      "grad_norm": 9.197378934889331,
      "learning_rate": 1.205173177274604e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.015625,
      "logps/chosen": -524.0,
      "logps/rejected": -924.0,
      "loss": 0.1711,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.40625,
      "step": 9780
    },
    {
      "epoch": 0.7067061286363965,
      "grad_norm": 11.793483434858606,
      "learning_rate": 1.1997882886224497e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.140625,
      "logps/chosen": -552.0,
      "logps/rejected": -944.0,
      "loss": 0.1515,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.546875,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.71875,
      "step": 9790
    },
    {
      "epoch": 0.7074279939363315,
      "grad_norm": 11.148734697425425,
      "learning_rate": 1.194411656350238e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.25,
      "logps/chosen": -548.0,
      "logps/rejected": -908.0,
      "loss": 0.1776,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.59375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.46875,
      "step": 9800
    },
    {
      "epoch": 0.7081498592362665,
      "grad_norm": 8.42082775531952,
      "learning_rate": 1.1890433145997327e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.265625,
      "logps/chosen": -520.0,
      "logps/rejected": -868.0,
      "loss": 0.1612,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.640625,
      "rewards/rejected": -7.03125,
      "step": 9810
    },
    {
      "epoch": 0.7088717245362015,
      "grad_norm": 12.576234475904954,
      "learning_rate": 1.1836832974600483e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.828125,
      "logps/chosen": -524.0,
      "logps/rejected": -872.0,
      "loss": 0.1823,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.6875,
      "rewards/rejected": -7.09375,
      "step": 9820
    },
    {
      "epoch": 0.7095935898361366,
      "grad_norm": 9.296328778410011,
      "learning_rate": 1.1783316389674406e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -3.109375,
      "logps/chosen": -532.0,
      "logps/rejected": -900.0,
      "loss": 0.1595,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 3.640625,
      "rewards/rejected": -7.125,
      "step": 9830
    },
    {
      "epoch": 0.7103154551360716,
      "grad_norm": 11.725811633276853,
      "learning_rate": 1.1729883731050899e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.015625,
      "logps/chosen": -532.0,
      "logps/rejected": -908.0,
      "loss": 0.1386,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.40625,
      "step": 9840
    },
    {
      "epoch": 0.7110373204360066,
      "grad_norm": 10.39742315469531,
      "learning_rate": 1.1676535338028781e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.09375,
      "logps/chosen": -552.0,
      "logps/rejected": -908.0,
      "loss": 0.1602,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.484375,
      "rewards/margins": 3.765625,
      "rewards/rejected": -7.25,
      "step": 9850
    },
    {
      "epoch": 0.7117591857359417,
      "grad_norm": 12.90629435547991,
      "learning_rate": 1.1623271549371808e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.21875,
      "logps/chosen": -510.0,
      "logps/rejected": -908.0,
      "loss": 0.1956,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.296875,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.5,
      "step": 9860
    },
    {
      "epoch": 0.7124810510358767,
      "grad_norm": 12.75087817763919,
      "learning_rate": 1.1570092703306523e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.140625,
      "logps/chosen": -510.0,
      "logps/rejected": -896.0,
      "loss": 0.1779,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.09375,
      "step": 9870
    },
    {
      "epoch": 0.7132029163358118,
      "grad_norm": 13.00408625154444,
      "learning_rate": 1.1516999137520023e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.890625,
      "logps/chosen": -528.0,
      "logps/rejected": -872.0,
      "loss": 0.1621,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.6875,
      "rewards/rejected": -6.96875,
      "step": 9880
    },
    {
      "epoch": 0.7139247816357468,
      "grad_norm": 12.420447777727329,
      "learning_rate": 1.1463991189157917e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -3.25,
      "logps/chosen": -516.0,
      "logps/rejected": -884.0,
      "loss": 0.1884,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.171875,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.0625,
      "step": 9890
    },
    {
      "epoch": 0.7146466469356818,
      "grad_norm": 8.832732530159305,
      "learning_rate": 1.1411069194822124e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -3.0625,
      "logps/chosen": -520.0,
      "logps/rejected": -912.0,
      "loss": 0.1572,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.3125,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.3125,
      "step": 9900
    },
    {
      "epoch": 0.7153685122356168,
      "grad_norm": 12.919438925426455,
      "learning_rate": 1.1358233490568758e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.171875,
      "logps/chosen": -520.0,
      "logps/rejected": -900.0,
      "loss": 0.1484,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.125,
      "step": 9910
    },
    {
      "epoch": 0.7160903775355518,
      "grad_norm": 13.28401189280203,
      "learning_rate": 1.1305484411905986e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.03125,
      "logps/chosen": -552.0,
      "logps/rejected": -964.0,
      "loss": 0.1574,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.59375,
      "rewards/margins": 4.25,
      "rewards/rejected": -7.84375,
      "step": 9920
    },
    {
      "epoch": 0.716812242835487,
      "grad_norm": 8.688958610732897,
      "learning_rate": 1.12528222937919e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -3.0625,
      "logps/chosen": -524.0,
      "logps/rejected": -892.0,
      "loss": 0.17,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.25,
      "step": 9930
    },
    {
      "epoch": 0.717534108135422,
      "grad_norm": 13.048531217026252,
      "learning_rate": 1.1200247470632392e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.734375,
      "logps/chosen": -548.0,
      "logps/rejected": -928.0,
      "loss": 0.1852,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.625,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.5,
      "step": 9940
    },
    {
      "epoch": 0.718255973435357,
      "grad_norm": 11.768300316769222,
      "learning_rate": 1.1147760276279028e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.703125,
      "logps/chosen": -552.0,
      "logps/rejected": -920.0,
      "loss": 0.1738,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.640625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.40625,
      "step": 9950
    },
    {
      "epoch": 0.718977838735292,
      "grad_norm": 15.715309468308565,
      "learning_rate": 1.1095361044026927e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.046875,
      "logps/chosen": -548.0,
      "logps/rejected": -900.0,
      "loss": 0.1741,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -7.21875,
      "step": 9960
    },
    {
      "epoch": 0.719699704035227,
      "grad_norm": 9.627940893510306,
      "learning_rate": 1.1043050106612656e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.234375,
      "logps/chosen": -544.0,
      "logps/rejected": -876.0,
      "loss": 0.1738,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.53125,
      "rewards/margins": 3.59375,
      "rewards/rejected": -7.125,
      "step": 9970
    },
    {
      "epoch": 0.7204215693351621,
      "grad_norm": 13.456671696812624,
      "learning_rate": 1.0990827796212074e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.984375,
      "logps/chosen": -544.0,
      "logps/rejected": -920.0,
      "loss": 0.1659,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.65625,
      "rewards/margins": 3.875,
      "rewards/rejected": -7.53125,
      "step": 9980
    },
    {
      "epoch": 0.7211434346350971,
      "grad_norm": 13.429298531291938,
      "learning_rate": 1.0938694444438307e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.25,
      "logps/chosen": -528.0,
      "logps/rejected": -920.0,
      "loss": 0.1468,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.46875,
      "step": 9990
    },
    {
      "epoch": 0.7218652999350321,
      "grad_norm": 10.49538345041886,
      "learning_rate": 1.0886650382339566e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.109375,
      "logps/chosen": -532.0,
      "logps/rejected": -892.0,
      "loss": 0.1662,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.703125,
      "rewards/rejected": -7.15625,
      "step": 10000
    },
    {
      "epoch": 0.7218652999350321,
      "eval_logits/chosen": -2.9375,
      "eval_logits/rejected": -3.078125,
      "eval_logps/chosen": -564.0,
      "eval_logps/rejected": -900.0,
      "eval_loss": 0.2485724240541458,
      "eval_rewards/accuracies": 0.8979707956314087,
      "eval_rewards/chosen": -3.65625,
      "eval_rewards/margins": 3.59375,
      "eval_rewards/rejected": -7.25,
      "eval_runtime": 2908.8448,
      "eval_samples_per_second": 33.863,
      "eval_steps_per_second": 0.529,
      "step": 10000
    },
    {
      "epoch": 0.7225871652349671,
      "grad_norm": 9.591204561450711,
      "learning_rate": 1.083469594039704e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.984375,
      "logps/chosen": -524.0,
      "logps/rejected": -912.0,
      "loss": 0.1685,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.296875,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.21875,
      "step": 10010
    },
    {
      "epoch": 0.7233090305349021,
      "grad_norm": 13.16797947073937,
      "learning_rate": 1.0782831448522886e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.171875,
      "logps/chosen": -528.0,
      "logps/rejected": -896.0,
      "loss": 0.1587,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.421875,
      "rewards/margins": 3.75,
      "rewards/rejected": -7.15625,
      "step": 10020
    },
    {
      "epoch": 0.7240308958348373,
      "grad_norm": 17.893609832591608,
      "learning_rate": 1.0731057236058047e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.84375,
      "logps/chosen": -536.0,
      "logps/rejected": -868.0,
      "loss": 0.1835,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.9375,
      "step": 10030
    },
    {
      "epoch": 0.7247527611347723,
      "grad_norm": 16.255770230650498,
      "learning_rate": 1.0679373631770162e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.9375,
      "logps/chosen": -524.0,
      "logps/rejected": -948.0,
      "loss": 0.1709,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.296875,
      "rewards/margins": 4.40625,
      "rewards/rejected": -7.71875,
      "step": 10040
    },
    {
      "epoch": 0.7254746264347073,
      "grad_norm": 14.132033776382704,
      "learning_rate": 1.0627780963851562e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.015625,
      "logps/chosen": -498.0,
      "logps/rejected": -872.0,
      "loss": 0.1753,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.25,
      "rewards/margins": 3.796875,
      "rewards/rejected": -7.03125,
      "step": 10050
    },
    {
      "epoch": 0.7261964917346423,
      "grad_norm": 11.653629630616956,
      "learning_rate": 1.0576279559917081e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.1875,
      "logps/chosen": -536.0,
      "logps/rejected": -900.0,
      "loss": 0.1776,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.65625,
      "rewards/rejected": -7.125,
      "step": 10060
    },
    {
      "epoch": 0.7269183570345773,
      "grad_norm": 11.590257045403085,
      "learning_rate": 1.0524869747002041e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.953125,
      "logps/chosen": -476.0,
      "logps/rejected": -888.0,
      "loss": 0.1639,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.0,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.1875,
      "step": 10070
    },
    {
      "epoch": 0.7276402223345124,
      "grad_norm": 8.856685676122096,
      "learning_rate": 1.0473551851560184e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.09375,
      "logps/chosen": -532.0,
      "logps/rejected": -884.0,
      "loss": 0.157,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.765625,
      "rewards/rejected": -7.09375,
      "step": 10080
    },
    {
      "epoch": 0.7283620876344474,
      "grad_norm": 15.096444355674352,
      "learning_rate": 1.0422326199461526e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.90625,
      "logps/chosen": -516.0,
      "logps/rejected": -888.0,
      "loss": 0.1588,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.75,
      "rewards/rejected": -7.0625,
      "step": 10090
    },
    {
      "epoch": 0.7290839529343824,
      "grad_norm": 14.45542631114454,
      "learning_rate": 1.0371193115990362e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.171875,
      "logps/chosen": -560.0,
      "logps/rejected": -972.0,
      "loss": 0.1611,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.46875,
      "rewards/margins": 4.5,
      "rewards/rejected": -7.9375,
      "step": 10100
    },
    {
      "epoch": 0.7298058182343174,
      "grad_norm": 16.192343154843932,
      "learning_rate": 1.0320152925843192e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.1875,
      "logps/chosen": -516.0,
      "logps/rejected": -952.0,
      "loss": 0.19,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.359375,
      "rewards/margins": 4.40625,
      "rewards/rejected": -7.75,
      "step": 10110
    },
    {
      "epoch": 0.7305276835342525,
      "grad_norm": 11.572817559208797,
      "learning_rate": 1.02692059531266e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.84375,
      "logps/chosen": -510.0,
      "logps/rejected": -880.0,
      "loss": 0.1657,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.671875,
      "rewards/rejected": -6.9375,
      "step": 10120
    },
    {
      "epoch": 0.7312495488341876,
      "grad_norm": 15.201914382298016,
      "learning_rate": 1.0218352521355262e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.015625,
      "logps/chosen": -552.0,
      "logps/rejected": -924.0,
      "loss": 0.1568,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.4375,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.4375,
      "step": 10130
    },
    {
      "epoch": 0.7319714141341226,
      "grad_norm": 9.188775774128196,
      "learning_rate": 1.0167592953449858e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.296875,
      "logps/chosen": -544.0,
      "logps/rejected": -924.0,
      "loss": 0.165,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.515625,
      "rewards/margins": 3.984375,
      "rewards/rejected": -7.5,
      "step": 10140
    },
    {
      "epoch": 0.7326932794340576,
      "grad_norm": 13.60506952662718,
      "learning_rate": 1.011692757173504e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.890625,
      "logps/chosen": -560.0,
      "logps/rejected": -980.0,
      "loss": 0.178,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.625,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.84375,
      "step": 10150
    },
    {
      "epoch": 0.7334151447339926,
      "grad_norm": 13.94312688112025,
      "learning_rate": 1.0066356697937362e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.015625,
      "logps/chosen": -536.0,
      "logps/rejected": -952.0,
      "loss": 0.1663,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.53125,
      "rewards/margins": 4.34375,
      "rewards/rejected": -7.875,
      "step": 10160
    },
    {
      "epoch": 0.7341370100339276,
      "grad_norm": 15.468208504085602,
      "learning_rate": 1.0015880653183259e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.21875,
      "logps/chosen": -556.0,
      "logps/rejected": -956.0,
      "loss": 0.1926,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.65625,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.5625,
      "step": 10170
    },
    {
      "epoch": 0.7348588753338627,
      "grad_norm": 12.75058572975185,
      "learning_rate": 9.965499757996997e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.15625,
      "logps/chosen": -520.0,
      "logps/rejected": -888.0,
      "loss": 0.1553,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.296875,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.1875,
      "step": 10180
    },
    {
      "epoch": 0.7355807406337977,
      "grad_norm": 14.72650654384829,
      "learning_rate": 9.915214332298638e-08,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.078125,
      "logps/chosen": -528.0,
      "logps/rejected": -936.0,
      "loss": 0.1845,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.390625,
      "rewards/margins": 4.28125,
      "rewards/rejected": -7.65625,
      "step": 10190
    },
    {
      "epoch": 0.7363026059337328,
      "grad_norm": 12.457001671367488,
      "learning_rate": 9.865024695402014e-08,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.109375,
      "logps/chosen": -524.0,
      "logps/rejected": -960.0,
      "loss": 0.1636,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.25,
      "rewards/rejected": -7.65625,
      "step": 10200
    },
    {
      "epoch": 0.7370244712336678,
      "grad_norm": 15.173036652763688,
      "learning_rate": 9.814931166012694e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -3.015625,
      "logps/chosen": -536.0,
      "logps/rejected": -896.0,
      "loss": 0.1595,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.234375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.3125,
      "step": 10210
    },
    {
      "epoch": 0.7377463365336028,
      "grad_norm": 18.30642539369029,
      "learning_rate": 9.76493406222594e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.0625,
      "logps/chosen": -552.0,
      "logps/rejected": -952.0,
      "loss": 0.1561,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 4.3125,
      "rewards/rejected": -7.8125,
      "step": 10220
    },
    {
      "epoch": 0.7384682018335379,
      "grad_norm": 16.253150796543157,
      "learning_rate": 9.715033701524756e-08,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.984375,
      "logps/chosen": -532.0,
      "logps/rejected": -900.0,
      "loss": 0.1756,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -7.03125,
      "step": 10230
    },
    {
      "epoch": 0.7391900671334729,
      "grad_norm": 16.10179341586736,
      "learning_rate": 9.665230400777793e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -3.25,
      "logps/chosen": -528.0,
      "logps/rejected": -896.0,
      "loss": 0.1743,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.953125,
      "rewards/rejected": -7.34375,
      "step": 10240
    },
    {
      "epoch": 0.7399119324334079,
      "grad_norm": 9.39201917709881,
      "learning_rate": 9.615524476237358e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.15625,
      "logps/chosen": -528.0,
      "logps/rejected": -900.0,
      "loss": 0.1686,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.34375,
      "step": 10250
    },
    {
      "epoch": 0.7406337977333429,
      "grad_norm": 14.195145895107842,
      "learning_rate": 9.565916243537434e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.109375,
      "logps/chosen": -536.0,
      "logps/rejected": -868.0,
      "loss": 0.1641,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.5,
      "rewards/rejected": -6.875,
      "step": 10260
    },
    {
      "epoch": 0.741355663033278,
      "grad_norm": 13.564604814493594,
      "learning_rate": 9.51640601769168e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.984375,
      "logps/chosen": -520.0,
      "logps/rejected": -892.0,
      "loss": 0.1698,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.3125,
      "step": 10270
    },
    {
      "epoch": 0.742077528333213,
      "grad_norm": 10.875158029544309,
      "learning_rate": 9.466994113091359e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.046875,
      "logps/chosen": -524.0,
      "logps/rejected": -900.0,
      "loss": 0.1801,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.984375,
      "rewards/rejected": -7.375,
      "step": 10280
    },
    {
      "epoch": 0.7427993936331481,
      "grad_norm": 8.021424505958798,
      "learning_rate": 9.417680843503426e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.25,
      "logps/chosen": -560.0,
      "logps/rejected": -932.0,
      "loss": 0.1667,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.59375,
      "step": 10290
    },
    {
      "epoch": 0.7435212589330831,
      "grad_norm": 10.178836761828853,
      "learning_rate": 9.368466522068494e-08,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.21875,
      "logps/chosen": -508.0,
      "logps/rejected": -928.0,
      "loss": 0.1686,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.296875,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.40625,
      "step": 10300
    },
    {
      "epoch": 0.7442431242330181,
      "grad_norm": 11.19358002960795,
      "learning_rate": 9.319351461298847e-08,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.84375,
      "logps/chosen": -508.0,
      "logps/rejected": -900.0,
      "loss": 0.1664,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.21875,
      "step": 10310
    },
    {
      "epoch": 0.7449649895329532,
      "grad_norm": 9.704447459690613,
      "learning_rate": 9.270335973076468e-08,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.96875,
      "logps/chosen": -510.0,
      "logps/rejected": -916.0,
      "loss": 0.1623,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.3125,
      "step": 10320
    },
    {
      "epoch": 0.7456868548328882,
      "grad_norm": 13.178860773129225,
      "learning_rate": 9.221420368651045e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.484375,
      "logps/chosen": -544.0,
      "logps/rejected": -920.0,
      "loss": 0.1844,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.5625,
      "step": 10330
    },
    {
      "epoch": 0.7464087201328232,
      "grad_norm": 13.040643517577488,
      "learning_rate": 9.172604958638008e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -2.984375,
      "logps/chosen": -532.0,
      "logps/rejected": -920.0,
      "loss": 0.1678,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.3125,
      "step": 10340
    },
    {
      "epoch": 0.7471305854327582,
      "grad_norm": 17.76660313199596,
      "learning_rate": 9.123890053016545e-08,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.28125,
      "logps/chosen": -520.0,
      "logps/rejected": -908.0,
      "loss": 0.1572,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.875,
      "rewards/rejected": -7.28125,
      "step": 10350
    },
    {
      "epoch": 0.7478524507326932,
      "grad_norm": 13.756367345767458,
      "learning_rate": 9.07527596112764e-08,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.140625,
      "logps/chosen": -524.0,
      "logps/rejected": -904.0,
      "loss": 0.1704,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.21875,
      "step": 10360
    },
    {
      "epoch": 0.7485743160326284,
      "grad_norm": 11.724015133095397,
      "learning_rate": 9.026762991672105e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.0,
      "logps/chosen": -572.0,
      "logps/rejected": -976.0,
      "loss": 0.1559,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.734375,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.875,
      "step": 10370
    },
    {
      "epoch": 0.7492961813325634,
      "grad_norm": 11.689000630303346,
      "learning_rate": 8.978351452708626e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -3.015625,
      "logps/chosen": -548.0,
      "logps/rejected": -920.0,
      "loss": 0.1744,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.625,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.4375,
      "step": 10380
    },
    {
      "epoch": 0.7500180466324984,
      "grad_norm": 13.579319774025386,
      "learning_rate": 8.930041651651795e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.09375,
      "logps/chosen": -568.0,
      "logps/rejected": -988.0,
      "loss": 0.1728,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.6875,
      "rewards/margins": 4.28125,
      "rewards/rejected": -7.96875,
      "step": 10390
    },
    {
      "epoch": 0.7507399119324334,
      "grad_norm": 16.78019503585487,
      "learning_rate": 8.881833895270186e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.4375,
      "logps/chosen": -548.0,
      "logps/rejected": -916.0,
      "loss": 0.1532,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.640625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.4375,
      "step": 10400
    },
    {
      "epoch": 0.7514617772323684,
      "grad_norm": 13.8293823768564,
      "learning_rate": 8.833728489684341e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.84375,
      "logps/chosen": -544.0,
      "logps/rejected": -964.0,
      "loss": 0.1445,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.65625,
      "step": 10410
    },
    {
      "epoch": 0.7521836425323035,
      "grad_norm": 10.713979138476,
      "learning_rate": 8.78572574036493e-08,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.484375,
      "logps/chosen": -552.0,
      "logps/rejected": -904.0,
      "loss": 0.1643,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.46875,
      "step": 10420
    },
    {
      "epoch": 0.7529055078322385,
      "grad_norm": 10.294896759625827,
      "learning_rate": 8.73782595213072e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.234375,
      "logps/chosen": -536.0,
      "logps/rejected": -956.0,
      "loss": 0.1713,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.375,
      "rewards/rejected": -7.78125,
      "step": 10430
    },
    {
      "epoch": 0.7536273731321735,
      "grad_norm": 12.532005184244243,
      "learning_rate": 8.690029429146656e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.140625,
      "logps/chosen": -536.0,
      "logps/rejected": -916.0,
      "loss": 0.1825,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.421875,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.46875,
      "step": 10440
    },
    {
      "epoch": 0.7543492384321085,
      "grad_norm": 14.261516552752152,
      "learning_rate": 8.642336474922e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.078125,
      "logps/chosen": -488.0,
      "logps/rejected": -944.0,
      "loss": 0.1707,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.0625,
      "rewards/margins": 4.75,
      "rewards/rejected": -7.8125,
      "step": 10450
    },
    {
      "epoch": 0.7550711037320436,
      "grad_norm": 11.206538798549959,
      "learning_rate": 8.594747392308288e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.0,
      "logps/chosen": -524.0,
      "logps/rejected": -900.0,
      "loss": 0.1565,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.09375,
      "step": 10460
    },
    {
      "epoch": 0.7557929690319787,
      "grad_norm": 10.263847455305772,
      "learning_rate": 8.5472624834975e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.15625,
      "logps/chosen": -506.0,
      "logps/rejected": -916.0,
      "loss": 0.1365,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.046875,
      "rewards/margins": 4.40625,
      "rewards/rejected": -7.4375,
      "step": 10470
    },
    {
      "epoch": 0.7565148343319137,
      "grad_norm": 17.837857236337555,
      "learning_rate": 8.499882050020129e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.765625,
      "logps/chosen": -528.0,
      "logps/rejected": -948.0,
      "loss": 0.1836,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.265625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -7.59375,
      "step": 10480
    },
    {
      "epoch": 0.7572366996318487,
      "grad_norm": 13.401306628908374,
      "learning_rate": 8.4526063927432e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.28125,
      "logps/chosen": -544.0,
      "logps/rejected": -852.0,
      "loss": 0.1931,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.609375,
      "rewards/margins": 3.34375,
      "rewards/rejected": -6.9375,
      "step": 10490
    },
    {
      "epoch": 0.7579585649317837,
      "grad_norm": 12.547271728466068,
      "learning_rate": 8.40543581186843e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -3.015625,
      "logps/chosen": -524.0,
      "logps/rejected": -884.0,
      "loss": 0.1574,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.96875,
      "step": 10500
    },
    {
      "epoch": 0.7586804302317187,
      "grad_norm": 13.260023448874522,
      "learning_rate": 8.358370606930324e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.296875,
      "logps/chosen": -520.0,
      "logps/rejected": -892.0,
      "loss": 0.1593,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.28125,
      "step": 10510
    },
    {
      "epoch": 0.7594022955316538,
      "grad_norm": 13.798388008034175,
      "learning_rate": 8.311411076794195e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.109375,
      "logps/chosen": -532.0,
      "logps/rejected": -892.0,
      "loss": 0.1736,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.296875,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.1875,
      "step": 10520
    },
    {
      "epoch": 0.7601241608315888,
      "grad_norm": 10.81225735850356,
      "learning_rate": 8.264557519654353e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -3.109375,
      "logps/chosen": -532.0,
      "logps/rejected": -884.0,
      "loss": 0.178,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.421875,
      "rewards/margins": 3.65625,
      "rewards/rejected": -7.0625,
      "step": 10530
    },
    {
      "epoch": 0.7608460261315239,
      "grad_norm": 12.088528618093111,
      "learning_rate": 8.217810233032168e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.953125,
      "logps/chosen": -520.0,
      "logps/rejected": -876.0,
      "loss": 0.1643,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.65625,
      "rewards/rejected": -6.96875,
      "step": 10540
    },
    {
      "epoch": 0.7615678914314589,
      "grad_norm": 13.265131887373293,
      "learning_rate": 8.171169513774189e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.640625,
      "logps/chosen": -536.0,
      "logps/rejected": -944.0,
      "loss": 0.1645,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 3.984375,
      "rewards/rejected": -7.5625,
      "step": 10550
    },
    {
      "epoch": 0.7622897567313939,
      "grad_norm": 14.103079856248533,
      "learning_rate": 8.12463565805026e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.828125,
      "logps/chosen": -532.0,
      "logps/rejected": -920.0,
      "loss": 0.1616,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.40625,
      "step": 10560
    },
    {
      "epoch": 0.763011622031329,
      "grad_norm": 13.632525765668166,
      "learning_rate": 8.078208961351637e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.96875,
      "logps/chosen": -564.0,
      "logps/rejected": -940.0,
      "loss": 0.1763,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.640625,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.5625,
      "step": 10570
    },
    {
      "epoch": 0.763733487331264,
      "grad_norm": 9.770618582491265,
      "learning_rate": 8.031889718489124e-08,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.984375,
      "logps/chosen": -536.0,
      "logps/rejected": -952.0,
      "loss": 0.1663,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.515625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.71875,
      "step": 10580
    },
    {
      "epoch": 0.764455352631199,
      "grad_norm": 12.759122757829408,
      "learning_rate": 7.985678223591155e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.703125,
      "logps/chosen": -532.0,
      "logps/rejected": -948.0,
      "loss": 0.1713,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.375,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.5,
      "step": 10590
    },
    {
      "epoch": 0.765177217931134,
      "grad_norm": 13.516999431924708,
      "learning_rate": 7.939574770102011e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.84375,
      "logps/chosen": -520.0,
      "logps/rejected": -904.0,
      "loss": 0.1693,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.265625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.3125,
      "step": 10600
    },
    {
      "epoch": 0.765899083231069,
      "grad_norm": 13.394439209902245,
      "learning_rate": 7.893579650779883e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.078125,
      "logps/chosen": -516.0,
      "logps/rejected": -900.0,
      "loss": 0.1432,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.265625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.3125,
      "step": 10610
    },
    {
      "epoch": 0.7666209485310042,
      "grad_norm": 17.872099419791386,
      "learning_rate": 7.847693157695015e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.921875,
      "logps/chosen": -548.0,
      "logps/rejected": -924.0,
      "loss": 0.1676,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.59375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.5,
      "step": 10620
    },
    {
      "epoch": 0.7673428138309392,
      "grad_norm": 18.522346334877977,
      "learning_rate": 7.801915582227916e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.25,
      "logps/chosen": -552.0,
      "logps/rejected": -916.0,
      "loss": 0.1751,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.625,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.4375,
      "step": 10630
    },
    {
      "epoch": 0.7680646791308742,
      "grad_norm": 11.303586270574135,
      "learning_rate": 7.756247215067444e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.0625,
      "logps/chosen": -540.0,
      "logps/rejected": -944.0,
      "loss": 0.1716,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.5625,
      "step": 10640
    },
    {
      "epoch": 0.7687865444308092,
      "grad_norm": 9.827345047745544,
      "learning_rate": 7.710688346208952e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.765625,
      "logps/chosen": -528.0,
      "logps/rejected": -932.0,
      "loss": 0.1583,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.46875,
      "step": 10650
    },
    {
      "epoch": 0.7695084097307442,
      "grad_norm": 11.467357741453178,
      "learning_rate": 7.665239264952541e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.03125,
      "logps/chosen": -500.0,
      "logps/rejected": -904.0,
      "loss": 0.1705,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.171875,
      "rewards/margins": 4.3125,
      "rewards/rejected": -7.46875,
      "step": 10660
    },
    {
      "epoch": 0.7702302750306793,
      "grad_norm": 14.291013053006402,
      "learning_rate": 7.619900259901097e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -3.328125,
      "logps/chosen": -528.0,
      "logps/rejected": -900.0,
      "loss": 0.1401,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.9375,
      "rewards/rejected": -7.3125,
      "step": 10670
    },
    {
      "epoch": 0.7709521403306143,
      "grad_norm": 12.721448629054342,
      "learning_rate": 7.574671618958544e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.25,
      "logps/chosen": -524.0,
      "logps/rejected": -940.0,
      "loss": 0.1524,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.328125,
      "rewards/margins": 4.375,
      "rewards/rejected": -7.71875,
      "step": 10680
    },
    {
      "epoch": 0.7716740056305493,
      "grad_norm": 9.892841085736922,
      "learning_rate": 7.529553629327994e-08,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.09375,
      "logps/chosen": -536.0,
      "logps/rejected": -952.0,
      "loss": 0.1467,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.53125,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.6875,
      "step": 10690
    },
    {
      "epoch": 0.7723958709304843,
      "grad_norm": 16.251645624591912,
      "learning_rate": 7.484546577509918e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.046875,
      "logps/chosen": -548.0,
      "logps/rejected": -948.0,
      "loss": 0.1741,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.515625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.71875,
      "step": 10700
    },
    {
      "epoch": 0.7731177362304194,
      "grad_norm": 11.90502305646809,
      "learning_rate": 7.439650749300322e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.78125,
      "logps/chosen": -556.0,
      "logps/rejected": -928.0,
      "loss": 0.1753,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.953125,
      "rewards/rejected": -7.3125,
      "step": 10710
    },
    {
      "epoch": 0.7738396015303545,
      "grad_norm": 8.861815337799422,
      "learning_rate": 7.394866429788945e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -3.25,
      "logps/chosen": -520.0,
      "logps/rejected": -944.0,
      "loss": 0.1493,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.375,
      "rewards/margins": 4.34375,
      "rewards/rejected": -7.6875,
      "step": 10720
    },
    {
      "epoch": 0.7745614668302895,
      "grad_norm": 10.623269974351793,
      "learning_rate": 7.350193903357444e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -3.109375,
      "logps/chosen": -516.0,
      "logps/rejected": -900.0,
      "loss": 0.1786,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.34375,
      "rewards/margins": 3.984375,
      "rewards/rejected": -7.3125,
      "step": 10730
    },
    {
      "epoch": 0.7752833321302245,
      "grad_norm": 7.603870664079461,
      "learning_rate": 7.305633453677579e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -3.125,
      "logps/chosen": -544.0,
      "logps/rejected": -964.0,
      "loss": 0.1641,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.5,
      "rewards/rejected": -7.90625,
      "step": 10740
    },
    {
      "epoch": 0.7760051974301595,
      "grad_norm": 12.201854546288375,
      "learning_rate": 7.26118536370943e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.28125,
      "logps/chosen": -560.0,
      "logps/rejected": -968.0,
      "loss": 0.1549,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -7.875,
      "step": 10750
    },
    {
      "epoch": 0.7767270627300946,
      "grad_norm": 12.201655257946776,
      "learning_rate": 7.21684991569958e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.953125,
      "logps/chosen": -552.0,
      "logps/rejected": -928.0,
      "loss": 0.1655,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.578125,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.5625,
      "step": 10760
    },
    {
      "epoch": 0.7774489280300296,
      "grad_norm": 13.389699272877754,
      "learning_rate": 7.17262739117934e-08,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.109375,
      "logps/chosen": -502.0,
      "logps/rejected": -904.0,
      "loss": 0.1776,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.359375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.375,
      "step": 10770
    },
    {
      "epoch": 0.7781707933299646,
      "grad_norm": 7.634244774612653,
      "learning_rate": 7.128518070962947e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.9375,
      "logps/chosen": -572.0,
      "logps/rejected": -924.0,
      "loss": 0.155,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.671875,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.46875,
      "step": 10780
    },
    {
      "epoch": 0.7788926586298996,
      "grad_norm": 11.334214479521611,
      "learning_rate": 7.084522235145796e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.109375,
      "logps/chosen": -564.0,
      "logps/rejected": -888.0,
      "loss": 0.1689,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 3.671875,
      "rewards/rejected": -7.25,
      "step": 10790
    },
    {
      "epoch": 0.7796145239298347,
      "grad_norm": 8.818506357399086,
      "learning_rate": 7.040640163102646e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.15625,
      "logps/chosen": -536.0,
      "logps/rejected": -1000.0,
      "loss": 0.1625,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.5625,
      "rewards/margins": 4.71875,
      "rewards/rejected": -8.25,
      "step": 10800
    },
    {
      "epoch": 0.7803363892297698,
      "grad_norm": 14.798962957637091,
      "learning_rate": 6.996872133485854e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.765625,
      "logps/chosen": -532.0,
      "logps/rejected": -888.0,
      "loss": 0.1702,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.828125,
      "rewards/rejected": -7.28125,
      "step": 10810
    },
    {
      "epoch": 0.7810582545297048,
      "grad_norm": 14.326033325866666,
      "learning_rate": 6.953218424223617e-08,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.3125,
      "logps/chosen": -528.0,
      "logps/rejected": -876.0,
      "loss": 0.1495,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.671875,
      "rewards/rejected": -7.125,
      "step": 10820
    },
    {
      "epoch": 0.7817801198296398,
      "grad_norm": 13.715545236093703,
      "learning_rate": 6.909679312518163e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.84375,
      "logps/chosen": -528.0,
      "logps/rejected": -944.0,
      "loss": 0.1736,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.390625,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.59375,
      "step": 10830
    },
    {
      "epoch": 0.7825019851295748,
      "grad_norm": 11.011312632934782,
      "learning_rate": 6.866255074844046e-08,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.015625,
      "logps/chosen": -512.0,
      "logps/rejected": -904.0,
      "loss": 0.1485,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.3125,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.3125,
      "step": 10840
    },
    {
      "epoch": 0.7832238504295098,
      "grad_norm": 10.242386151593784,
      "learning_rate": 6.822945986946385e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -3.046875,
      "logps/chosen": -544.0,
      "logps/rejected": -920.0,
      "loss": 0.1685,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.484375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.5625,
      "step": 10850
    },
    {
      "epoch": 0.7839457157294449,
      "grad_norm": 11.2954505935089,
      "learning_rate": 6.77975232383905e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.890625,
      "logps/chosen": -504.0,
      "logps/rejected": -992.0,
      "loss": 0.1681,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.21875,
      "rewards/margins": 4.8125,
      "rewards/rejected": -8.0,
      "step": 10860
    },
    {
      "epoch": 0.78466758102938,
      "grad_norm": 10.278620391956363,
      "learning_rate": 6.736674359802986e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.171875,
      "logps/chosen": -544.0,
      "logps/rejected": -900.0,
      "loss": 0.1645,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.25,
      "step": 10870
    },
    {
      "epoch": 0.785389446329315,
      "grad_norm": 9.199248383936057,
      "learning_rate": 6.693712368384463e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.328125,
      "logps/chosen": -540.0,
      "logps/rejected": -920.0,
      "loss": 0.158,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.53125,
      "step": 10880
    },
    {
      "epoch": 0.78611131162925,
      "grad_norm": 14.764651568469134,
      "learning_rate": 6.650866622393281e-08,
      "logits/chosen": -3.125,
      "logits/rejected": -3.28125,
      "logps/chosen": -552.0,
      "logps/rejected": -976.0,
      "loss": 0.1734,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.796875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.9375,
      "step": 10890
    },
    {
      "epoch": 0.786833176929185,
      "grad_norm": 15.36538068615802,
      "learning_rate": 6.608137393901106e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.03125,
      "logps/chosen": -568.0,
      "logps/rejected": -960.0,
      "loss": 0.1799,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.6875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.78125,
      "step": 10900
    },
    {
      "epoch": 0.7875550422291201,
      "grad_norm": 10.875309966433587,
      "learning_rate": 6.565524954239709e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.40625,
      "logps/chosen": -540.0,
      "logps/rejected": -932.0,
      "loss": 0.1575,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.53125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.59375,
      "step": 10910
    },
    {
      "epoch": 0.7882769075290551,
      "grad_norm": 11.383336057891066,
      "learning_rate": 6.523029573999247e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.96875,
      "logps/chosen": -528.0,
      "logps/rejected": -928.0,
      "loss": 0.1436,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.4375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.46875,
      "step": 10920
    },
    {
      "epoch": 0.7889987728289901,
      "grad_norm": 12.276607985485473,
      "learning_rate": 6.480651523026548e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.859375,
      "logps/chosen": -528.0,
      "logps/rejected": -920.0,
      "loss": 0.1689,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.359375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.53125,
      "step": 10930
    },
    {
      "epoch": 0.7897206381289251,
      "grad_norm": 13.787353287422713,
      "learning_rate": 6.438391070423396e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -3.03125,
      "logps/chosen": -532.0,
      "logps/rejected": -932.0,
      "loss": 0.1628,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.53125,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.53125,
      "step": 10940
    },
    {
      "epoch": 0.7904425034288601,
      "grad_norm": 10.958034944513365,
      "learning_rate": 6.396248484544817e-08,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.140625,
      "logps/chosen": -512.0,
      "logps/rejected": -916.0,
      "loss": 0.1627,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.4375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.53125,
      "step": 10950
    },
    {
      "epoch": 0.7911643687287953,
      "grad_norm": 16.777503537854884,
      "learning_rate": 6.354224032997391e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.25,
      "logps/chosen": -576.0,
      "logps/rejected": -904.0,
      "loss": 0.1603,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.734375,
      "rewards/margins": 3.546875,
      "rewards/rejected": -7.28125,
      "step": 10960
    },
    {
      "epoch": 0.7918862340287303,
      "grad_norm": 19.47722314156324,
      "learning_rate": 6.31231798263753e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.296875,
      "logps/chosen": -520.0,
      "logps/rejected": -920.0,
      "loss": 0.1719,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.3125,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.4375,
      "step": 10970
    },
    {
      "epoch": 0.7926080993286653,
      "grad_norm": 11.879151295383368,
      "learning_rate": 6.27053059956981e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.984375,
      "logps/chosen": -540.0,
      "logps/rejected": -904.0,
      "loss": 0.1592,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.828125,
      "rewards/rejected": -7.1875,
      "step": 10980
    },
    {
      "epoch": 0.7933299646286003,
      "grad_norm": 9.483979494501549,
      "learning_rate": 6.228862149145234e-08,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.390625,
      "logps/chosen": -560.0,
      "logps/rejected": -916.0,
      "loss": 0.1658,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.71875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -7.4375,
      "step": 10990
    },
    {
      "epoch": 0.7940518299285353,
      "grad_norm": 15.755614858528979,
      "learning_rate": 6.187312895959623e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.859375,
      "logps/chosen": -512.0,
      "logps/rejected": -928.0,
      "loss": 0.1673,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.34375,
      "rewards/margins": 4.28125,
      "rewards/rejected": -7.625,
      "step": 11000
    },
    {
      "epoch": 0.7947736952284704,
      "grad_norm": 16.833253840092855,
      "learning_rate": 6.145883103851876e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.765625,
      "logps/chosen": -552.0,
      "logps/rejected": -912.0,
      "loss": 0.1709,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.625,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.4375,
      "step": 11010
    },
    {
      "epoch": 0.7954955605284054,
      "grad_norm": 10.152413531076892,
      "learning_rate": 6.10457303590228e-08,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.875,
      "logps/chosen": -536.0,
      "logps/rejected": -940.0,
      "loss": 0.1637,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.53125,
      "rewards/margins": 3.984375,
      "rewards/rejected": -7.53125,
      "step": 11020
    },
    {
      "epoch": 0.7962174258283404,
      "grad_norm": 18.61168833663538,
      "learning_rate": 6.063382954430921e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.140625,
      "logps/chosen": -512.0,
      "logps/rejected": -904.0,
      "loss": 0.1694,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.4375,
      "step": 11030
    },
    {
      "epoch": 0.7969392911282754,
      "grad_norm": 13.201897960727187,
      "learning_rate": 6.022313120995942e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.140625,
      "logps/chosen": -532.0,
      "logps/rejected": -932.0,
      "loss": 0.1771,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.546875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.625,
      "step": 11040
    },
    {
      "epoch": 0.7976611564282104,
      "grad_norm": 14.077224631797534,
      "learning_rate": 5.981363796391889e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.21875,
      "logps/chosen": -540.0,
      "logps/rejected": -900.0,
      "loss": 0.1539,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.546875,
      "rewards/margins": 3.65625,
      "rewards/rejected": -7.1875,
      "step": 11050
    },
    {
      "epoch": 0.7983830217281456,
      "grad_norm": 10.494979358459764,
      "learning_rate": 5.9405352406481226e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -3.125,
      "logps/chosen": -532.0,
      "logps/rejected": -924.0,
      "loss": 0.1651,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.34375,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.5,
      "step": 11060
    },
    {
      "epoch": 0.7991048870280806,
      "grad_norm": 17.98065314156054,
      "learning_rate": 5.899827713027064e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.03125,
      "logps/chosen": -536.0,
      "logps/rejected": -888.0,
      "loss": 0.1852,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.125,
      "step": 11070
    },
    {
      "epoch": 0.7998267523280156,
      "grad_norm": 13.97757651750582,
      "learning_rate": 5.859241472022633e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.171875,
      "logps/chosen": -540.0,
      "logps/rejected": -924.0,
      "loss": 0.1516,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 3.875,
      "rewards/rejected": -7.4375,
      "step": 11080
    },
    {
      "epoch": 0.8005486176279506,
      "grad_norm": 10.867966345561388,
      "learning_rate": 5.818776775358586e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.3125,
      "logps/chosen": -564.0,
      "logps/rejected": -904.0,
      "loss": 0.166,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.671875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -7.375,
      "step": 11090
    },
    {
      "epoch": 0.8012704829278856,
      "grad_norm": 12.796251797570147,
      "learning_rate": 5.778433879986835e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.96875,
      "logps/chosen": -520.0,
      "logps/rejected": -908.0,
      "loss": 0.1691,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.3125,
      "step": 11100
    },
    {
      "epoch": 0.8019923482278207,
      "grad_norm": 11.822403640425167,
      "learning_rate": 5.7382130420858794e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.96875,
      "logps/chosen": -552.0,
      "logps/rejected": -912.0,
      "loss": 0.1795,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.640625,
      "rewards/margins": 3.765625,
      "rewards/rejected": -7.40625,
      "step": 11110
    },
    {
      "epoch": 0.8027142135277557,
      "grad_norm": 16.88410757995049,
      "learning_rate": 5.698114517059135e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.203125,
      "logps/chosen": -568.0,
      "logps/rejected": -916.0,
      "loss": 0.1682,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.671875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -7.375,
      "step": 11120
    },
    {
      "epoch": 0.8034360788276907,
      "grad_norm": 12.955186466099862,
      "learning_rate": 5.658138559533338e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.1875,
      "logps/chosen": -524.0,
      "logps/rejected": -924.0,
      "loss": 0.1853,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.59375,
      "step": 11130
    },
    {
      "epoch": 0.8041579441276258,
      "grad_norm": 13.532716059195163,
      "learning_rate": 5.6182854233569084e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -2.96875,
      "logps/chosen": -548.0,
      "logps/rejected": -904.0,
      "loss": 0.1684,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.796875,
      "rewards/rejected": -7.25,
      "step": 11140
    },
    {
      "epoch": 0.8048798094275608,
      "grad_norm": 17.479746841447046,
      "learning_rate": 5.578555361598353e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.140625,
      "logps/chosen": -512.0,
      "logps/rejected": -940.0,
      "loss": 0.1513,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.625,
      "step": 11150
    },
    {
      "epoch": 0.8056016747274959,
      "grad_norm": 9.948778469232593,
      "learning_rate": 5.538948626544651e-08,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.28125,
      "logps/chosen": -536.0,
      "logps/rejected": -904.0,
      "loss": 0.1575,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.609375,
      "rewards/margins": 3.734375,
      "rewards/rejected": -7.34375,
      "step": 11160
    },
    {
      "epoch": 0.8063235400274309,
      "grad_norm": 12.631614430826355,
      "learning_rate": 5.4994654696996545e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.90625,
      "logps/chosen": -536.0,
      "logps/rejected": -948.0,
      "loss": 0.1744,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.5625,
      "step": 11170
    },
    {
      "epoch": 0.8070454053273659,
      "grad_norm": 9.39771926263778,
      "learning_rate": 5.460106141782492e-08,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.078125,
      "logps/chosen": -528.0,
      "logps/rejected": -916.0,
      "loss": 0.1806,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.25,
      "step": 11180
    },
    {
      "epoch": 0.8077672706273009,
      "grad_norm": 19.32514127168474,
      "learning_rate": 5.420870892725965e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.140625,
      "logps/chosen": -536.0,
      "logps/rejected": -892.0,
      "loss": 0.1667,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.25,
      "step": 11190
    },
    {
      "epoch": 0.8084891359272359,
      "grad_norm": 12.27443466056232,
      "learning_rate": 5.381759971674987e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.921875,
      "logps/chosen": -548.0,
      "logps/rejected": -936.0,
      "loss": 0.1728,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.515625,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.625,
      "step": 11200
    },
    {
      "epoch": 0.809211001227171,
      "grad_norm": 15.15657267682835,
      "learning_rate": 5.3427736269849724e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.28125,
      "logps/chosen": -504.0,
      "logps/rejected": -860.0,
      "loss": 0.1663,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.0625,
      "step": 11210
    },
    {
      "epoch": 0.809932866527106,
      "grad_norm": 10.90481187087727,
      "learning_rate": 5.303912106220285e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -3.265625,
      "logps/chosen": -524.0,
      "logps/rejected": -900.0,
      "loss": 0.1527,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.984375,
      "rewards/rejected": -7.375,
      "step": 11220
    },
    {
      "epoch": 0.8106547318270411,
      "grad_norm": 11.516853498939279,
      "learning_rate": 5.265175656152621e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.234375,
      "logps/chosen": -552.0,
      "logps/rejected": -888.0,
      "loss": 0.1749,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.75,
      "rewards/margins": 3.546875,
      "rewards/rejected": -7.3125,
      "step": 11230
    },
    {
      "epoch": 0.8113765971269761,
      "grad_norm": 15.300191113210456,
      "learning_rate": 5.226564522759525e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.90625,
      "logps/chosen": -552.0,
      "logps/rejected": -900.0,
      "loss": 0.1744,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.6875,
      "rewards/margins": 3.53125,
      "rewards/rejected": -7.21875,
      "step": 11240
    },
    {
      "epoch": 0.8120984624269111,
      "grad_norm": 15.366551287473474,
      "learning_rate": 5.1880789512227434e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.84375,
      "logps/chosen": -528.0,
      "logps/rejected": -928.0,
      "loss": 0.1733,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.828125,
      "rewards/rejected": -7.25,
      "step": 11250
    },
    {
      "epoch": 0.8128203277268462,
      "grad_norm": 16.85525877485589,
      "learning_rate": 5.1497191859267014e-08,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.28125,
      "logps/chosen": -516.0,
      "logps/rejected": -884.0,
      "loss": 0.1862,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.28125,
      "step": 11260
    },
    {
      "epoch": 0.8135421930267812,
      "grad_norm": 14.707705096257053,
      "learning_rate": 5.111485470456953e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -3.046875,
      "logps/chosen": -520.0,
      "logps/rejected": -916.0,
      "loss": 0.1509,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.3125,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.4375,
      "step": 11270
    },
    {
      "epoch": 0.8142640583267162,
      "grad_norm": 12.010026321333243,
      "learning_rate": 5.0733780475986617e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.75,
      "logps/chosen": -532.0,
      "logps/rejected": -920.0,
      "loss": 0.1684,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.515625,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.375,
      "step": 11280
    },
    {
      "epoch": 0.8149859236266512,
      "grad_norm": 12.816377492748522,
      "learning_rate": 5.035397159334984e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.90625,
      "logps/chosen": -568.0,
      "logps/rejected": -884.0,
      "loss": 0.1582,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.609375,
      "rewards/margins": 3.453125,
      "rewards/rejected": -7.0625,
      "step": 11290
    },
    {
      "epoch": 0.8157077889265864,
      "grad_norm": 11.9706262821004,
      "learning_rate": 4.997543046845604e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -3.15625,
      "logps/chosen": -512.0,
      "logps/rejected": -960.0,
      "loss": 0.1573,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.375,
      "rewards/margins": 4.28125,
      "rewards/rejected": -7.65625,
      "step": 11300
    },
    {
      "epoch": 0.8164296542265214,
      "grad_norm": 10.026191502708413,
      "learning_rate": 4.959815950505175e-08,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.421875,
      "logps/chosen": -536.0,
      "logps/rejected": -912.0,
      "loss": 0.1596,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.640625,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.53125,
      "step": 11310
    },
    {
      "epoch": 0.8171515195264564,
      "grad_norm": 13.631301934918074,
      "learning_rate": 4.922216109881791e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.25,
      "logps/chosen": -532.0,
      "logps/rejected": -896.0,
      "loss": 0.1685,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.515625,
      "rewards/margins": 3.734375,
      "rewards/rejected": -7.25,
      "step": 11320
    },
    {
      "epoch": 0.8178733848263914,
      "grad_norm": 9.305627832329417,
      "learning_rate": 4.8847437637354664e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.921875,
      "logps/chosen": -508.0,
      "logps/rejected": -936.0,
      "loss": 0.1682,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.296875,
      "rewards/margins": 4.4375,
      "rewards/rejected": -7.71875,
      "step": 11330
    },
    {
      "epoch": 0.8185952501263264,
      "grad_norm": 12.354228049396218,
      "learning_rate": 4.8473991500166234e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.0,
      "logps/chosen": -516.0,
      "logps/rejected": -912.0,
      "loss": 0.1382,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.328125,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.28125,
      "step": 11340
    },
    {
      "epoch": 0.8193171154262615,
      "grad_norm": 14.99801550963248,
      "learning_rate": 4.810182505864585e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.15625,
      "logps/chosen": -552.0,
      "logps/rejected": -960.0,
      "loss": 0.1656,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 4.4375,
      "rewards/rejected": -7.84375,
      "step": 11350
    },
    {
      "epoch": 0.8200389807261965,
      "grad_norm": 11.135652437951032,
      "learning_rate": 4.7730940676060564e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.09375,
      "logps/chosen": -552.0,
      "logps/rejected": -912.0,
      "loss": 0.1536,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.609375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.375,
      "step": 11360
    },
    {
      "epoch": 0.8207608460261315,
      "grad_norm": 12.025831269278603,
      "learning_rate": 4.736134070753639e-08,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.046875,
      "logps/chosen": -520.0,
      "logps/rejected": -928.0,
      "loss": 0.1522,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.625,
      "step": 11370
    },
    {
      "epoch": 0.8214827113260665,
      "grad_norm": 13.936064129778199,
      "learning_rate": 4.6993027500043275e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.09375,
      "logps/chosen": -548.0,
      "logps/rejected": -964.0,
      "loss": 0.1599,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.53125,
      "rewards/margins": 4.28125,
      "rewards/rejected": -7.8125,
      "step": 11380
    },
    {
      "epoch": 0.8222045766260015,
      "grad_norm": 15.807781675899243,
      "learning_rate": 4.662600339237999e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.140625,
      "logps/chosen": -576.0,
      "logps/rejected": -900.0,
      "loss": 0.1752,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.640625,
      "rewards/margins": 3.625,
      "rewards/rejected": -7.28125,
      "step": 11390
    },
    {
      "epoch": 0.8229264419259367,
      "grad_norm": 12.363819755349816,
      "learning_rate": 4.6260270715159836e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.328125,
      "logps/chosen": -516.0,
      "logps/rejected": -900.0,
      "loss": 0.1795,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.21875,
      "step": 11400
    },
    {
      "epoch": 0.8236483072258717,
      "grad_norm": 8.662814753819399,
      "learning_rate": 4.589583179079531e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.1875,
      "logps/chosen": -516.0,
      "logps/rejected": -932.0,
      "loss": 0.145,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 4.3125,
      "rewards/rejected": -7.71875,
      "step": 11410
    },
    {
      "epoch": 0.8243701725258067,
      "grad_norm": 14.002406289377165,
      "learning_rate": 4.553268893348339e-08,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.875,
      "logps/chosen": -552.0,
      "logps/rejected": -1000.0,
      "loss": 0.1698,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.515625,
      "rewards/margins": 4.53125,
      "rewards/rejected": -8.0625,
      "step": 11420
    },
    {
      "epoch": 0.8250920378257417,
      "grad_norm": 17.722383936444416,
      "learning_rate": 4.5170844449191364e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -3.1875,
      "logps/chosen": -544.0,
      "logps/rejected": -944.0,
      "loss": 0.1763,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.609375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.6875,
      "step": 11430
    },
    {
      "epoch": 0.8258139031256767,
      "grad_norm": 10.346479463118243,
      "learning_rate": 4.481030063564156e-08,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.0,
      "logps/chosen": -556.0,
      "logps/rejected": -924.0,
      "loss": 0.1665,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.75,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.59375,
      "step": 11440
    },
    {
      "epoch": 0.8265357684256118,
      "grad_norm": 16.113070530674534,
      "learning_rate": 4.445105978229688e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.84375,
      "logps/chosen": -524.0,
      "logps/rejected": -964.0,
      "loss": 0.1745,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.21875,
      "rewards/margins": 4.71875,
      "rewards/rejected": -7.9375,
      "step": 11450
    },
    {
      "epoch": 0.8272576337255468,
      "grad_norm": 16.733489417256834,
      "learning_rate": 4.409312417034683e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.1875,
      "logps/chosen": -516.0,
      "logps/rejected": -928.0,
      "loss": 0.1573,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.328125,
      "rewards/margins": 4.25,
      "rewards/rejected": -7.5625,
      "step": 11460
    },
    {
      "epoch": 0.8279794990254818,
      "grad_norm": 10.975319173406424,
      "learning_rate": 4.373649607269217e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.265625,
      "logps/chosen": -564.0,
      "logps/rejected": -932.0,
      "loss": 0.1582,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.640625,
      "rewards/margins": 3.9375,
      "rewards/rejected": -7.59375,
      "step": 11470
    },
    {
      "epoch": 0.8287013643254169,
      "grad_norm": 14.506786696191984,
      "learning_rate": 4.338117775393107e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.640625,
      "logps/chosen": -572.0,
      "logps/rejected": -920.0,
      "loss": 0.1734,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.78125,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.59375,
      "step": 11480
    },
    {
      "epoch": 0.8294232296253519,
      "grad_norm": 13.07832503841599,
      "learning_rate": 4.3027171470344767e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.078125,
      "logps/chosen": -552.0,
      "logps/rejected": -936.0,
      "loss": 0.1542,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.5,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.5625,
      "step": 11490
    },
    {
      "epoch": 0.830145094925287,
      "grad_norm": 11.343244901876497,
      "learning_rate": 4.267447946988262e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -3.140625,
      "logps/chosen": -552.0,
      "logps/rejected": -884.0,
      "loss": 0.1733,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.53125,
      "rewards/margins": 3.53125,
      "rewards/rejected": -7.0625,
      "step": 11500
    },
    {
      "epoch": 0.830866960225222,
      "grad_norm": 11.623107275540574,
      "learning_rate": 4.232310399214853e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.109375,
      "logps/chosen": -504.0,
      "logps/rejected": -948.0,
      "loss": 0.1757,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 4.3125,
      "rewards/rejected": -7.59375,
      "step": 11510
    },
    {
      "epoch": 0.831588825525157,
      "grad_norm": 9.746838980066709,
      "learning_rate": 4.1973047268386545e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.015625,
      "logps/chosen": -520.0,
      "logps/rejected": -952.0,
      "loss": 0.1428,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.34375,
      "rewards/margins": 4.375,
      "rewards/rejected": -7.71875,
      "step": 11520
    },
    {
      "epoch": 0.832310690825092,
      "grad_norm": 16.88542249318133,
      "learning_rate": 4.162431152146631e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.078125,
      "logps/chosen": -560.0,
      "logps/rejected": -920.0,
      "loss": 0.2009,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.609375,
      "rewards/margins": 3.671875,
      "rewards/rejected": -7.28125,
      "step": 11530
    },
    {
      "epoch": 0.833032556125027,
      "grad_norm": 12.140312787677615,
      "learning_rate": 4.127689896586936e-08,
      "logits/chosen": -2.8125,
      "logits/rejected": -3.171875,
      "logps/chosen": -540.0,
      "logps/rejected": -964.0,
      "loss": 0.1512,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.28125,
      "rewards/rejected": -7.6875,
      "step": 11540
    },
    {
      "epoch": 0.8337544214249621,
      "grad_norm": 11.908341620243924,
      "learning_rate": 4.093081180767494e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.265625,
      "logps/chosen": -536.0,
      "logps/rejected": -940.0,
      "loss": 0.1543,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.59375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.75,
      "step": 11550
    },
    {
      "epoch": 0.8344762867248972,
      "grad_norm": 11.532205435206242,
      "learning_rate": 4.0586052244546e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.15625,
      "logps/chosen": -560.0,
      "logps/rejected": -896.0,
      "loss": 0.1579,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.609375,
      "rewards/margins": 3.71875,
      "rewards/rejected": -7.34375,
      "step": 11560
    },
    {
      "epoch": 0.8351981520248322,
      "grad_norm": 12.588972501276345,
      "learning_rate": 4.0242622465715196e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.984375,
      "logps/chosen": -560.0,
      "logps/rejected": -944.0,
      "loss": 0.1534,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.671875,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.65625,
      "step": 11570
    },
    {
      "epoch": 0.8359200173247672,
      "grad_norm": 12.787691068894008,
      "learning_rate": 3.9900524651970995e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -3.171875,
      "logps/chosen": -544.0,
      "logps/rejected": -936.0,
      "loss": 0.1655,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.421875,
      "rewards/margins": 4.28125,
      "rewards/rejected": -7.71875,
      "step": 11580
    },
    {
      "epoch": 0.8366418826247022,
      "grad_norm": 14.89656293990655,
      "learning_rate": 3.9559760975643897e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.125,
      "logps/chosen": -536.0,
      "logps/rejected": -932.0,
      "loss": 0.1667,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.984375,
      "rewards/rejected": -7.5,
      "step": 11590
    },
    {
      "epoch": 0.8373637479246373,
      "grad_norm": 12.063195299073906,
      "learning_rate": 3.922033360059254e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.15625,
      "logps/chosen": -540.0,
      "logps/rejected": -912.0,
      "loss": 0.1619,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.515625,
      "rewards/margins": 3.9375,
      "rewards/rejected": -7.4375,
      "step": 11600
    },
    {
      "epoch": 0.8380856132245723,
      "grad_norm": 11.5120893310907,
      "learning_rate": 3.8882244682190094e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.34375,
      "logps/chosen": -516.0,
      "logps/rejected": -892.0,
      "loss": 0.157,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.4375,
      "step": 11610
    },
    {
      "epoch": 0.8388074785245073,
      "grad_norm": 12.70255070234517,
      "learning_rate": 3.854549636731047e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.09375,
      "logps/chosen": -516.0,
      "logps/rejected": -908.0,
      "loss": 0.1626,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.4375,
      "step": 11620
    },
    {
      "epoch": 0.8395293438244423,
      "grad_norm": 12.094344845396897,
      "learning_rate": 3.82100907943145e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.3125,
      "logps/chosen": -544.0,
      "logps/rejected": -928.0,
      "loss": 0.1808,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.59375,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.59375,
      "step": 11630
    },
    {
      "epoch": 0.8402512091243773,
      "grad_norm": 12.764982018596312,
      "learning_rate": 3.7876030093036945e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.265625,
      "logps/chosen": -524.0,
      "logps/rejected": -924.0,
      "loss": 0.1777,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.4375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.625,
      "step": 11640
    },
    {
      "epoch": 0.8409730744243125,
      "grad_norm": 15.100739052207306,
      "learning_rate": 3.7543316384772375e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -3.03125,
      "logps/chosen": -536.0,
      "logps/rejected": -924.0,
      "loss": 0.1656,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.359375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.40625,
      "step": 11650
    },
    {
      "epoch": 0.8416949397242475,
      "grad_norm": 10.018683532722237,
      "learning_rate": 3.721195178226178e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.15625,
      "logps/chosen": -504.0,
      "logps/rejected": -928.0,
      "loss": 0.1659,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.296875,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.46875,
      "step": 11660
    },
    {
      "epoch": 0.8424168050241825,
      "grad_norm": 14.335545825191495,
      "learning_rate": 3.68819383896796e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -3.15625,
      "logps/chosen": -532.0,
      "logps/rejected": -924.0,
      "loss": 0.165,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.984375,
      "rewards/rejected": -7.375,
      "step": 11670
    },
    {
      "epoch": 0.8431386703241175,
      "grad_norm": 10.125856871383494,
      "learning_rate": 3.655327830261995e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.25,
      "logps/chosen": -564.0,
      "logps/rejected": -896.0,
      "loss": 0.1711,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.625,
      "rewards/margins": 3.515625,
      "rewards/rejected": -7.125,
      "step": 11680
    },
    {
      "epoch": 0.8438605356240525,
      "grad_norm": 12.936495673886116,
      "learning_rate": 3.622597360808324e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -3.046875,
      "logps/chosen": -564.0,
      "logps/rejected": -972.0,
      "loss": 0.1821,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.625,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.75,
      "step": 11690
    },
    {
      "epoch": 0.8445824009239876,
      "grad_norm": 11.7262480689566,
      "learning_rate": 3.5900026384463324e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.0625,
      "logps/chosen": -524.0,
      "logps/rejected": -892.0,
      "loss": 0.1738,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.421875,
      "rewards/margins": 3.828125,
      "rewards/rejected": -7.25,
      "step": 11700
    },
    {
      "epoch": 0.8453042662239226,
      "grad_norm": 12.312194344113369,
      "learning_rate": 3.557543870153393e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.015625,
      "logps/chosen": -524.0,
      "logps/rejected": -936.0,
      "loss": 0.1647,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.5625,
      "step": 11710
    },
    {
      "epoch": 0.8460261315238576,
      "grad_norm": 12.709994849088284,
      "learning_rate": 3.5252212620435764e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.203125,
      "logps/chosen": -528.0,
      "logps/rejected": -888.0,
      "loss": 0.1548,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.625,
      "rewards/rejected": -7.125,
      "step": 11720
    },
    {
      "epoch": 0.8467479968237926,
      "grad_norm": 14.068320846423171,
      "learning_rate": 3.493035019366328e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -2.96875,
      "logps/chosen": -516.0,
      "logps/rejected": -948.0,
      "loss": 0.1636,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 4.3125,
      "rewards/rejected": -7.6875,
      "step": 11730
    },
    {
      "epoch": 0.8474698621237277,
      "grad_norm": 11.456118307713433,
      "learning_rate": 3.460985346505169e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.15625,
      "logps/chosen": -548.0,
      "logps/rejected": -912.0,
      "loss": 0.1656,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.21875,
      "step": 11740
    },
    {
      "epoch": 0.8481917274236628,
      "grad_norm": 10.199802773729761,
      "learning_rate": 3.4290724469764e-08,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.25,
      "logps/chosen": -520.0,
      "logps/rejected": -920.0,
      "loss": 0.1655,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.953125,
      "rewards/rejected": -7.40625,
      "step": 11750
    },
    {
      "epoch": 0.8489135927235978,
      "grad_norm": 14.683247398653602,
      "learning_rate": 3.397296523427806e-08,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.171875,
      "logps/chosen": -528.0,
      "logps/rejected": -896.0,
      "loss": 0.1671,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.5625,
      "rewards/margins": 3.71875,
      "rewards/rejected": -7.28125,
      "step": 11760
    },
    {
      "epoch": 0.8496354580235328,
      "grad_norm": 14.155838208168849,
      "learning_rate": 3.3656577776373706e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -3.015625,
      "logps/chosen": -520.0,
      "logps/rejected": -936.0,
      "loss": 0.1559,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.34375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.25,
      "step": 11770
    },
    {
      "epoch": 0.8503573233234678,
      "grad_norm": 17.75164556977513,
      "learning_rate": 3.334156410511993e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -3.03125,
      "logps/chosen": -536.0,
      "logps/rejected": -980.0,
      "loss": 0.1717,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.375,
      "rewards/margins": 4.5,
      "rewards/rejected": -7.875,
      "step": 11780
    },
    {
      "epoch": 0.8510791886234029,
      "grad_norm": 14.678253579685139,
      "learning_rate": 3.30279262208622e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.140625,
      "logps/chosen": -588.0,
      "logps/rejected": -924.0,
      "loss": 0.1634,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.859375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -7.4375,
      "step": 11790
    },
    {
      "epoch": 0.8518010539233379,
      "grad_norm": 14.258551284835313,
      "learning_rate": 3.271566611520965e-08,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.515625,
      "logps/chosen": -524.0,
      "logps/rejected": -880.0,
      "loss": 0.169,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.828125,
      "rewards/rejected": -7.1875,
      "step": 11800
    },
    {
      "epoch": 0.852522919223273,
      "grad_norm": 11.717492393989401,
      "learning_rate": 3.240478577102254e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.0,
      "logps/chosen": -596.0,
      "logps/rejected": -960.0,
      "loss": 0.165,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.890625,
      "rewards/margins": 3.59375,
      "rewards/rejected": -7.46875,
      "step": 11810
    },
    {
      "epoch": 0.853244784523208,
      "grad_norm": 12.700659978854983,
      "learning_rate": 3.2095287162399396e-08,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.125,
      "logps/chosen": -544.0,
      "logps/rejected": -884.0,
      "loss": 0.1598,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 3.734375,
      "rewards/rejected": -7.3125,
      "step": 11820
    },
    {
      "epoch": 0.853966649823143,
      "grad_norm": 16.44572741024403,
      "learning_rate": 3.178717225466504e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.65625,
      "logps/chosen": -544.0,
      "logps/rejected": -940.0,
      "loss": 0.1588,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.5625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.625,
      "step": 11830
    },
    {
      "epoch": 0.8546885151230781,
      "grad_norm": 11.314574948066843,
      "learning_rate": 3.1480443004357535e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.890625,
      "logps/chosen": -528.0,
      "logps/rejected": -932.0,
      "loss": 0.1519,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.59375,
      "step": 11840
    },
    {
      "epoch": 0.8554103804230131,
      "grad_norm": 7.521037494252695,
      "learning_rate": 3.117510135921589e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -3.0625,
      "logps/chosen": -524.0,
      "logps/rejected": -888.0,
      "loss": 0.1426,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.28125,
      "step": 11850
    },
    {
      "epoch": 0.8561322457229481,
      "grad_norm": 11.781422699654149,
      "learning_rate": 3.087114925816808e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.0,
      "logps/chosen": -556.0,
      "logps/rejected": -956.0,
      "loss": 0.1704,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.546875,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.75,
      "step": 11860
    },
    {
      "epoch": 0.8568541110228831,
      "grad_norm": 11.866120676904785,
      "learning_rate": 3.0568588631318115e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.078125,
      "logps/chosen": -528.0,
      "logps/rejected": -960.0,
      "loss": 0.1549,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.453125,
      "rewards/margins": 4.375,
      "rewards/rejected": -7.8125,
      "step": 11870
    },
    {
      "epoch": 0.8575759763228181,
      "grad_norm": 15.834345651025451,
      "learning_rate": 3.026742139993427e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.9375,
      "logps/chosen": -560.0,
      "logps/rejected": -952.0,
      "loss": 0.1679,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.65625,
      "rewards/margins": 3.953125,
      "rewards/rejected": -7.59375,
      "step": 11880
    },
    {
      "epoch": 0.8582978416227532,
      "grad_norm": 9.939794963672531,
      "learning_rate": 2.996764947643685e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.796875,
      "logps/chosen": -564.0,
      "logps/rejected": -968.0,
      "loss": 0.1569,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.75,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.8125,
      "step": 11890
    },
    {
      "epoch": 0.8590197069226883,
      "grad_norm": 11.668646930493974,
      "learning_rate": 2.9669274764385604e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.953125,
      "logps/chosen": -536.0,
      "logps/rejected": -876.0,
      "loss": 0.1636,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.59375,
      "rewards/rejected": -7.0,
      "step": 11900
    },
    {
      "epoch": 0.8597415722226233,
      "grad_norm": 11.528790065607312,
      "learning_rate": 2.9372299158468144e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.046875,
      "logps/chosen": -516.0,
      "logps/rejected": -936.0,
      "loss": 0.1537,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.203125,
      "rewards/margins": 4.34375,
      "rewards/rejected": -7.53125,
      "step": 11910
    },
    {
      "epoch": 0.8604634375225583,
      "grad_norm": 14.058244032238608,
      "learning_rate": 2.907672454448784e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.8125,
      "logps/chosen": -532.0,
      "logps/rejected": -924.0,
      "loss": 0.1614,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.4375,
      "step": 11920
    },
    {
      "epoch": 0.8611853028224933,
      "grad_norm": 21.926747312915236,
      "learning_rate": 2.8782552799351404e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.171875,
      "logps/chosen": -532.0,
      "logps/rejected": -932.0,
      "loss": 0.1649,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.53125,
      "step": 11930
    },
    {
      "epoch": 0.8619071681224284,
      "grad_norm": 10.244165289725355,
      "learning_rate": 2.8489785791057543e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.125,
      "logps/chosen": -532.0,
      "logps/rejected": -932.0,
      "loss": 0.172,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.421875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.5,
      "step": 11940
    },
    {
      "epoch": 0.8626290334223634,
      "grad_norm": 11.811377007541093,
      "learning_rate": 2.819842537868475e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -3.171875,
      "logps/chosen": -564.0,
      "logps/rejected": -932.0,
      "loss": 0.1632,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.609375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.53125,
      "step": 11950
    },
    {
      "epoch": 0.8633508987222984,
      "grad_norm": 14.844752900615154,
      "learning_rate": 2.7908473412379618e-08,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.34375,
      "logps/chosen": -536.0,
      "logps/rejected": -912.0,
      "loss": 0.1899,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.625,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.53125,
      "step": 11960
    },
    {
      "epoch": 0.8640727640222334,
      "grad_norm": 12.704565459846489,
      "learning_rate": 2.761993173334509e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.0625,
      "logps/chosen": -568.0,
      "logps/rejected": -908.0,
      "loss": 0.1743,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.53125,
      "rewards/margins": 3.75,
      "rewards/rejected": -7.28125,
      "step": 11970
    },
    {
      "epoch": 0.8647946293221684,
      "grad_norm": 14.281995105238705,
      "learning_rate": 2.733280217382869e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.390625,
      "logps/chosen": -540.0,
      "logps/rejected": -928.0,
      "loss": 0.1605,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.4375,
      "step": 11980
    },
    {
      "epoch": 0.8655164946221036,
      "grad_norm": 13.029401644129456,
      "learning_rate": 2.704708655711102e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.0625,
      "logps/chosen": -516.0,
      "logps/rejected": -916.0,
      "loss": 0.1621,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.34375,
      "rewards/margins": 3.953125,
      "rewards/rejected": -7.3125,
      "step": 11990
    },
    {
      "epoch": 0.8662383599220386,
      "grad_norm": 15.35638592137265,
      "learning_rate": 2.6762786697494016e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.21875,
      "logps/chosen": -568.0,
      "logps/rejected": -948.0,
      "loss": 0.1675,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.8125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.875,
      "step": 12000
    },
    {
      "epoch": 0.8669602252219736,
      "grad_norm": 14.235830002799881,
      "learning_rate": 2.6479904400289578e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.09375,
      "logps/chosen": -548.0,
      "logps/rejected": -916.0,
      "loss": 0.1737,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.671875,
      "rewards/margins": 3.625,
      "rewards/rejected": -7.3125,
      "step": 12010
    },
    {
      "epoch": 0.8676820905219086,
      "grad_norm": 12.66355087778341,
      "learning_rate": 2.6198441461808106e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.625,
      "logps/chosen": -556.0,
      "logps/rejected": -960.0,
      "loss": 0.152,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.671875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.75,
      "step": 12020
    },
    {
      "epoch": 0.8684039558218436,
      "grad_norm": 11.822656756190407,
      "learning_rate": 2.5918399669346808e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.171875,
      "logps/chosen": -520.0,
      "logps/rejected": -936.0,
      "loss": 0.1671,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.5625,
      "step": 12030
    },
    {
      "epoch": 0.8691258211217787,
      "grad_norm": 13.7141069432265,
      "learning_rate": 2.5639780801178844e-08,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.140625,
      "logps/chosen": -552.0,
      "logps/rejected": -928.0,
      "loss": 0.1932,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.625,
      "rewards/margins": 3.9375,
      "rewards/rejected": -7.5625,
      "step": 12040
    },
    {
      "epoch": 0.8698476864217137,
      "grad_norm": 13.530242760105123,
      "learning_rate": 2.53625866265417e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.21875,
      "logps/chosen": -572.0,
      "logps/rejected": -936.0,
      "loss": 0.1655,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.46875,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.6875,
      "step": 12050
    },
    {
      "epoch": 0.8705695517216487,
      "grad_norm": 10.728820961782203,
      "learning_rate": 2.508681890562575e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.96875,
      "logps/chosen": -540.0,
      "logps/rejected": -936.0,
      "loss": 0.1388,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.5625,
      "step": 12060
    },
    {
      "epoch": 0.8712914170215837,
      "grad_norm": 11.197819296477624,
      "learning_rate": 2.4812479389563794e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.953125,
      "logps/chosen": -540.0,
      "logps/rejected": -964.0,
      "loss": 0.1552,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.5,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.5,
      "step": 12070
    },
    {
      "epoch": 0.8720132823215188,
      "grad_norm": 12.76256482559689,
      "learning_rate": 2.4539569820419213e-08,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.140625,
      "logps/chosen": -544.0,
      "logps/rejected": -968.0,
      "loss": 0.1699,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.578125,
      "rewards/margins": 4.28125,
      "rewards/rejected": -7.84375,
      "step": 12080
    },
    {
      "epoch": 0.8727351476214539,
      "grad_norm": 11.93631200744782,
      "learning_rate": 2.4268091931175128e-08,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.25,
      "logps/chosen": -536.0,
      "logps/rejected": -920.0,
      "loss": 0.1466,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.375,
      "step": 12090
    },
    {
      "epoch": 0.8734570129213889,
      "grad_norm": 14.485828962256923,
      "learning_rate": 2.3998047445723728e-08,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.359375,
      "logps/chosen": -498.0,
      "logps/rejected": -892.0,
      "loss": 0.1657,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.234375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.28125,
      "step": 12100
    },
    {
      "epoch": 0.8741788782213239,
      "grad_norm": 9.526490381924093,
      "learning_rate": 2.3729438078854748e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.90625,
      "logps/chosen": -564.0,
      "logps/rejected": -956.0,
      "loss": 0.1681,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.765625,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.78125,
      "step": 12110
    },
    {
      "epoch": 0.8749007435212589,
      "grad_norm": 13.075507832926181,
      "learning_rate": 2.3462265536245085e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -3.203125,
      "logps/chosen": -556.0,
      "logps/rejected": -936.0,
      "loss": 0.1675,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.65625,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.53125,
      "step": 12120
    },
    {
      "epoch": 0.8756226088211939,
      "grad_norm": 10.79208139697678,
      "learning_rate": 2.3196531514447643e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.453125,
      "logps/chosen": -512.0,
      "logps/rejected": -876.0,
      "loss": 0.1532,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.875,
      "rewards/rejected": -7.25,
      "step": 12130
    },
    {
      "epoch": 0.876344474121129,
      "grad_norm": 13.901943968022751,
      "learning_rate": 2.293223770088079e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -3.109375,
      "logps/chosen": -552.0,
      "logps/rejected": -908.0,
      "loss": 0.1665,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.546875,
      "rewards/margins": 3.703125,
      "rewards/rejected": -7.25,
      "step": 12140
    },
    {
      "epoch": 0.877066339421064,
      "grad_norm": 18.039261717044518,
      "learning_rate": 2.2669385773817467e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.96875,
      "logps/chosen": -512.0,
      "logps/rejected": -964.0,
      "loss": 0.1477,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 4.53125,
      "rewards/rejected": -7.8125,
      "step": 12150
    },
    {
      "epoch": 0.877788204720999,
      "grad_norm": 14.767421261716349,
      "learning_rate": 2.2407977402374545e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.078125,
      "logps/chosen": -548.0,
      "logps/rejected": -948.0,
      "loss": 0.1637,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.578125,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.59375,
      "step": 12160
    },
    {
      "epoch": 0.8785100700209341,
      "grad_norm": 12.981100096028143,
      "learning_rate": 2.2148014246502395e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -3.09375,
      "logps/chosen": -524.0,
      "logps/rejected": -880.0,
      "loss": 0.1896,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.75,
      "rewards/rejected": -7.0625,
      "step": 12170
    },
    {
      "epoch": 0.8792319353208691,
      "grad_norm": 16.939012299120577,
      "learning_rate": 2.1889497956974146e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.3125,
      "logps/chosen": -540.0,
      "logps/rejected": -884.0,
      "loss": 0.1744,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.734375,
      "rewards/rejected": -7.125,
      "step": 12180
    },
    {
      "epoch": 0.8799538006208042,
      "grad_norm": 13.168484910562103,
      "learning_rate": 2.1632430175375332e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.078125,
      "logps/chosen": -568.0,
      "logps/rejected": -944.0,
      "loss": 0.1779,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.828125,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.6875,
      "step": 12190
    },
    {
      "epoch": 0.8806756659207392,
      "grad_norm": 10.756960323899968,
      "learning_rate": 2.1376812534093375e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.3125,
      "logps/chosen": -520.0,
      "logps/rejected": -912.0,
      "loss": 0.1612,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.265625,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.375,
      "step": 12200
    },
    {
      "epoch": 0.8813975312206742,
      "grad_norm": 18.9960028886757,
      "learning_rate": 2.112264665630728e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.15625,
      "logps/chosen": -502.0,
      "logps/rejected": -876.0,
      "loss": 0.1674,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.046875,
      "rewards/margins": 3.921875,
      "rewards/rejected": -6.96875,
      "step": 12210
    },
    {
      "epoch": 0.8821193965206092,
      "grad_norm": 14.79583572905297,
      "learning_rate": 2.0869934155977348e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.78125,
      "logps/chosen": -536.0,
      "logps/rejected": -960.0,
      "loss": 0.1687,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.53125,
      "rewards/margins": 4.25,
      "rewards/rejected": -7.78125,
      "step": 12220
    },
    {
      "epoch": 0.8828412618205442,
      "grad_norm": 11.964214159104781,
      "learning_rate": 2.0618676637834924e-08,
      "logits/chosen": -3.140625,
      "logits/rejected": -3.09375,
      "logps/chosen": -512.0,
      "logps/rejected": -944.0,
      "loss": 0.146,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 4.5,
      "rewards/rejected": -7.78125,
      "step": 12230
    },
    {
      "epoch": 0.8835631271204794,
      "grad_norm": 13.070396396075434,
      "learning_rate": 2.0368875697372028e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.90625,
      "logps/chosen": -508.0,
      "logps/rejected": -912.0,
      "loss": 0.1651,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.1875,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.25,
      "step": 12240
    },
    {
      "epoch": 0.8842849924204144,
      "grad_norm": 9.893605517838626,
      "learning_rate": 2.0120532920831436e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -3.0625,
      "logps/chosen": -552.0,
      "logps/rejected": -964.0,
      "loss": 0.1735,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.546875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.6875,
      "step": 12250
    },
    {
      "epoch": 0.8850068577203494,
      "grad_norm": 13.147528825432609,
      "learning_rate": 1.9873649885196742e-08,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.109375,
      "logps/chosen": -540.0,
      "logps/rejected": -904.0,
      "loss": 0.1679,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.609375,
      "rewards/margins": 3.640625,
      "rewards/rejected": -7.25,
      "step": 12260
    },
    {
      "epoch": 0.8857287230202844,
      "grad_norm": 12.600070100218279,
      "learning_rate": 1.9628228158181853e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.921875,
      "logps/chosen": -536.0,
      "logps/rejected": -884.0,
      "loss": 0.1684,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.671875,
      "rewards/rejected": -7.09375,
      "step": 12270
    },
    {
      "epoch": 0.8864505883202195,
      "grad_norm": 16.46044339271557,
      "learning_rate": 1.93842692982214e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.890625,
      "logps/chosen": -548.0,
      "logps/rejected": -908.0,
      "loss": 0.159,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.59375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -7.40625,
      "step": 12280
    },
    {
      "epoch": 0.8871724536201545,
      "grad_norm": 11.341426619789788,
      "learning_rate": 1.9141774854461e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -3.1875,
      "logps/chosen": -544.0,
      "logps/rejected": -932.0,
      "loss": 0.1532,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.5625,
      "rewards/margins": 3.953125,
      "rewards/rejected": -7.5,
      "step": 12290
    },
    {
      "epoch": 0.8878943189200895,
      "grad_norm": 11.256391427688461,
      "learning_rate": 1.890074636674685e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.96875,
      "logps/chosen": -536.0,
      "logps/rejected": -876.0,
      "loss": 0.1715,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.09375,
      "step": 12300
    },
    {
      "epoch": 0.8886161842200245,
      "grad_norm": 15.873594185576023,
      "learning_rate": 1.8661185365616478e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.28125,
      "logps/chosen": -520.0,
      "logps/rejected": -932.0,
      "loss": 0.1495,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.375,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.5,
      "step": 12310
    },
    {
      "epoch": 0.8893380495199595,
      "grad_norm": 12.48639143841317,
      "learning_rate": 1.842309337228884e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.28125,
      "logps/chosen": -572.0,
      "logps/rejected": -952.0,
      "loss": 0.1603,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.671875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.78125,
      "step": 12320
    },
    {
      "epoch": 0.8900599148198947,
      "grad_norm": 11.778640840964513,
      "learning_rate": 1.818647189865455e-08,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.390625,
      "logps/chosen": -508.0,
      "logps/rejected": -924.0,
      "loss": 0.174,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.234375,
      "rewards/margins": 4.3125,
      "rewards/rejected": -7.53125,
      "step": 12330
    },
    {
      "epoch": 0.8907817801198297,
      "grad_norm": 11.634395513317223,
      "learning_rate": 1.7951322447266493e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.859375,
      "logps/chosen": -520.0,
      "logps/rejected": -960.0,
      "loss": 0.1663,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.34375,
      "rewards/margins": 4.28125,
      "rewards/rejected": -7.625,
      "step": 12340
    },
    {
      "epoch": 0.8915036454197647,
      "grad_norm": 17.060036430957545,
      "learning_rate": 1.7717646511330063e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.6875,
      "logps/chosen": -528.0,
      "logps/rejected": -956.0,
      "loss": 0.1736,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 4.40625,
      "rewards/rejected": -7.6875,
      "step": 12350
    },
    {
      "epoch": 0.8922255107196997,
      "grad_norm": 14.254858456690274,
      "learning_rate": 1.748544557469392e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.921875,
      "logps/chosen": -548.0,
      "logps/rejected": -952.0,
      "loss": 0.1559,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.625,
      "rewards/margins": 3.9375,
      "rewards/rejected": -7.5625,
      "step": 12360
    },
    {
      "epoch": 0.8929473760196347,
      "grad_norm": 10.596856240818077,
      "learning_rate": 1.7254721111840287e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.125,
      "logps/chosen": -520.0,
      "logps/rejected": -940.0,
      "loss": 0.1588,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.375,
      "rewards/margins": 4.34375,
      "rewards/rejected": -7.71875,
      "step": 12370
    },
    {
      "epoch": 0.8936692413195698,
      "grad_norm": 11.342440268615379,
      "learning_rate": 1.7025474587875872e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.375,
      "logps/chosen": -524.0,
      "logps/rejected": -884.0,
      "loss": 0.1553,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.21875,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.125,
      "step": 12380
    },
    {
      "epoch": 0.8943911066195048,
      "grad_norm": 11.627724625934398,
      "learning_rate": 1.6797707458522382e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.296875,
      "logps/chosen": -528.0,
      "logps/rejected": -920.0,
      "loss": 0.1464,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.421875,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.53125,
      "step": 12390
    },
    {
      "epoch": 0.8951129719194398,
      "grad_norm": 13.435638879923305,
      "learning_rate": 1.6571421170107236e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.9375,
      "logps/chosen": -548.0,
      "logps/rejected": -976.0,
      "loss": 0.1639,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.53125,
      "rewards/margins": 4.375,
      "rewards/rejected": -7.9375,
      "step": 12400
    },
    {
      "epoch": 0.8958348372193748,
      "grad_norm": 13.063971345625463,
      "learning_rate": 1.6346617159554628e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -3.21875,
      "logps/chosen": -532.0,
      "logps/rejected": -872.0,
      "loss": 0.1635,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -7.03125,
      "step": 12410
    },
    {
      "epoch": 0.8965567025193099,
      "grad_norm": 13.147315612876698,
      "learning_rate": 1.6123296854376217e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.96875,
      "logps/chosen": -536.0,
      "logps/rejected": -920.0,
      "loss": 0.1939,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.390625,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.375,
      "step": 12420
    },
    {
      "epoch": 0.897278567819245,
      "grad_norm": 12.40105057640074,
      "learning_rate": 1.5901461672661904e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -3.078125,
      "logps/chosen": -544.0,
      "logps/rejected": -920.0,
      "loss": 0.1862,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.46875,
      "step": 12430
    },
    {
      "epoch": 0.89800043311918,
      "grad_norm": 10.454569456941549,
      "learning_rate": 1.5681113023071318e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.65625,
      "logps/chosen": -548.0,
      "logps/rejected": -952.0,
      "loss": 0.1665,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.671875,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.65625,
      "step": 12440
    },
    {
      "epoch": 0.898722298419115,
      "grad_norm": 8.886038829556064,
      "learning_rate": 1.5462252304824368e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -3.109375,
      "logps/chosen": -564.0,
      "logps/rejected": -956.0,
      "loss": 0.1473,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.53125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.59375,
      "step": 12450
    },
    {
      "epoch": 0.89944416371905,
      "grad_norm": 12.682063783879679,
      "learning_rate": 1.524488090769252e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.1875,
      "logps/chosen": -532.0,
      "logps/rejected": -888.0,
      "loss": 0.1714,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.75,
      "rewards/rejected": -7.15625,
      "step": 12460
    },
    {
      "epoch": 0.900166029018985,
      "grad_norm": 12.148030506805691,
      "learning_rate": 1.5029000211990216e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.125,
      "logps/chosen": -552.0,
      "logps/rejected": -908.0,
      "loss": 0.1689,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.796875,
      "rewards/rejected": -7.3125,
      "step": 12470
    },
    {
      "epoch": 0.9008878943189201,
      "grad_norm": 13.43580298909997,
      "learning_rate": 1.4814611588565701e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.46875,
      "logps/chosen": -528.0,
      "logps/rejected": -892.0,
      "loss": 0.1554,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.75,
      "rewards/rejected": -7.21875,
      "step": 12480
    },
    {
      "epoch": 0.9016097596188551,
      "grad_norm": 15.827939601512334,
      "learning_rate": 1.4601716398792595e-08,
      "logits/chosen": -3.140625,
      "logits/rejected": -3.34375,
      "logps/chosen": -556.0,
      "logps/rejected": -924.0,
      "loss": 0.1834,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.40625,
      "step": 12490
    },
    {
      "epoch": 0.9023316249187902,
      "grad_norm": 13.792794727503592,
      "learning_rate": 1.4390315994561253e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.140625,
      "logps/chosen": -532.0,
      "logps/rejected": -900.0,
      "loss": 0.1661,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 3.75,
      "rewards/rejected": -7.25,
      "step": 12500
    },
    {
      "epoch": 0.9030534902187252,
      "grad_norm": 13.932246263769382,
      "learning_rate": 1.4180411718269974e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -3.265625,
      "logps/chosen": -532.0,
      "logps/rejected": -912.0,
      "loss": 0.156,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.828125,
      "rewards/rejected": -7.34375,
      "step": 12510
    },
    {
      "epoch": 0.9037753555186602,
      "grad_norm": 12.116649837278286,
      "learning_rate": 1.3972004902816609e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.078125,
      "logps/chosen": -520.0,
      "logps/rejected": -908.0,
      "loss": 0.1714,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.9375,
      "rewards/rejected": -7.34375,
      "step": 12520
    },
    {
      "epoch": 0.9044972208185953,
      "grad_norm": 15.343223927341187,
      "learning_rate": 1.3765096871590248e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -3.046875,
      "logps/chosen": -528.0,
      "logps/rejected": -900.0,
      "loss": 0.1579,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -7.125,
      "step": 12530
    },
    {
      "epoch": 0.9052190861185303,
      "grad_norm": 13.971378657897587,
      "learning_rate": 1.355968893846246e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.265625,
      "logps/chosen": -532.0,
      "logps/rejected": -884.0,
      "loss": 0.1731,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.375,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.1875,
      "step": 12540
    },
    {
      "epoch": 0.9059409514184653,
      "grad_norm": 13.891389076547908,
      "learning_rate": 1.3355782407779292e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.765625,
      "logps/chosen": -516.0,
      "logps/rejected": -924.0,
      "loss": 0.1695,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.3125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.40625,
      "step": 12550
    },
    {
      "epoch": 0.9066628167184003,
      "grad_norm": 12.446163633902653,
      "learning_rate": 1.3153378574352753e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.96875,
      "logps/chosen": -544.0,
      "logps/rejected": -872.0,
      "loss": 0.1615,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.53125,
      "rewards/margins": 3.421875,
      "rewards/rejected": -6.9375,
      "step": 12560
    },
    {
      "epoch": 0.9073846820183353,
      "grad_norm": 15.400904243402238,
      "learning_rate": 1.2952478723452759e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.09375,
      "logps/chosen": -532.0,
      "logps/rejected": -920.0,
      "loss": 0.1614,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.828125,
      "rewards/rejected": -7.21875,
      "step": 12570
    },
    {
      "epoch": 0.9081065473182705,
      "grad_norm": 12.907741138304852,
      "learning_rate": 1.2753084130798841e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.984375,
      "logps/chosen": -516.0,
      "logps/rejected": -892.0,
      "loss": 0.1605,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.21875,
      "step": 12580
    },
    {
      "epoch": 0.9088284126182055,
      "grad_norm": 10.842978828970327,
      "learning_rate": 1.2555196062552121e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.765625,
      "logps/chosen": -528.0,
      "logps/rejected": -916.0,
      "loss": 0.1531,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.203125,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.34375,
      "step": 12590
    },
    {
      "epoch": 0.9095502779181405,
      "grad_norm": 13.355868319850176,
      "learning_rate": 1.2358815775307257e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.765625,
      "logps/chosen": -576.0,
      "logps/rejected": -984.0,
      "loss": 0.158,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.59375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.75,
      "step": 12600
    },
    {
      "epoch": 0.9102721432180755,
      "grad_norm": 12.72464710877496,
      "learning_rate": 1.2163944516084434e-08,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.0,
      "logps/chosen": -528.0,
      "logps/rejected": -944.0,
      "loss": 0.173,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.421875,
      "rewards/margins": 4.28125,
      "rewards/rejected": -7.71875,
      "step": 12610
    },
    {
      "epoch": 0.9109940085180105,
      "grad_norm": 11.428389687698242,
      "learning_rate": 1.197058352232147e-08,
      "logits/chosen": -3.109375,
      "logits/rejected": -2.9375,
      "logps/chosen": -506.0,
      "logps/rejected": -964.0,
      "loss": 0.1578,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.328125,
      "rewards/margins": 4.28125,
      "rewards/rejected": -7.59375,
      "step": 12620
    },
    {
      "epoch": 0.9117158738179456,
      "grad_norm": 12.134022161155206,
      "learning_rate": 1.1778734021866022e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.171875,
      "logps/chosen": -532.0,
      "logps/rejected": -880.0,
      "loss": 0.1728,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.484375,
      "rewards/margins": 3.765625,
      "rewards/rejected": -7.25,
      "step": 12630
    },
    {
      "epoch": 0.9124377391178806,
      "grad_norm": 11.804093319675003,
      "learning_rate": 1.1588397232967561e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.953125,
      "logps/chosen": -492.0,
      "logps/rejected": -908.0,
      "loss": 0.1637,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.0625,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.1875,
      "step": 12640
    },
    {
      "epoch": 0.9131596044178156,
      "grad_norm": 9.350898325989752,
      "learning_rate": 1.1399574364269997e-08,
      "logits/chosen": -3.078125,
      "logits/rejected": -2.9375,
      "logps/chosen": -532.0,
      "logps/rejected": -932.0,
      "loss": 0.1654,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.359375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.4375,
      "step": 12650
    },
    {
      "epoch": 0.9138814697177506,
      "grad_norm": 13.360066458808197,
      "learning_rate": 1.1212266614803706e-08,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.984375,
      "logps/chosen": -536.0,
      "logps/rejected": -920.0,
      "loss": 0.1618,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.875,
      "rewards/rejected": -7.28125,
      "step": 12660
    },
    {
      "epoch": 0.9146033350176856,
      "grad_norm": 12.830360003331549,
      "learning_rate": 1.1026475173977978e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.984375,
      "logps/chosen": -540.0,
      "logps/rejected": -972.0,
      "loss": 0.174,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.421875,
      "rewards/margins": 4.4375,
      "rewards/rejected": -7.84375,
      "step": 12670
    },
    {
      "epoch": 0.9153252003176208,
      "grad_norm": 17.014496847789825,
      "learning_rate": 1.0842201221573532e-08,
      "logits/chosen": -3.125,
      "logits/rejected": -3.21875,
      "logps/chosen": -548.0,
      "logps/rejected": -900.0,
      "loss": 0.1561,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.609375,
      "rewards/margins": 3.65625,
      "rewards/rejected": -7.25,
      "step": 12680
    },
    {
      "epoch": 0.9160470656175558,
      "grad_norm": 9.53092850095292,
      "learning_rate": 1.0659445927735127e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -3.1875,
      "logps/chosen": -528.0,
      "logps/rejected": -928.0,
      "loss": 0.1486,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.421875,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.46875,
      "step": 12690
    },
    {
      "epoch": 0.9167689309174908,
      "grad_norm": 10.850767005423643,
      "learning_rate": 1.0478210452963737e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.890625,
      "logps/chosen": -532.0,
      "logps/rejected": -920.0,
      "loss": 0.1661,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.4375,
      "step": 12700
    },
    {
      "epoch": 0.9174907962174258,
      "grad_norm": 13.23351230811876,
      "learning_rate": 1.0298495948109665e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.046875,
      "logps/chosen": -532.0,
      "logps/rejected": -888.0,
      "loss": 0.1744,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.65625,
      "rewards/rejected": -7.0625,
      "step": 12710
    },
    {
      "epoch": 0.9182126615173608,
      "grad_norm": 11.642282329746104,
      "learning_rate": 1.0120303554364912e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.265625,
      "logps/chosen": -524.0,
      "logps/rejected": -908.0,
      "loss": 0.1565,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.5,
      "step": 12720
    },
    {
      "epoch": 0.9189345268172959,
      "grad_norm": 19.868717242445925,
      "learning_rate": 9.943634403256046e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.09375,
      "logps/chosen": -512.0,
      "logps/rejected": -972.0,
      "loss": 0.1633,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.34375,
      "rewards/margins": 4.5,
      "rewards/rejected": -7.84375,
      "step": 12730
    },
    {
      "epoch": 0.9196563921172309,
      "grad_norm": 12.337420160773002,
      "learning_rate": 9.768489616637038e-09,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.25,
      "logps/chosen": -540.0,
      "logps/rejected": -904.0,
      "loss": 0.1794,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.640625,
      "rewards/margins": 3.6875,
      "rewards/rejected": -7.3125,
      "step": 12740
    },
    {
      "epoch": 0.920378257417166,
      "grad_norm": 11.787244059796334,
      "learning_rate": 9.594870306682045e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.296875,
      "logps/chosen": -520.0,
      "logps/rejected": -892.0,
      "loss": 0.1701,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.875,
      "rewards/rejected": -7.1875,
      "step": 12750
    },
    {
      "epoch": 0.921100122717101,
      "grad_norm": 12.447020409484137,
      "learning_rate": 9.42277757587842e-09,
      "logits/chosen": -3.0,
      "logits/rejected": -3.28125,
      "logps/chosen": -540.0,
      "logps/rejected": -936.0,
      "loss": 0.1552,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.609375,
      "rewards/margins": 3.984375,
      "rewards/rejected": -7.59375,
      "step": 12760
    },
    {
      "epoch": 0.9218219880170361,
      "grad_norm": 11.084885647171987,
      "learning_rate": 9.25221251701977e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.046875,
      "logps/chosen": -524.0,
      "logps/rejected": -944.0,
      "loss": 0.1624,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.53125,
      "step": 12770
    },
    {
      "epoch": 0.9225438533169711,
      "grad_norm": 13.954494700514417,
      "learning_rate": 9.083176213198874e-09,
      "logits/chosen": -2.8125,
      "logits/rejected": -3.140625,
      "logps/chosen": -532.0,
      "logps/rejected": -924.0,
      "loss": 0.1758,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.4375,
      "step": 12780
    },
    {
      "epoch": 0.9232657186169061,
      "grad_norm": 16.349505947969504,
      "learning_rate": 8.915669737800835e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.90625,
      "logps/chosen": -544.0,
      "logps/rejected": -916.0,
      "loss": 0.1653,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.5625,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.375,
      "step": 12790
    },
    {
      "epoch": 0.9239875839168411,
      "grad_norm": 8.694243190011187,
      "learning_rate": 8.74969415449639e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.03125,
      "logps/chosen": -528.0,
      "logps/rejected": -924.0,
      "loss": 0.1469,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.46875,
      "step": 12800
    },
    {
      "epoch": 0.9247094492167761,
      "grad_norm": 10.538304226881124,
      "learning_rate": 8.585250517235048e-09,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.25,
      "logps/chosen": -576.0,
      "logps/rejected": -960.0,
      "loss": 0.1535,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.765625,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.75,
      "step": 12810
    },
    {
      "epoch": 0.9254313145167112,
      "grad_norm": 19.448935621081525,
      "learning_rate": 8.422339870238409e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.953125,
      "logps/chosen": -548.0,
      "logps/rejected": -936.0,
      "loss": 0.1605,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.5,
      "step": 12820
    },
    {
      "epoch": 0.9261531798166462,
      "grad_norm": 12.959542187422079,
      "learning_rate": 8.26096324799347e-09,
      "logits/chosen": -3.0,
      "logits/rejected": -3.0625,
      "logps/chosen": -528.0,
      "logps/rejected": -916.0,
      "loss": 0.1787,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.421875,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.40625,
      "step": 12830
    },
    {
      "epoch": 0.9268750451165813,
      "grad_norm": 12.089971796300393,
      "learning_rate": 8.101121675246293e-09,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.09375,
      "logps/chosen": -508.0,
      "logps/rejected": -924.0,
      "loss": 0.1454,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.171875,
      "rewards/margins": 4.34375,
      "rewards/rejected": -7.5,
      "step": 12840
    },
    {
      "epoch": 0.9275969104165163,
      "grad_norm": 14.543057225666367,
      "learning_rate": 7.942816166995187e-09,
      "logits/chosen": -2.828125,
      "logits/rejected": -3.296875,
      "logps/chosen": -536.0,
      "logps/rejected": -892.0,
      "loss": 0.1674,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.421875,
      "rewards/margins": 3.828125,
      "rewards/rejected": -7.25,
      "step": 12850
    },
    {
      "epoch": 0.9283187757164513,
      "grad_norm": 14.065495839285813,
      "learning_rate": 7.786047728484429e-09,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.046875,
      "logps/chosen": -536.0,
      "logps/rejected": -952.0,
      "loss": 0.1618,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.3125,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.4375,
      "step": 12860
    },
    {
      "epoch": 0.9290406410163864,
      "grad_norm": 12.393878730121433,
      "learning_rate": 7.630817355197994e-09,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.078125,
      "logps/chosen": -520.0,
      "logps/rejected": -864.0,
      "loss": 0.1565,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -7.0625,
      "step": 12870
    },
    {
      "epoch": 0.9297625063163214,
      "grad_norm": 12.38947218968365,
      "learning_rate": 7.477126032852888e-09,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.15625,
      "logps/chosen": -548.0,
      "logps/rejected": -908.0,
      "loss": 0.1605,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.609375,
      "rewards/margins": 3.734375,
      "rewards/rejected": -7.34375,
      "step": 12880
    },
    {
      "epoch": 0.9304843716162564,
      "grad_norm": 9.837346131069198,
      "learning_rate": 7.324974737393241e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.28125,
      "logps/chosen": -556.0,
      "logps/rejected": -916.0,
      "loss": 0.1675,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.53125,
      "step": 12890
    },
    {
      "epoch": 0.9312062369161914,
      "grad_norm": 10.373617029275584,
      "learning_rate": 7.174364434984009e-09,
      "logits/chosen": -2.875,
      "logits/rejected": -2.859375,
      "logps/chosen": -544.0,
      "logps/rejected": -900.0,
      "loss": 0.1735,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.21875,
      "step": 12900
    },
    {
      "epoch": 0.9319281022161264,
      "grad_norm": 14.5525441770712,
      "learning_rate": 7.025296082004667e-09,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.046875,
      "logps/chosen": -520.0,
      "logps/rejected": -892.0,
      "loss": 0.1861,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.421875,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.25,
      "step": 12910
    },
    {
      "epoch": 0.9326499675160616,
      "grad_norm": 14.49213901340877,
      "learning_rate": 6.8777706250433274e-09,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.953125,
      "logps/chosen": -544.0,
      "logps/rejected": -956.0,
      "loss": 0.1676,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.359375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.46875,
      "step": 12920
    },
    {
      "epoch": 0.9333718328159966,
      "grad_norm": 13.60957626683181,
      "learning_rate": 6.731789000890775e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.03125,
      "logps/chosen": -516.0,
      "logps/rejected": -892.0,
      "loss": 0.1554,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.921875,
      "rewards/rejected": -7.3125,
      "step": 12930
    },
    {
      "epoch": 0.9340936981159316,
      "grad_norm": 13.747648724735505,
      "learning_rate": 6.587352136534219e-09,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.859375,
      "logps/chosen": -516.0,
      "logps/rejected": -908.0,
      "loss": 0.1719,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.1875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.34375,
      "step": 12940
    },
    {
      "epoch": 0.9348155634158666,
      "grad_norm": 12.979191247301829,
      "learning_rate": 6.444460949151714e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.359375,
      "logps/chosen": -508.0,
      "logps/rejected": -896.0,
      "loss": 0.1558,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.21875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.28125,
      "step": 12950
    },
    {
      "epoch": 0.9355374287158016,
      "grad_norm": 15.091253466859595,
      "learning_rate": 6.303116346106224e-09,
      "logits/chosen": -3.0,
      "logits/rejected": -3.109375,
      "logps/chosen": -556.0,
      "logps/rejected": -928.0,
      "loss": 0.1639,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.703125,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.53125,
      "step": 12960
    },
    {
      "epoch": 0.9362592940157367,
      "grad_norm": 11.046342773750485,
      "learning_rate": 6.163319224939872e-09,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.25,
      "logps/chosen": -528.0,
      "logps/rejected": -920.0,
      "loss": 0.1503,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.421875,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.4375,
      "step": 12970
    },
    {
      "epoch": 0.9369811593156717,
      "grad_norm": 16.19720873702694,
      "learning_rate": 6.025070473368144e-09,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.09375,
      "logps/chosen": -540.0,
      "logps/rejected": -904.0,
      "loss": 0.1578,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.546875,
      "rewards/margins": 3.765625,
      "rewards/rejected": -7.3125,
      "step": 12980
    },
    {
      "epoch": 0.9377030246156067,
      "grad_norm": 10.791941229188499,
      "learning_rate": 5.888370969274442e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.15625,
      "logps/chosen": -540.0,
      "logps/rejected": -888.0,
      "loss": 0.1624,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.796875,
      "rewards/rejected": -7.28125,
      "step": 12990
    },
    {
      "epoch": 0.9384248899155417,
      "grad_norm": 13.62350253441902,
      "learning_rate": 5.7532215807043486e-09,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.0625,
      "logps/chosen": -512.0,
      "logps/rejected": -888.0,
      "loss": 0.1694,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.296875,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.28125,
      "step": 13000
    },
    {
      "epoch": 0.9391467552154767,
      "grad_norm": 10.269840583777631,
      "learning_rate": 5.6196231658601764e-09,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.234375,
      "logps/chosen": -564.0,
      "logps/rejected": -952.0,
      "loss": 0.1706,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.734375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -7.59375,
      "step": 13010
    },
    {
      "epoch": 0.9398686205154119,
      "grad_norm": 12.669457137843622,
      "learning_rate": 5.48757657309551e-09,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.09375,
      "logps/chosen": -520.0,
      "logps/rejected": -916.0,
      "loss": 0.1638,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.28125,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.4375,
      "step": 13020
    },
    {
      "epoch": 0.9405904858153469,
      "grad_norm": 8.64415463036948,
      "learning_rate": 5.357082640909866e-09,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.015625,
      "logps/chosen": -520.0,
      "logps/rejected": -924.0,
      "loss": 0.148,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.390625,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.40625,
      "step": 13030
    },
    {
      "epoch": 0.9413123511152819,
      "grad_norm": 14.216241467139987,
      "learning_rate": 5.22814219794318e-09,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.9375,
      "logps/chosen": -548.0,
      "logps/rejected": -972.0,
      "loss": 0.1527,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.53125,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.75,
      "step": 13040
    },
    {
      "epoch": 0.9420342164152169,
      "grad_norm": 14.133225804762551,
      "learning_rate": 5.10075606297089e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.953125,
      "logps/chosen": -536.0,
      "logps/rejected": -916.0,
      "loss": 0.152,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.390625,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.5,
      "step": 13050
    },
    {
      "epoch": 0.9427560817151519,
      "grad_norm": 13.69100932781106,
      "learning_rate": 4.974925044898437e-09,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.125,
      "logps/chosen": -556.0,
      "logps/rejected": -916.0,
      "loss": 0.1819,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.578125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -7.28125,
      "step": 13060
    },
    {
      "epoch": 0.943477947015087,
      "grad_norm": 13.072310335738745,
      "learning_rate": 4.850649942756135e-09,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.125,
      "logps/chosen": -552.0,
      "logps/rejected": -884.0,
      "loss": 0.1454,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.640625,
      "rewards/rejected": -7.03125,
      "step": 13070
    },
    {
      "epoch": 0.944199812315022,
      "grad_norm": 11.459551539661485,
      "learning_rate": 4.727931545694397e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.0625,
      "logps/chosen": -524.0,
      "logps/rejected": -912.0,
      "loss": 0.1545,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.375,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.375,
      "step": 13080
    },
    {
      "epoch": 0.944921677614957,
      "grad_norm": 12.75099880482577,
      "learning_rate": 4.606770632978374e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.0,
      "logps/chosen": -532.0,
      "logps/rejected": -900.0,
      "loss": 0.1708,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.828125,
      "rewards/rejected": -7.28125,
      "step": 13090
    },
    {
      "epoch": 0.945643542914892,
      "grad_norm": 14.36464886873897,
      "learning_rate": 4.4871679739831304e-09,
      "logits/chosen": -2.875,
      "logits/rejected": -2.859375,
      "logps/chosen": -560.0,
      "logps/rejected": -968.0,
      "loss": 0.1502,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.546875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.65625,
      "step": 13100
    },
    {
      "epoch": 0.9463654082148271,
      "grad_norm": 19.04078252595615,
      "learning_rate": 4.369124328188839e-09,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.40625,
      "logps/chosen": -528.0,
      "logps/rejected": -900.0,
      "loss": 0.1718,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.34375,
      "step": 13110
    },
    {
      "epoch": 0.9470872735147622,
      "grad_norm": 11.02132935385711,
      "learning_rate": 4.252640445175898e-09,
      "logits/chosen": -2.875,
      "logits/rejected": -3.125,
      "logps/chosen": -556.0,
      "logps/rejected": -948.0,
      "loss": 0.15,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.625,
      "step": 13120
    },
    {
      "epoch": 0.9478091388146972,
      "grad_norm": 15.270937349025917,
      "learning_rate": 4.1377170646201555e-09,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.09375,
      "logps/chosen": -536.0,
      "logps/rejected": -936.0,
      "loss": 0.1597,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.4375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.625,
      "step": 13130
    },
    {
      "epoch": 0.9485310041146322,
      "grad_norm": 16.572758424888704,
      "learning_rate": 4.024354916288192e-09,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.078125,
      "logps/chosen": -532.0,
      "logps/rejected": -896.0,
      "loss": 0.1815,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.765625,
      "rewards/rejected": -7.1875,
      "step": 13140
    },
    {
      "epoch": 0.9492528694145672,
      "grad_norm": 12.803599831095697,
      "learning_rate": 3.912554720032796e-09,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.109375,
      "logps/chosen": -548.0,
      "logps/rejected": -908.0,
      "loss": 0.1629,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.53125,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.40625,
      "step": 13150
    },
    {
      "epoch": 0.9499747347145022,
      "grad_norm": 9.810637140222564,
      "learning_rate": 3.8023171857882456e-09,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.296875,
      "logps/chosen": -532.0,
      "logps/rejected": -896.0,
      "loss": 0.154,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.828125,
      "rewards/rejected": -7.28125,
      "step": 13160
    },
    {
      "epoch": 0.9506966000144373,
      "grad_norm": 7.39709337126921,
      "learning_rate": 3.693643013565867e-09,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.140625,
      "logps/chosen": -528.0,
      "logps/rejected": -908.0,
      "loss": 0.1579,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.328125,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.46875,
      "step": 13170
    },
    {
      "epoch": 0.9514184653143724,
      "grad_norm": 9.364697987891455,
      "learning_rate": 3.586532893449706e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.078125,
      "logps/chosen": -504.0,
      "logps/rejected": -920.0,
      "loss": 0.1807,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.21875,
      "rewards/margins": 4.25,
      "rewards/rejected": -7.46875,
      "step": 13180
    },
    {
      "epoch": 0.9521403306143074,
      "grad_norm": 13.27286911353558,
      "learning_rate": 3.4809875055918923e-09,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.296875,
      "logps/chosen": -524.0,
      "logps/rejected": -932.0,
      "loss": 0.1481,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.421875,
      "rewards/margins": 4.28125,
      "rewards/rejected": -7.6875,
      "step": 13190
    },
    {
      "epoch": 0.9528621959142424,
      "grad_norm": 15.044485128017348,
      "learning_rate": 3.3770075202085304e-09,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.125,
      "logps/chosen": -544.0,
      "logps/rejected": -956.0,
      "loss": 0.1652,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.484375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.6875,
      "step": 13200
    },
    {
      "epoch": 0.9535840612141774,
      "grad_norm": 9.817595834372558,
      "learning_rate": 3.274593597575398e-09,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.296875,
      "logps/chosen": -532.0,
      "logps/rejected": -908.0,
      "loss": 0.1754,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.78125,
      "rewards/rejected": -7.25,
      "step": 13210
    },
    {
      "epoch": 0.9543059265141125,
      "grad_norm": 8.571626121486629,
      "learning_rate": 3.173746388023729e-09,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.796875,
      "logps/chosen": -552.0,
      "logps/rejected": -1008.0,
      "loss": 0.1605,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.515625,
      "rewards/margins": 4.6875,
      "rewards/rejected": -8.1875,
      "step": 13220
    },
    {
      "epoch": 0.9550277918140475,
      "grad_norm": 13.3963682522003,
      "learning_rate": 3.074466531935993e-09,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.171875,
      "logps/chosen": -548.0,
      "logps/rejected": -928.0,
      "loss": 0.172,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.640625,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.59375,
      "step": 13230
    },
    {
      "epoch": 0.9557496571139825,
      "grad_norm": 13.57616376629907,
      "learning_rate": 2.976754659742037e-09,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.984375,
      "logps/chosen": -552.0,
      "logps/rejected": -916.0,
      "loss": 0.1554,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.421875,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.3125,
      "step": 13240
    },
    {
      "epoch": 0.9564715224139175,
      "grad_norm": 9.591701747132877,
      "learning_rate": 2.88061139191495e-09,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.140625,
      "logps/chosen": -528.0,
      "logps/rejected": -924.0,
      "loss": 0.1405,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.28125,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.4375,
      "step": 13250
    },
    {
      "epoch": 0.9571933877138527,
      "grad_norm": 18.23557716114033,
      "learning_rate": 2.7860373389670398e-09,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.15625,
      "logps/chosen": -532.0,
      "logps/rejected": -908.0,
      "loss": 0.1692,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.9375,
      "rewards/rejected": -7.375,
      "step": 13260
    },
    {
      "epoch": 0.9579152530137877,
      "grad_norm": 17.77505727965563,
      "learning_rate": 2.693033101446196e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.15625,
      "logps/chosen": -524.0,
      "logps/rejected": -928.0,
      "loss": 0.1801,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.328125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.375,
      "step": 13270
    },
    {
      "epoch": 0.9586371183137227,
      "grad_norm": 13.248523363265328,
      "learning_rate": 2.6015992699318366e-09,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.890625,
      "logps/chosen": -524.0,
      "logps/rejected": -884.0,
      "loss": 0.1425,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.34375,
      "rewards/margins": 3.875,
      "rewards/rejected": -7.21875,
      "step": 13280
    },
    {
      "epoch": 0.9593589836136577,
      "grad_norm": 12.369291329134638,
      "learning_rate": 2.5117364250312744e-09,
      "logits/chosen": -3.0,
      "logits/rejected": -3.015625,
      "logps/chosen": -532.0,
      "logps/rejected": -928.0,
      "loss": 0.1613,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.59375,
      "step": 13290
    },
    {
      "epoch": 0.9600808489135927,
      "grad_norm": 17.881751335358693,
      "learning_rate": 2.4234451373761068e-09,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.1875,
      "logps/chosen": -548.0,
      "logps/rejected": -956.0,
      "loss": 0.1752,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.59375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.78125,
      "step": 13300
    },
    {
      "epoch": 0.9608027142135278,
      "grad_norm": 11.020548865038906,
      "learning_rate": 2.336725967618358e-09,
      "logits/chosen": -2.875,
      "logits/rejected": -3.15625,
      "logps/chosen": -524.0,
      "logps/rejected": -892.0,
      "loss": 0.1733,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -7.09375,
      "step": 13310
    },
    {
      "epoch": 0.9615245795134628,
      "grad_norm": 14.274700146943823,
      "learning_rate": 2.2515794664271502e-09,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.0,
      "logps/chosen": -520.0,
      "logps/rejected": -940.0,
      "loss": 0.1507,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 4.28125,
      "rewards/rejected": -7.65625,
      "step": 13320
    },
    {
      "epoch": 0.9622464448133978,
      "grad_norm": 13.113399607684638,
      "learning_rate": 2.168006174485121e-09,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.171875,
      "logps/chosen": -532.0,
      "logps/rejected": -920.0,
      "loss": 0.1748,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.5,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.53125,
      "step": 13330
    },
    {
      "epoch": 0.9629683101133328,
      "grad_norm": 13.37394107865432,
      "learning_rate": 2.0860066224848983e-09,
      "logits/chosen": -2.875,
      "logits/rejected": -3.078125,
      "logps/chosen": -540.0,
      "logps/rejected": -912.0,
      "loss": 0.1525,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.71875,
      "rewards/rejected": -7.15625,
      "step": 13340
    },
    {
      "epoch": 0.9636901754132678,
      "grad_norm": 13.537344593854824,
      "learning_rate": 2.0055813311259383e-09,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.15625,
      "logps/chosen": -532.0,
      "logps/rejected": -924.0,
      "loss": 0.1497,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.953125,
      "rewards/rejected": -7.4375,
      "step": 13350
    },
    {
      "epoch": 0.964412040713203,
      "grad_norm": 10.949937440471876,
      "learning_rate": 1.926730811111027e-09,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.953125,
      "logps/chosen": -536.0,
      "logps/rejected": -888.0,
      "loss": 0.1608,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.515625,
      "rewards/margins": 3.5,
      "rewards/rejected": -7.0,
      "step": 13360
    },
    {
      "epoch": 0.965133906013138,
      "grad_norm": 9.85385788788045,
      "learning_rate": 1.849455563143143e-09,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.4375,
      "logps/chosen": -552.0,
      "logps/rejected": -936.0,
      "loss": 0.1726,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.59375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -7.625,
      "step": 13370
    },
    {
      "epoch": 0.965855771313073,
      "grad_norm": 12.876846373426144,
      "learning_rate": 1.773756077922156e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.046875,
      "logps/chosen": -556.0,
      "logps/rejected": -896.0,
      "loss": 0.1596,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.6875,
      "rewards/rejected": -7.125,
      "step": 13380
    },
    {
      "epoch": 0.966577636613008,
      "grad_norm": 13.193103603231162,
      "learning_rate": 1.69963283614194e-09,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.953125,
      "logps/chosen": -540.0,
      "logps/rejected": -868.0,
      "loss": 0.1702,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.4375,
      "rewards/margins": 3.546875,
      "rewards/rejected": -6.96875,
      "step": 13390
    },
    {
      "epoch": 0.967299501912943,
      "grad_norm": 12.420641074686111,
      "learning_rate": 1.6270863084870967e-09,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.359375,
      "logps/chosen": -540.0,
      "logps/rejected": -880.0,
      "loss": 0.148,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.390625,
      "rewards/margins": 3.671875,
      "rewards/rejected": -7.0625,
      "step": 13400
    },
    {
      "epoch": 0.9680213672128781,
      "grad_norm": 9.844508908233774,
      "learning_rate": 1.5561169556300157e-09,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.1875,
      "logps/chosen": -512.0,
      "logps/rejected": -904.0,
      "loss": 0.1576,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.25,
      "step": 13410
    },
    {
      "epoch": 0.9687432325128131,
      "grad_norm": 12.453119946126321,
      "learning_rate": 1.4867252282280974e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.0625,
      "logps/chosen": -544.0,
      "logps/rejected": -960.0,
      "loss": 0.1563,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.40625,
      "rewards/rejected": -7.8125,
      "step": 13420
    },
    {
      "epoch": 0.9694650978127481,
      "grad_norm": 11.824663689415372,
      "learning_rate": 1.4189115669206719e-09,
      "logits/chosen": -3.0,
      "logits/rejected": -3.390625,
      "logps/chosen": -540.0,
      "logps/rejected": -920.0,
      "loss": 0.1548,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.578125,
      "rewards/margins": 3.90625,
      "rewards/rejected": -7.46875,
      "step": 13430
    },
    {
      "epoch": 0.9701869631126832,
      "grad_norm": 15.647873809699043,
      "learning_rate": 1.3526764023263082e-09,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.390625,
      "logps/chosen": -540.0,
      "logps/rejected": -904.0,
      "loss": 0.1709,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.359375,
      "rewards/margins": 3.875,
      "rewards/rejected": -7.25,
      "step": 13440
    },
    {
      "epoch": 0.9709088284126182,
      "grad_norm": 11.8007155165165,
      "learning_rate": 1.288020155040176e-09,
      "logits/chosen": -3.0,
      "logits/rejected": -3.171875,
      "logps/chosen": -508.0,
      "logps/rejected": -944.0,
      "loss": 0.1505,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.203125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.25,
      "step": 13450
    },
    {
      "epoch": 0.9716306937125533,
      "grad_norm": 12.010615171294127,
      "learning_rate": 1.2249432356311874e-09,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.453125,
      "logps/chosen": -506.0,
      "logps/rejected": -880.0,
      "loss": 0.1671,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.3125,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.3125,
      "step": 13460
    },
    {
      "epoch": 0.9723525590124883,
      "grad_norm": 13.556117427552593,
      "learning_rate": 1.163446044639499e-09,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.796875,
      "logps/chosen": -540.0,
      "logps/rejected": -940.0,
      "loss": 0.1765,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.59375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.6875,
      "step": 13470
    },
    {
      "epoch": 0.9730744243124233,
      "grad_norm": 13.806622517835361,
      "learning_rate": 1.103528972573986e-09,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.859375,
      "logps/chosen": -540.0,
      "logps/rejected": -956.0,
      "loss": 0.1647,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.5,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.71875,
      "step": 13480
    },
    {
      "epoch": 0.9737962896123583,
      "grad_norm": 15.628697565755765,
      "learning_rate": 1.045192399909689e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.828125,
      "logps/chosen": -548.0,
      "logps/rejected": -976.0,
      "loss": 0.1585,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.640625,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.78125,
      "step": 13490
    },
    {
      "epoch": 0.9745181549122933,
      "grad_norm": 12.16635166254457,
      "learning_rate": 9.884366970853986e-10,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.21875,
      "logps/chosen": -536.0,
      "logps/rejected": -948.0,
      "loss": 0.1584,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.453125,
      "rewards/margins": 4.34375,
      "rewards/rejected": -7.8125,
      "step": 13500
    },
    {
      "epoch": 0.9752400202122284,
      "grad_norm": 12.93875014991639,
      "learning_rate": 9.332622245014355e-10,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.984375,
      "logps/chosen": -524.0,
      "logps/rejected": -964.0,
      "loss": 0.1503,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.25,
      "rewards/margins": 4.59375,
      "rewards/rejected": -7.84375,
      "step": 13510
    },
    {
      "epoch": 0.9759618855121635,
      "grad_norm": 18.226868104549972,
      "learning_rate": 8.796693325171522e-10,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.09375,
      "logps/chosen": -532.0,
      "logps/rejected": -928.0,
      "loss": 0.1489,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.46875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.5625,
      "step": 13520
    },
    {
      "epoch": 0.9766837508120985,
      "grad_norm": 12.6827344116078,
      "learning_rate": 8.276583614489352e-10,
      "logits/chosen": -2.875,
      "logits/rejected": -3.234375,
      "logps/chosen": -552.0,
      "logps/rejected": -908.0,
      "loss": 0.1578,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.875,
      "rewards/rejected": -7.34375,
      "step": 13530
    },
    {
      "epoch": 0.9774056161120335,
      "grad_norm": 11.334653937418897,
      "learning_rate": 7.772296415678447e-10,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.078125,
      "logps/chosen": -544.0,
      "logps/rejected": -940.0,
      "loss": 0.1541,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.5625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -7.71875,
      "step": 13540
    },
    {
      "epoch": 0.9781274814119685,
      "grad_norm": 11.560484586823176,
      "learning_rate": 7.283834930976451e-10,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.265625,
      "logps/chosen": -506.0,
      "logps/rejected": -924.0,
      "loss": 0.1381,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.21875,
      "rewards/margins": 4.375,
      "rewards/rejected": -7.59375,
      "step": 13550
    },
    {
      "epoch": 0.9788493467119036,
      "grad_norm": 16.534602374208244,
      "learning_rate": 6.811202262126947e-10,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.859375,
      "logps/chosen": -540.0,
      "logps/rejected": -936.0,
      "loss": 0.1629,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.46875,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.5,
      "step": 13560
    },
    {
      "epoch": 0.9795712120118386,
      "grad_norm": 12.178888562067037,
      "learning_rate": 6.354401410360588e-10,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.375,
      "logps/chosen": -510.0,
      "logps/rejected": -868.0,
      "loss": 0.1622,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.765625,
      "rewards/rejected": -7.0625,
      "step": 13570
    },
    {
      "epoch": 0.9802930773117736,
      "grad_norm": 12.104653249403503,
      "learning_rate": 5.913435276374834e-10,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.03125,
      "logps/chosen": -544.0,
      "logps/rejected": -900.0,
      "loss": 0.1723,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.609375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -7.28125,
      "step": 13580
    },
    {
      "epoch": 0.9810149426117086,
      "grad_norm": 12.795565755806471,
      "learning_rate": 5.488306660317299e-10,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.40625,
      "logps/chosen": -524.0,
      "logps/rejected": -848.0,
      "loss": 0.1652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.40625,
      "rewards/margins": 3.46875,
      "rewards/rejected": -6.875,
      "step": 13590
    },
    {
      "epoch": 0.9817368079116436,
      "grad_norm": 12.42725307238431,
      "learning_rate": 5.079018261766044e-10,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.671875,
      "logps/chosen": -524.0,
      "logps/rejected": -944.0,
      "loss": 0.1573,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.25,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.46875,
      "step": 13600
    },
    {
      "epoch": 0.9824586732115788,
      "grad_norm": 13.525367254020225,
      "learning_rate": 4.685572679713478e-10,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.890625,
      "logps/chosen": -528.0,
      "logps/rejected": -900.0,
      "loss": 0.1493,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.1875,
      "step": 13610
    },
    {
      "epoch": 0.9831805385115138,
      "grad_norm": 18.04289910085876,
      "learning_rate": 4.3079724125499875e-10,
      "logits/chosen": -2.921875,
      "logits/rejected": -3.015625,
      "logps/chosen": -564.0,
      "logps/rejected": -876.0,
      "loss": 0.1807,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.6875,
      "rewards/margins": 3.375,
      "rewards/rejected": -7.0625,
      "step": 13620
    },
    {
      "epoch": 0.9839024038114488,
      "grad_norm": 14.452570058188659,
      "learning_rate": 3.9462198580475505e-10,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.203125,
      "logps/chosen": -556.0,
      "logps/rejected": -948.0,
      "loss": 0.1674,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.5625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.75,
      "step": 13630
    },
    {
      "epoch": 0.9846242691113838,
      "grad_norm": 15.838282800932271,
      "learning_rate": 3.6003173133447585e-10,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.90625,
      "logps/chosen": -536.0,
      "logps/rejected": -928.0,
      "loss": 0.1584,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.484375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.5625,
      "step": 13640
    },
    {
      "epoch": 0.9853461344113188,
      "grad_norm": 15.032447234351043,
      "learning_rate": 3.270266974932101e-10,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.984375,
      "logps/chosen": -540.0,
      "logps/rejected": -968.0,
      "loss": 0.1628,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 4.25,
      "rewards/rejected": -7.65625,
      "step": 13650
    },
    {
      "epoch": 0.9860679997112539,
      "grad_norm": 11.871190818549715,
      "learning_rate": 2.956070938638644e-10,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.296875,
      "logps/chosen": -502.0,
      "logps/rejected": -948.0,
      "loss": 0.1701,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.28125,
      "rewards/margins": 4.53125,
      "rewards/rejected": -7.8125,
      "step": 13660
    },
    {
      "epoch": 0.9867898650111889,
      "grad_norm": 12.61696232543241,
      "learning_rate": 2.6577311996175964e-10,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.015625,
      "logps/chosen": -516.0,
      "logps/rejected": -940.0,
      "loss": 0.17,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.421875,
      "rewards/margins": 4.1875,
      "rewards/rejected": -7.59375,
      "step": 13670
    },
    {
      "epoch": 0.9875117303111239,
      "grad_norm": 10.272387803332773,
      "learning_rate": 2.3752496523343767e-10,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.359375,
      "logps/chosen": -502.0,
      "logps/rejected": -900.0,
      "loss": 0.1562,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.1875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.25,
      "step": 13680
    },
    {
      "epoch": 0.988233595611059,
      "grad_norm": 20.152073213447306,
      "learning_rate": 2.1086280905543984e-10,
      "logits/chosen": -2.828125,
      "logits/rejected": -3.171875,
      "logps/chosen": -528.0,
      "logps/rejected": -916.0,
      "loss": 0.1555,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.25,
      "rewards/margins": 4.25,
      "rewards/rejected": -7.5,
      "step": 13690
    },
    {
      "epoch": 0.988955460910994,
      "grad_norm": 10.343175378829422,
      "learning_rate": 1.8578682073322472e-10,
      "logits/chosen": -2.84375,
      "logits/rejected": -3.0625,
      "logps/chosen": -544.0,
      "logps/rejected": -908.0,
      "loss": 0.1485,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.34375,
      "step": 13700
    },
    {
      "epoch": 0.9896773262109291,
      "grad_norm": 14.470276470411942,
      "learning_rate": 1.6229715950000221e-10,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.125,
      "logps/chosen": -544.0,
      "logps/rejected": -952.0,
      "loss": 0.1674,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.4375,
      "rewards/margins": 4.28125,
      "rewards/rejected": -7.71875,
      "step": 13710
    },
    {
      "epoch": 0.9903991915108641,
      "grad_norm": 10.30865995703702,
      "learning_rate": 1.4039397451573455e-10,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.1875,
      "logps/chosen": -524.0,
      "logps/rejected": -876.0,
      "loss": 0.1474,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.234375,
      "rewards/margins": 3.96875,
      "rewards/rejected": -7.1875,
      "step": 13720
    },
    {
      "epoch": 0.9911210568107991,
      "grad_norm": 10.47719265160857,
      "learning_rate": 1.2007740486624785e-10,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.171875,
      "logps/chosen": -520.0,
      "logps/rejected": -940.0,
      "loss": 0.1576,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.453125,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.6875,
      "step": 13730
    },
    {
      "epoch": 0.9918429221107341,
      "grad_norm": 14.565924806972445,
      "learning_rate": 1.0134757956234418e-10,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.28125,
      "logps/chosen": -540.0,
      "logps/rejected": -916.0,
      "loss": 0.1677,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.84375,
      "rewards/rejected": -7.3125,
      "step": 13740
    },
    {
      "epoch": 0.9925647874106691,
      "grad_norm": 12.384489187451797,
      "learning_rate": 8.420461753891327e-11,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.109375,
      "logps/chosen": -520.0,
      "logps/rejected": -888.0,
      "loss": 0.1676,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.28125,
      "rewards/margins": 3.890625,
      "rewards/rejected": -7.15625,
      "step": 13750
    },
    {
      "epoch": 0.9932866527106042,
      "grad_norm": 14.915479684561454,
      "learning_rate": 6.864862765421087e-11,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.3125,
      "logps/chosen": -512.0,
      "logps/rejected": -920.0,
      "loss": 0.1594,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.28125,
      "rewards/margins": 4.21875,
      "rewards/rejected": -7.5,
      "step": 13760
    },
    {
      "epoch": 0.9940085180105392,
      "grad_norm": 15.639170421509949,
      "learning_rate": 5.4679708689248095e-11,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.265625,
      "logps/chosen": -524.0,
      "logps/rejected": -880.0,
      "loss": 0.1365,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.3125,
      "rewards/margins": 3.8125,
      "rewards/rejected": -7.125,
      "step": 13770
    },
    {
      "epoch": 0.9947303833104743,
      "grad_norm": 13.275598433048923,
      "learning_rate": 4.2297949346986606e-11,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.046875,
      "logps/chosen": -524.0,
      "logps/rejected": -920.0,
      "loss": 0.1479,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.328125,
      "rewards/margins": 4.0,
      "rewards/rejected": -7.34375,
      "step": 13780
    },
    {
      "epoch": 0.9954522486104093,
      "grad_norm": 12.62454409735428,
      "learning_rate": 3.150342825197771e-11,
      "logits/chosen": -2.953125,
      "logits/rejected": -3.0625,
      "logps/chosen": -544.0,
      "logps/rejected": -952.0,
      "loss": 0.1455,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.484375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -7.59375,
      "step": 13790
    },
    {
      "epoch": 0.9961741139103444,
      "grad_norm": 17.396252831844382,
      "learning_rate": 2.2296213949696273e-11,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.109375,
      "logps/chosen": -544.0,
      "logps/rejected": -912.0,
      "loss": 0.1675,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.46875,
      "rewards/margins": 3.75,
      "rewards/rejected": -7.21875,
      "step": 13800
    },
    {
      "epoch": 0.9968959792102794,
      "grad_norm": 8.032660218791134,
      "learning_rate": 1.4676364906235362e-11,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.953125,
      "logps/chosen": -532.0,
      "logps/rejected": -896.0,
      "loss": 0.1619,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.265625,
      "rewards/margins": 3.984375,
      "rewards/rejected": -7.25,
      "step": 13810
    },
    {
      "epoch": 0.9976178445102144,
      "grad_norm": 12.413401210834536,
      "learning_rate": 8.643929507834435e-12,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.140625,
      "logps/chosen": -528.0,
      "logps/rejected": -932.0,
      "loss": 0.1422,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.484375,
      "rewards/margins": 4.125,
      "rewards/rejected": -7.59375,
      "step": 13820
    },
    {
      "epoch": 0.9983397098101494,
      "grad_norm": 10.649363904373544,
      "learning_rate": 4.198946060601782e-12,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.15625,
      "logps/chosen": -552.0,
      "logps/rejected": -928.0,
      "loss": 0.1725,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.5625,
      "step": 13830
    },
    {
      "epoch": 0.9990615751100844,
      "grad_norm": 11.817557960337783,
      "learning_rate": 1.3414427903202242e-12,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.78125,
      "logps/chosen": -544.0,
      "logps/rejected": -956.0,
      "loss": 0.1614,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.46875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -7.53125,
      "step": 13840
    },
    {
      "epoch": 0.9997834404100195,
      "grad_norm": 9.337660145457852,
      "learning_rate": 7.143784222507499e-14,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.15625,
      "logps/chosen": -540.0,
      "logps/rejected": -932.0,
      "loss": 0.1567,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.453125,
      "rewards/margins": 3.984375,
      "rewards/rejected": -7.4375,
      "step": 13850
    },
    {
      "epoch": 1.0,
      "step": 13853,
      "total_flos": 0.0,
      "train_loss": 0.23414572194140112,
      "train_runtime": 130626.0521,
      "train_samples_per_second": 13.574,
      "train_steps_per_second": 0.106
    }
  ],
  "logging_steps": 10,
  "max_steps": 13853,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 10000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}