{
  "best_metric": 0.2634149193763733,
  "best_model_checkpoint": "models/llama3.2-3b-dpo-coarse/checkpoint-10000",
  "epoch": 1.0,
  "eval_steps": 1000,
  "global_step": 13291,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 7.523888345496953e-05,
      "grad_norm": 4.341258647569029,
      "learning_rate": 3.7593984962406016e-10,
      "logits/chosen": -0.609375,
      "logits/rejected": 0.1015625,
      "logps/chosen": -78.0,
      "logps/rejected": -107.0,
      "loss": 1.3828,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0007523888345496953,
      "grad_norm": 4.8117137917376915,
      "learning_rate": 3.759398496240601e-09,
      "logits/chosen": -0.90625,
      "logits/rejected": -0.455078125,
      "logps/chosen": -114.5,
      "logps/rejected": -112.5,
      "loss": 1.3848,
      "rewards/accuracies": 0.2708333432674408,
      "rewards/chosen": -8.726119995117188e-05,
      "rewards/margins": 0.00022792816162109375,
      "rewards/rejected": -0.000316619873046875,
      "step": 10
    },
    {
      "epoch": 0.0015047776690993906,
      "grad_norm": 4.17165744575527,
      "learning_rate": 7.518796992481202e-09,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.5625,
      "logps/chosen": -135.0,
      "logps/rejected": -121.0,
      "loss": 1.3852,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 5.8650970458984375e-05,
      "rewards/margins": -0.00011014938354492188,
      "rewards/rejected": 0.00017070770263671875,
      "step": 20
    },
    {
      "epoch": 0.0022571665036490857,
      "grad_norm": 4.054569109985969,
      "learning_rate": 1.1278195488721804e-08,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.484375,
      "logps/chosen": -136.0,
      "logps/rejected": -135.0,
      "loss": 1.3852,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": -0.00012683868408203125,
      "rewards/margins": -0.000675201416015625,
      "rewards/rejected": 0.00054931640625,
      "step": 30
    },
    {
      "epoch": 0.003009555338198781,
      "grad_norm": 4.645516476554239,
      "learning_rate": 1.5037593984962404e-08,
      "logits/chosen": -0.9453125,
      "logits/rejected": -0.546875,
      "logps/chosen": -122.5,
      "logps/rejected": -124.5,
      "loss": 1.3853,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.0003910064697265625,
      "rewards/margins": -0.0003757476806640625,
      "rewards/rejected": -1.5616416931152344e-05,
      "step": 40
    },
    {
      "epoch": 0.0037619441727484763,
      "grad_norm": 11.963806926950527,
      "learning_rate": 1.8796992481203004e-08,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.56640625,
      "logps/chosen": -115.0,
      "logps/rejected": -121.0,
      "loss": 1.3849,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.00011873245239257812,
      "rewards/margins": -4.482269287109375e-05,
      "rewards/rejected": 0.0001621246337890625,
      "step": 50
    },
    {
      "epoch": 0.004514333007298171,
      "grad_norm": 4.555380211140793,
      "learning_rate": 2.2556390977443608e-08,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.515625,
      "logps/chosen": -118.5,
      "logps/rejected": -110.5,
      "loss": 1.3849,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.0004596710205078125,
      "rewards/margins": -0.000545501708984375,
      "rewards/rejected": 8.821487426757812e-05,
      "step": 60
    },
    {
      "epoch": 0.005266721841847867,
      "grad_norm": 4.293838501994424,
      "learning_rate": 2.6315789473684208e-08,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.345703125,
      "logps/chosen": -129.0,
      "logps/rejected": -136.0,
      "loss": 1.3851,
      "rewards/accuracies": 0.3187499940395355,
      "rewards/chosen": -0.00016498565673828125,
      "rewards/margins": -0.0005950927734375,
      "rewards/rejected": 0.0004291534423828125,
      "step": 70
    },
    {
      "epoch": 0.006019110676397562,
      "grad_norm": 4.051959159424516,
      "learning_rate": 3.007518796992481e-08,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.498046875,
      "logps/chosen": -119.0,
      "logps/rejected": -112.0,
      "loss": 1.3847,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": 0.000530242919921875,
      "rewards/margins": 0.00115203857421875,
      "rewards/rejected": -0.000621795654296875,
      "step": 80
    },
    {
      "epoch": 0.006771499510947258,
      "grad_norm": 3.8907459253766343,
      "learning_rate": 3.383458646616541e-08,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.62109375,
      "logps/chosen": -132.0,
      "logps/rejected": -114.5,
      "loss": 1.3845,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": -0.000354766845703125,
      "rewards/margins": -0.0004405975341796875,
      "rewards/rejected": 8.630752563476562e-05,
      "step": 90
    },
    {
      "epoch": 0.0075238883454969525,
      "grad_norm": 4.433387762372475,
      "learning_rate": 3.759398496240601e-08,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.48046875,
      "logps/chosen": -126.5,
      "logps/rejected": -111.0,
      "loss": 1.3852,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.0002899169921875,
      "rewards/margins": 1.3232231140136719e-05,
      "rewards/rejected": -0.000301361083984375,
      "step": 100
    },
    {
      "epoch": 0.008276277180046648,
      "grad_norm": 4.194659972910355,
      "learning_rate": 4.1353383458646615e-08,
      "logits/chosen": -0.796875,
      "logits/rejected": -0.40625,
      "logps/chosen": -119.5,
      "logps/rejected": -118.0,
      "loss": 1.3845,
      "rewards/accuracies": 0.3812499940395355,
      "rewards/chosen": 0.000713348388671875,
      "rewards/margins": 0.0005950927734375,
      "rewards/rejected": 0.00011777877807617188,
      "step": 110
    },
    {
      "epoch": 0.009028666014596343,
      "grad_norm": 4.495926980431221,
      "learning_rate": 4.5112781954887216e-08,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.498046875,
      "logps/chosen": -119.0,
      "logps/rejected": -104.5,
      "loss": 1.385,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.000499725341796875,
      "rewards/margins": 0.001068115234375,
      "rewards/rejected": -0.00057220458984375,
      "step": 120
    },
    {
      "epoch": 0.009781054849146039,
      "grad_norm": 4.340842622486846,
      "learning_rate": 4.8872180451127816e-08,
      "logits/chosen": -1.125,
      "logits/rejected": -0.65234375,
      "logps/chosen": -95.0,
      "logps/rejected": -108.0,
      "loss": 1.3841,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -9.107589721679688e-05,
      "rewards/margins": 0.000919342041015625,
      "rewards/rejected": -0.001007080078125,
      "step": 130
    },
    {
      "epoch": 0.010533443683695734,
      "grad_norm": 4.666054593835034,
      "learning_rate": 5.2631578947368416e-08,
      "logits/chosen": -0.9140625,
      "logits/rejected": -0.46484375,
      "logps/chosen": -120.5,
      "logps/rejected": -111.0,
      "loss": 1.384,
      "rewards/accuracies": 0.39375001192092896,
      "rewards/chosen": 0.00030517578125,
      "rewards/margins": 0.00102996826171875,
      "rewards/rejected": -0.000720977783203125,
      "step": 140
    },
    {
      "epoch": 0.01128583251824543,
      "grad_norm": 4.456717566796505,
      "learning_rate": 5.6390977443609016e-08,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.1943359375,
      "logps/chosen": -132.0,
      "logps/rejected": -117.0,
      "loss": 1.3835,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.00072479248046875,
      "rewards/margins": 0.0021209716796875,
      "rewards/rejected": -0.0013885498046875,
      "step": 150
    },
    {
      "epoch": 0.012038221352795125,
      "grad_norm": 4.7029983826734965,
      "learning_rate": 6.015037593984962e-08,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.59765625,
      "logps/chosen": -119.5,
      "logps/rejected": -109.5,
      "loss": 1.3828,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.000263214111328125,
      "rewards/margins": 0.002899169921875,
      "rewards/rejected": -0.00262451171875,
      "step": 160
    },
    {
      "epoch": 0.01279061018734482,
      "grad_norm": 4.743739583257617,
      "learning_rate": 6.390977443609022e-08,
      "logits/chosen": -0.98046875,
      "logits/rejected": -0.359375,
      "logps/chosen": -125.0,
      "logps/rejected": -115.5,
      "loss": 1.3829,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -6.67572021484375e-05,
      "rewards/margins": 0.002838134765625,
      "rewards/rejected": -0.0029144287109375,
      "step": 170
    },
    {
      "epoch": 0.013542999021894516,
      "grad_norm": 4.629801087127617,
      "learning_rate": 6.766917293233082e-08,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.5390625,
      "logps/chosen": -103.0,
      "logps/rejected": -116.0,
      "loss": 1.3818,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 7.05718994140625e-05,
      "rewards/margins": 0.0032196044921875,
      "rewards/rejected": -0.003143310546875,
      "step": 180
    },
    {
      "epoch": 0.01429538785644421,
      "grad_norm": 4.681963317016723,
      "learning_rate": 7.142857142857142e-08,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.63671875,
      "logps/chosen": -122.0,
      "logps/rejected": -105.5,
      "loss": 1.3812,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.00079345703125,
      "rewards/margins": 0.0042724609375,
      "rewards/rejected": -0.0034637451171875,
      "step": 190
    },
    {
      "epoch": 0.015047776690993905,
      "grad_norm": 4.315100863037011,
      "learning_rate": 7.518796992481202e-08,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.625,
      "logps/chosen": -122.5,
      "logps/rejected": -122.0,
      "loss": 1.38,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00042724609375,
      "rewards/margins": 0.004852294921875,
      "rewards/rejected": -0.004425048828125,
      "step": 200
    },
    {
      "epoch": 0.0158001655255436,
      "grad_norm": 4.061281687634099,
      "learning_rate": 7.894736842105262e-08,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.6875,
      "logps/chosen": -120.0,
      "logps/rejected": -117.5,
      "loss": 1.38,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.00058746337890625,
      "rewards/margins": 0.0038299560546875,
      "rewards/rejected": -0.004425048828125,
      "step": 210
    },
    {
      "epoch": 0.016552554360093296,
      "grad_norm": 4.635214974171382,
      "learning_rate": 8.270676691729323e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.71484375,
      "logps/chosen": -133.0,
      "logps/rejected": -123.0,
      "loss": 1.3781,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.000885009765625,
      "rewards/margins": 0.0079345703125,
      "rewards/rejected": -0.0087890625,
      "step": 220
    },
    {
      "epoch": 0.017304943194642992,
      "grad_norm": 7.026495529066015,
      "learning_rate": 8.646616541353382e-08,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.54296875,
      "logps/chosen": -111.5,
      "logps/rejected": -132.0,
      "loss": 1.3765,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.00167083740234375,
      "rewards/margins": 0.00909423828125,
      "rewards/rejected": -0.0107421875,
      "step": 230
    },
    {
      "epoch": 0.018057332029192685,
      "grad_norm": 4.801246207640324,
      "learning_rate": 9.022556390977443e-08,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.64453125,
      "logps/chosen": -122.5,
      "logps/rejected": -118.0,
      "loss": 1.3737,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.00017452239990234375,
      "rewards/margins": 0.01092529296875,
      "rewards/rejected": -0.0107421875,
      "step": 240
    },
    {
      "epoch": 0.01880972086374238,
      "grad_norm": 4.821750812422979,
      "learning_rate": 9.398496240601502e-08,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.435546875,
      "logps/chosen": -99.0,
      "logps/rejected": -107.0,
      "loss": 1.3724,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.000759124755859375,
      "rewards/margins": 0.01263427734375,
      "rewards/rejected": -0.01336669921875,
      "step": 250
    },
    {
      "epoch": 0.019562109698292078,
      "grad_norm": 4.10259118582979,
      "learning_rate": 9.774436090225563e-08,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.6796875,
      "logps/chosen": -106.0,
      "logps/rejected": -119.5,
      "loss": 1.3701,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.00101470947265625,
      "rewards/margins": 0.01312255859375,
      "rewards/rejected": -0.01409912109375,
      "step": 260
    },
    {
      "epoch": 0.02031449853284177,
      "grad_norm": 4.180618702238912,
      "learning_rate": 1.0150375939849622e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.76171875,
      "logps/chosen": -137.0,
      "logps/rejected": -121.5,
      "loss": 1.3689,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0022735595703125,
      "rewards/margins": 0.01708984375,
      "rewards/rejected": -0.0194091796875,
      "step": 270
    },
    {
      "epoch": 0.021066887367391467,
      "grad_norm": 35.42334697645909,
      "learning_rate": 1.0526315789473683e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.79296875,
      "logps/chosen": -130.0,
      "logps/rejected": -117.0,
      "loss": 1.3663,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.002044677734375,
      "rewards/margins": 0.0185546875,
      "rewards/rejected": -0.0206298828125,
      "step": 280
    },
    {
      "epoch": 0.021819276201941164,
      "grad_norm": 4.810335410134399,
      "learning_rate": 1.0902255639097744e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.69140625,
      "logps/chosen": -115.5,
      "logps/rejected": -123.5,
      "loss": 1.3622,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0037384033203125,
      "rewards/margins": 0.0233154296875,
      "rewards/rejected": -0.027099609375,
      "step": 290
    },
    {
      "epoch": 0.02257166503649086,
      "grad_norm": 4.984963116921631,
      "learning_rate": 1.1278195488721803e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.65625,
      "logps/chosen": -112.0,
      "logps/rejected": -109.5,
      "loss": 1.3524,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0020751953125,
      "rewards/margins": 0.0302734375,
      "rewards/rejected": -0.0322265625,
      "step": 300
    },
    {
      "epoch": 0.023324053871040553,
      "grad_norm": 4.464528757894068,
      "learning_rate": 1.1654135338345864e-07,
      "logits/chosen": -0.9609375,
      "logits/rejected": -0.5703125,
      "logps/chosen": -115.5,
      "logps/rejected": -96.5,
      "loss": 1.3494,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0074462890625,
      "rewards/margins": 0.035888671875,
      "rewards/rejected": -0.04345703125,
      "step": 310
    },
    {
      "epoch": 0.02407644270559025,
      "grad_norm": 5.394493726216202,
      "learning_rate": 1.2030075187969923e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.671875,
      "logps/chosen": -122.5,
      "logps/rejected": -123.5,
      "loss": 1.3438,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0067138671875,
      "rewards/margins": 0.048583984375,
      "rewards/rejected": -0.05517578125,
      "step": 320
    },
    {
      "epoch": 0.024828831540139946,
      "grad_norm": 4.534541139190117,
      "learning_rate": 1.2406015037593983e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.66015625,
      "logps/chosen": -121.0,
      "logps/rejected": -144.0,
      "loss": 1.3354,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.010498046875,
      "rewards/margins": 0.061279296875,
      "rewards/rejected": -0.07177734375,
      "step": 330
    },
    {
      "epoch": 0.02558122037468964,
      "grad_norm": 4.6625622738589945,
      "learning_rate": 1.2781954887218045e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.6953125,
      "logps/chosen": -131.0,
      "logps/rejected": -135.0,
      "loss": 1.3265,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.018310546875,
      "rewards/margins": 0.05859375,
      "rewards/rejected": -0.07666015625,
      "step": 340
    },
    {
      "epoch": 0.026333609209239335,
      "grad_norm": 4.536936769368363,
      "learning_rate": 1.3157894736842104e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.51953125,
      "logps/chosen": -137.0,
      "logps/rejected": -139.0,
      "loss": 1.3188,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.014404296875,
      "rewards/margins": 0.0830078125,
      "rewards/rejected": -0.09716796875,
      "step": 350
    },
    {
      "epoch": 0.02708599804378903,
      "grad_norm": 4.999380667888614,
      "learning_rate": 1.3533834586466163e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.57421875,
      "logps/chosen": -132.0,
      "logps/rejected": -126.5,
      "loss": 1.3144,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.0299072265625,
      "rewards/margins": 0.0830078125,
      "rewards/rejected": -0.11279296875,
      "step": 360
    },
    {
      "epoch": 0.027838386878338724,
      "grad_norm": 4.558742885287366,
      "learning_rate": 1.3909774436090225e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.703125,
      "logps/chosen": -111.0,
      "logps/rejected": -118.5,
      "loss": 1.298,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.044677734375,
      "rewards/margins": 0.09228515625,
      "rewards/rejected": -0.13671875,
      "step": 370
    },
    {
      "epoch": 0.02859077571288842,
      "grad_norm": 9.843888703154898,
      "learning_rate": 1.4285714285714285e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.61328125,
      "logps/chosen": -121.0,
      "logps/rejected": -122.5,
      "loss": 1.2861,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0634765625,
      "rewards/margins": 0.11669921875,
      "rewards/rejected": -0.1806640625,
      "step": 380
    },
    {
      "epoch": 0.029343164547438117,
      "grad_norm": 5.350833493354305,
      "learning_rate": 1.4661654135338344e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -0.73046875,
      "logps/chosen": -148.0,
      "logps/rejected": -138.0,
      "loss": 1.2845,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.08251953125,
      "rewards/margins": 0.11865234375,
      "rewards/rejected": -0.201171875,
      "step": 390
    },
    {
      "epoch": 0.03009555338198781,
      "grad_norm": 5.432620258794752,
      "learning_rate": 1.5037593984962403e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.63671875,
      "logps/chosen": -130.0,
      "logps/rejected": -135.0,
      "loss": 1.2695,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.10302734375,
      "rewards/margins": 0.1064453125,
      "rewards/rejected": -0.208984375,
      "step": 400
    },
    {
      "epoch": 0.030847942216537506,
      "grad_norm": 5.2477366871786515,
      "learning_rate": 1.5413533834586465e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.703125,
      "logps/chosen": -133.0,
      "logps/rejected": -135.0,
      "loss": 1.2596,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.08203125,
      "rewards/margins": 0.1484375,
      "rewards/rejected": -0.2294921875,
      "step": 410
    },
    {
      "epoch": 0.0316003310510872,
      "grad_norm": 10.829610851340542,
      "learning_rate": 1.5789473684210525e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.66796875,
      "logps/chosen": -122.0,
      "logps/rejected": -132.0,
      "loss": 1.2411,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.09912109375,
      "rewards/margins": 0.1357421875,
      "rewards/rejected": -0.2353515625,
      "step": 420
    },
    {
      "epoch": 0.0323527198856369,
      "grad_norm": 6.823107563176444,
      "learning_rate": 1.6165413533834584e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.6328125,
      "logps/chosen": -141.0,
      "logps/rejected": -140.0,
      "loss": 1.2337,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.10009765625,
      "rewards/margins": 0.1650390625,
      "rewards/rejected": -0.265625,
      "step": 430
    },
    {
      "epoch": 0.03310510872018659,
      "grad_norm": 6.428655642500386,
      "learning_rate": 1.6541353383458646e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.86328125,
      "logps/chosen": -164.0,
      "logps/rejected": -155.0,
      "loss": 1.2348,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.08154296875,
      "rewards/margins": 0.1826171875,
      "rewards/rejected": -0.263671875,
      "step": 440
    },
    {
      "epoch": 0.033857497554736285,
      "grad_norm": 5.858220580425247,
      "learning_rate": 1.6917293233082705e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.80078125,
      "logps/chosen": -148.0,
      "logps/rejected": -138.0,
      "loss": 1.2136,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.126953125,
      "rewards/margins": 0.1513671875,
      "rewards/rejected": -0.27734375,
      "step": 450
    },
    {
      "epoch": 0.034609886389285985,
      "grad_norm": 9.329943085898075,
      "learning_rate": 1.7293233082706765e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.7890625,
      "logps/chosen": -134.0,
      "logps/rejected": -158.0,
      "loss": 1.2067,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.10009765625,
      "rewards/margins": 0.208984375,
      "rewards/rejected": -0.30859375,
      "step": 460
    },
    {
      "epoch": 0.03536227522383568,
      "grad_norm": 6.481816739668089,
      "learning_rate": 1.7669172932330824e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.82421875,
      "logps/chosen": -138.0,
      "logps/rejected": -179.0,
      "loss": 1.209,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.1279296875,
      "rewards/margins": 0.2578125,
      "rewards/rejected": -0.38671875,
      "step": 470
    },
    {
      "epoch": 0.03611466405838537,
      "grad_norm": 6.375000081503385,
      "learning_rate": 1.8045112781954886e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.0078125,
      "logps/chosen": -142.0,
      "logps/rejected": -148.0,
      "loss": 1.1846,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.1533203125,
      "rewards/margins": 0.22265625,
      "rewards/rejected": -0.375,
      "step": 480
    },
    {
      "epoch": 0.03686705289293507,
      "grad_norm": 6.483632157324514,
      "learning_rate": 1.8421052631578946e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -0.9609375,
      "logps/chosen": -145.0,
      "logps/rejected": -168.0,
      "loss": 1.2057,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.1416015625,
      "rewards/margins": 0.2353515625,
      "rewards/rejected": -0.376953125,
      "step": 490
    },
    {
      "epoch": 0.03761944172748476,
      "grad_norm": 7.050232882797307,
      "learning_rate": 1.8796992481203005e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.71875,
      "logps/chosen": -146.0,
      "logps/rejected": -151.0,
      "loss": 1.1998,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.20703125,
      "rewards/margins": 0.19140625,
      "rewards/rejected": -0.3984375,
      "step": 500
    },
    {
      "epoch": 0.038371830562034456,
      "grad_norm": 7.049238996393664,
      "learning_rate": 1.9172932330827067e-07,
      "logits/chosen": -1.453125,
      "logits/rejected": -0.96875,
      "logps/chosen": -122.0,
      "logps/rejected": -136.0,
      "loss": 1.1846,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.146484375,
      "rewards/margins": 0.232421875,
      "rewards/rejected": -0.37890625,
      "step": 510
    },
    {
      "epoch": 0.039124219396584156,
      "grad_norm": 8.098559748937967,
      "learning_rate": 1.9548872180451126e-07,
      "logits/chosen": -1.4296875,
      "logits/rejected": -0.8125,
      "logps/chosen": -127.5,
      "logps/rejected": -148.0,
      "loss": 1.1697,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.1572265625,
      "rewards/margins": 0.259765625,
      "rewards/rejected": -0.416015625,
      "step": 520
    },
    {
      "epoch": 0.03987660823113385,
      "grad_norm": 12.14724386706595,
      "learning_rate": 1.9924812030075186e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.80859375,
      "logps/chosen": -132.0,
      "logps/rejected": -153.0,
      "loss": 1.1718,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.185546875,
      "rewards/margins": 0.224609375,
      "rewards/rejected": -0.41015625,
      "step": 530
    },
    {
      "epoch": 0.04062899706568354,
      "grad_norm": 11.535486009035319,
      "learning_rate": 2.0300751879699245e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -0.73046875,
      "logps/chosen": -133.0,
      "logps/rejected": -150.0,
      "loss": 1.1685,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.1767578125,
      "rewards/margins": 0.29296875,
      "rewards/rejected": -0.470703125,
      "step": 540
    },
    {
      "epoch": 0.04138138590023324,
      "grad_norm": 9.643293882651587,
      "learning_rate": 2.0676691729323307e-07,
      "logits/chosen": -1.453125,
      "logits/rejected": -0.84375,
      "logps/chosen": -124.0,
      "logps/rejected": -157.0,
      "loss": 1.1448,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.140625,
      "rewards/margins": 0.36328125,
      "rewards/rejected": -0.50390625,
      "step": 550
    },
    {
      "epoch": 0.042133774734782935,
      "grad_norm": 7.628568942474412,
      "learning_rate": 2.1052631578947366e-07,
      "logits/chosen": -1.5,
      "logits/rejected": -1.0546875,
      "logps/chosen": -136.0,
      "logps/rejected": -171.0,
      "loss": 1.1412,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1904296875,
      "rewards/margins": 0.30078125,
      "rewards/rejected": -0.4921875,
      "step": 560
    },
    {
      "epoch": 0.042886163569332635,
      "grad_norm": 8.138939777646883,
      "learning_rate": 2.1428571428571426e-07,
      "logits/chosen": -1.546875,
      "logits/rejected": -1.0,
      "logps/chosen": -149.0,
      "logps/rejected": -160.0,
      "loss": 1.1427,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.220703125,
      "rewards/margins": 0.29296875,
      "rewards/rejected": -0.515625,
      "step": 570
    },
    {
      "epoch": 0.04363855240388233,
      "grad_norm": 9.365285956215729,
      "learning_rate": 2.1804511278195488e-07,
      "logits/chosen": -1.5234375,
      "logits/rejected": -0.984375,
      "logps/chosen": -152.0,
      "logps/rejected": -160.0,
      "loss": 1.1565,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.2099609375,
      "rewards/margins": 0.3125,
      "rewards/rejected": -0.5234375,
      "step": 580
    },
    {
      "epoch": 0.04439094123843202,
      "grad_norm": 12.396323618982942,
      "learning_rate": 2.2180451127819547e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.7265625,
      "logps/chosen": -137.0,
      "logps/rejected": -160.0,
      "loss": 1.1287,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.21875,
      "rewards/margins": 0.37109375,
      "rewards/rejected": -0.58984375,
      "step": 590
    },
    {
      "epoch": 0.04514333007298172,
      "grad_norm": 10.868017521673984,
      "learning_rate": 2.2556390977443606e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.0703125,
      "logps/chosen": -158.0,
      "logps/rejected": -176.0,
      "loss": 1.1294,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.236328125,
      "rewards/margins": 0.28125,
      "rewards/rejected": -0.51953125,
      "step": 600
    },
    {
      "epoch": 0.04589571890753141,
      "grad_norm": 10.506405897297855,
      "learning_rate": 2.2932330827067666e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -0.83203125,
      "logps/chosen": -123.0,
      "logps/rejected": -161.0,
      "loss": 1.116,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.220703125,
      "rewards/margins": 0.33984375,
      "rewards/rejected": -0.5625,
      "step": 610
    },
    {
      "epoch": 0.046648107742081106,
      "grad_norm": 9.803209012042108,
      "learning_rate": 2.3308270676691728e-07,
      "logits/chosen": -1.5390625,
      "logits/rejected": -0.90234375,
      "logps/chosen": -132.0,
      "logps/rejected": -173.0,
      "loss": 1.1104,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26171875,
      "rewards/margins": 0.302734375,
      "rewards/rejected": -0.5625,
      "step": 620
    },
    {
      "epoch": 0.047400496576630806,
      "grad_norm": 12.589621806299409,
      "learning_rate": 2.3684210526315787e-07,
      "logits/chosen": -1.46875,
      "logits/rejected": -0.9140625,
      "logps/chosen": -156.0,
      "logps/rejected": -176.0,
      "loss": 1.1055,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.2734375,
      "rewards/margins": 0.33203125,
      "rewards/rejected": -0.60546875,
      "step": 630
    },
    {
      "epoch": 0.0481528854111805,
      "grad_norm": 9.0352756132327,
      "learning_rate": 2.4060150375939847e-07,
      "logits/chosen": -1.4453125,
      "logits/rejected": -0.875,
      "logps/chosen": -142.0,
      "logps/rejected": -182.0,
      "loss": 1.0938,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.2216796875,
      "rewards/margins": 0.4140625,
      "rewards/rejected": -0.6328125,
      "step": 640
    },
    {
      "epoch": 0.04890527424573019,
      "grad_norm": 21.087177570928517,
      "learning_rate": 2.443609022556391e-07,
      "logits/chosen": -1.6171875,
      "logits/rejected": -0.95703125,
      "logps/chosen": -139.0,
      "logps/rejected": -179.0,
      "loss": 1.1284,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.2578125,
      "rewards/margins": 0.306640625,
      "rewards/rejected": -0.5625,
      "step": 650
    },
    {
      "epoch": 0.04965766308027989,
      "grad_norm": 13.401671439040635,
      "learning_rate": 2.4812030075187965e-07,
      "logits/chosen": -1.5390625,
      "logits/rejected": -1.0234375,
      "logps/chosen": -132.0,
      "logps/rejected": -176.0,
      "loss": 1.0985,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.271484375,
      "rewards/margins": 0.3828125,
      "rewards/rejected": -0.65234375,
      "step": 660
    },
    {
      "epoch": 0.050410051914829584,
      "grad_norm": 13.294474926401744,
      "learning_rate": 2.518796992481203e-07,
      "logits/chosen": -1.484375,
      "logits/rejected": -1.0390625,
      "logps/chosen": -161.0,
      "logps/rejected": -190.0,
      "loss": 1.0669,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.30078125,
      "rewards/margins": 0.39453125,
      "rewards/rejected": -0.6953125,
      "step": 670
    },
    {
      "epoch": 0.05116244074937928,
      "grad_norm": 18.407146194272787,
      "learning_rate": 2.556390977443609e-07,
      "logits/chosen": -1.6953125,
      "logits/rejected": -1.15625,
      "logps/chosen": -158.0,
      "logps/rejected": -184.0,
      "loss": 1.0871,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.2412109375,
      "rewards/margins": 0.40234375,
      "rewards/rejected": -0.64453125,
      "step": 680
    },
    {
      "epoch": 0.05191482958392898,
      "grad_norm": 12.322784273369955,
      "learning_rate": 2.593984962406015e-07,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.15625,
      "logps/chosen": -158.0,
      "logps/rejected": -189.0,
      "loss": 1.0732,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.271484375,
      "rewards/margins": 0.40234375,
      "rewards/rejected": -0.671875,
      "step": 690
    },
    {
      "epoch": 0.05266721841847867,
      "grad_norm": 17.89633832287342,
      "learning_rate": 2.631578947368421e-07,
      "logits/chosen": -1.484375,
      "logits/rejected": -1.03125,
      "logps/chosen": -157.0,
      "logps/rejected": -193.0,
      "loss": 1.0646,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.2578125,
      "rewards/margins": 0.47265625,
      "rewards/rejected": -0.73046875,
      "step": 700
    },
    {
      "epoch": 0.05341960725302836,
      "grad_norm": 15.6259327924367,
      "learning_rate": 2.669172932330827e-07,
      "logits/chosen": -1.6796875,
      "logits/rejected": -1.15625,
      "logps/chosen": -166.0,
      "logps/rejected": -212.0,
      "loss": 1.0841,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.265625,
      "rewards/margins": 0.451171875,
      "rewards/rejected": -0.71484375,
      "step": 710
    },
    {
      "epoch": 0.05417199608757806,
      "grad_norm": 12.011659284099709,
      "learning_rate": 2.7067669172932327e-07,
      "logits/chosen": -1.5625,
      "logits/rejected": -1.1640625,
      "logps/chosen": -152.0,
      "logps/rejected": -208.0,
      "loss": 1.0246,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2265625,
      "rewards/margins": 0.53515625,
      "rewards/rejected": -0.7578125,
      "step": 720
    },
    {
      "epoch": 0.054924384922127756,
      "grad_norm": 19.31666791194093,
      "learning_rate": 2.744360902255639e-07,
      "logits/chosen": -1.5,
      "logits/rejected": -1.171875,
      "logps/chosen": -152.0,
      "logps/rejected": -178.0,
      "loss": 1.0507,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.302734375,
      "rewards/margins": 0.52734375,
      "rewards/rejected": -0.828125,
      "step": 730
    },
    {
      "epoch": 0.05567677375667745,
      "grad_norm": 14.4472654196538,
      "learning_rate": 2.781954887218045e-07,
      "logits/chosen": -1.640625,
      "logits/rejected": -1.2109375,
      "logps/chosen": -153.0,
      "logps/rejected": -198.0,
      "loss": 1.0496,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.31640625,
      "rewards/margins": 0.5234375,
      "rewards/rejected": -0.84375,
      "step": 740
    },
    {
      "epoch": 0.05642916259122715,
      "grad_norm": 14.840537793970611,
      "learning_rate": 2.8195488721804513e-07,
      "logits/chosen": -1.6796875,
      "logits/rejected": -1.15625,
      "logps/chosen": -158.0,
      "logps/rejected": -200.0,
      "loss": 1.0115,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.314453125,
      "rewards/margins": 0.5390625,
      "rewards/rejected": -0.8515625,
      "step": 750
    },
    {
      "epoch": 0.05718155142577684,
      "grad_norm": 14.819080015178072,
      "learning_rate": 2.857142857142857e-07,
      "logits/chosen": -1.578125,
      "logits/rejected": -1.09375,
      "logps/chosen": -163.0,
      "logps/rejected": -189.0,
      "loss": 1.0292,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3515625,
      "rewards/margins": 0.41796875,
      "rewards/rejected": -0.76953125,
      "step": 760
    },
    {
      "epoch": 0.057933940260326534,
      "grad_norm": 11.42558960615719,
      "learning_rate": 2.894736842105263e-07,
      "logits/chosen": -1.7421875,
      "logits/rejected": -1.2890625,
      "logps/chosen": -169.0,
      "logps/rejected": -194.0,
      "loss": 1.0136,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.294921875,
      "rewards/margins": 0.49609375,
      "rewards/rejected": -0.79296875,
      "step": 770
    },
    {
      "epoch": 0.058686329094876234,
      "grad_norm": 12.716142568356348,
      "learning_rate": 2.932330827067669e-07,
      "logits/chosen": -1.625,
      "logits/rejected": -1.234375,
      "logps/chosen": -146.0,
      "logps/rejected": -201.0,
      "loss": 1.0014,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.33984375,
      "rewards/margins": 0.57421875,
      "rewards/rejected": -0.9140625,
      "step": 780
    },
    {
      "epoch": 0.05943871792942593,
      "grad_norm": 12.590556564411969,
      "learning_rate": 2.969924812030075e-07,
      "logits/chosen": -1.6484375,
      "logits/rejected": -1.2734375,
      "logps/chosen": -170.0,
      "logps/rejected": -200.0,
      "loss": 0.9916,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.35546875,
      "rewards/margins": 0.5234375,
      "rewards/rejected": -0.87890625,
      "step": 790
    },
    {
      "epoch": 0.06019110676397562,
      "grad_norm": 15.209361328226734,
      "learning_rate": 3.0075187969924807e-07,
      "logits/chosen": -1.78125,
      "logits/rejected": -1.2890625,
      "logps/chosen": -166.0,
      "logps/rejected": -219.0,
      "loss": 1.0034,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.34375,
      "rewards/margins": 0.55078125,
      "rewards/rejected": -0.89453125,
      "step": 800
    },
    {
      "epoch": 0.06094349559852532,
      "grad_norm": 29.394031643094735,
      "learning_rate": 3.0451127819548874e-07,
      "logits/chosen": -1.546875,
      "logits/rejected": -1.1484375,
      "logps/chosen": -178.0,
      "logps/rejected": -224.0,
      "loss": 1.0059,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.42578125,
      "rewards/margins": 0.53515625,
      "rewards/rejected": -0.9609375,
      "step": 810
    },
    {
      "epoch": 0.06169588443307501,
      "grad_norm": 13.40769311727928,
      "learning_rate": 3.082706766917293e-07,
      "logits/chosen": -1.8125,
      "logits/rejected": -1.296875,
      "logps/chosen": -164.0,
      "logps/rejected": -222.0,
      "loss": 0.9588,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.34765625,
      "rewards/margins": 0.64453125,
      "rewards/rejected": -0.9921875,
      "step": 820
    },
    {
      "epoch": 0.062448273267624706,
      "grad_norm": 26.061363851180456,
      "learning_rate": 3.1203007518796993e-07,
      "logits/chosen": -1.875,
      "logits/rejected": -1.4140625,
      "logps/chosen": -157.0,
      "logps/rejected": -238.0,
      "loss": 1.0023,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.380859375,
      "rewards/margins": 0.55859375,
      "rewards/rejected": -0.94140625,
      "step": 830
    },
    {
      "epoch": 0.0632006621021744,
      "grad_norm": 16.844465366632328,
      "learning_rate": 3.157894736842105e-07,
      "logits/chosen": -1.90625,
      "logits/rejected": -1.2421875,
      "logps/chosen": -161.0,
      "logps/rejected": -229.0,
      "loss": 0.961,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.40234375,
      "rewards/margins": 0.703125,
      "rewards/rejected": -1.1015625,
      "step": 840
    },
    {
      "epoch": 0.0639530509367241,
      "grad_norm": 13.817122234386906,
      "learning_rate": 3.195488721804511e-07,
      "logits/chosen": -1.75,
      "logits/rejected": -1.359375,
      "logps/chosen": -174.0,
      "logps/rejected": -232.0,
      "loss": 0.9748,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.48046875,
      "rewards/margins": 0.66015625,
      "rewards/rejected": -1.140625,
      "step": 850
    },
    {
      "epoch": 0.0647054397712738,
      "grad_norm": 14.337185379686227,
      "learning_rate": 3.233082706766917e-07,
      "logits/chosen": -1.765625,
      "logits/rejected": -1.484375,
      "logps/chosen": -174.0,
      "logps/rejected": -234.0,
      "loss": 0.9438,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.376953125,
      "rewards/margins": 0.70703125,
      "rewards/rejected": -1.0859375,
      "step": 860
    },
    {
      "epoch": 0.06545782860582348,
      "grad_norm": 15.721234776565389,
      "learning_rate": 3.270676691729323e-07,
      "logits/chosen": -1.953125,
      "logits/rejected": -1.28125,
      "logps/chosen": -169.0,
      "logps/rejected": -212.0,
      "loss": 0.9649,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.435546875,
      "rewards/margins": 0.609375,
      "rewards/rejected": -1.046875,
      "step": 870
    },
    {
      "epoch": 0.06621021744037318,
      "grad_norm": 15.030764293651927,
      "learning_rate": 3.308270676691729e-07,
      "logits/chosen": -1.796875,
      "logits/rejected": -1.4140625,
      "logps/chosen": -160.0,
      "logps/rejected": -243.0,
      "loss": 0.9512,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.412109375,
      "rewards/margins": 0.75,
      "rewards/rejected": -1.1640625,
      "step": 880
    },
    {
      "epoch": 0.06696260627492288,
      "grad_norm": 12.709330732241797,
      "learning_rate": 3.3458646616541354e-07,
      "logits/chosen": -1.8671875,
      "logits/rejected": -1.4453125,
      "logps/chosen": -161.0,
      "logps/rejected": -226.0,
      "loss": 0.9335,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.427734375,
      "rewards/margins": 0.703125,
      "rewards/rejected": -1.1328125,
      "step": 890
    },
    {
      "epoch": 0.06771499510947257,
      "grad_norm": 13.329463729317034,
      "learning_rate": 3.383458646616541e-07,
      "logits/chosen": -1.9140625,
      "logits/rejected": -1.4375,
      "logps/chosen": -144.0,
      "logps/rejected": -203.0,
      "loss": 0.9476,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.421875,
      "rewards/margins": 0.60546875,
      "rewards/rejected": -1.03125,
      "step": 900
    },
    {
      "epoch": 0.06846738394402227,
      "grad_norm": 14.254589171753144,
      "learning_rate": 3.4210526315789473e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.3515625,
      "logps/chosen": -155.0,
      "logps/rejected": -237.0,
      "loss": 0.9861,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.470703125,
      "rewards/margins": 0.625,
      "rewards/rejected": -1.1015625,
      "step": 910
    },
    {
      "epoch": 0.06921977277857197,
      "grad_norm": 19.45051301011936,
      "learning_rate": 3.458646616541353e-07,
      "logits/chosen": -1.7421875,
      "logits/rejected": -1.4453125,
      "logps/chosen": -182.0,
      "logps/rejected": -234.0,
      "loss": 0.9378,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.44921875,
      "rewards/margins": 0.7734375,
      "rewards/rejected": -1.21875,
      "step": 920
    },
    {
      "epoch": 0.06997216161312166,
      "grad_norm": 17.175247665984763,
      "learning_rate": 3.496240601503759e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.625,
      "logps/chosen": -156.0,
      "logps/rejected": -224.0,
      "loss": 0.8934,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.404296875,
      "rewards/margins": 0.7734375,
      "rewards/rejected": -1.1796875,
      "step": 930
    },
    {
      "epoch": 0.07072455044767136,
      "grad_norm": 17.201323381588264,
      "learning_rate": 3.533834586466165e-07,
      "logits/chosen": -1.953125,
      "logits/rejected": -1.671875,
      "logps/chosen": -153.0,
      "logps/rejected": -242.0,
      "loss": 0.92,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.376953125,
      "rewards/margins": 0.84375,
      "rewards/rejected": -1.21875,
      "step": 940
    },
    {
      "epoch": 0.07147693928222106,
      "grad_norm": 19.086994540163545,
      "learning_rate": 3.5714285714285716e-07,
      "logits/chosen": -1.921875,
      "logits/rejected": -1.515625,
      "logps/chosen": -152.0,
      "logps/rejected": -247.0,
      "loss": 0.9364,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.478515625,
      "rewards/margins": 0.77734375,
      "rewards/rejected": -1.2578125,
      "step": 950
    },
    {
      "epoch": 0.07222932811677074,
      "grad_norm": 16.58190223521324,
      "learning_rate": 3.609022556390977e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.453125,
      "logps/chosen": -174.0,
      "logps/rejected": -253.0,
      "loss": 0.9269,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.5,
      "rewards/margins": 0.86328125,
      "rewards/rejected": -1.359375,
      "step": 960
    },
    {
      "epoch": 0.07298171695132044,
      "grad_norm": 15.408559250697381,
      "learning_rate": 3.6466165413533834e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.6796875,
      "logps/chosen": -206.0,
      "logps/rejected": -254.0,
      "loss": 0.9099,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5703125,
      "rewards/margins": 0.80078125,
      "rewards/rejected": -1.375,
      "step": 970
    },
    {
      "epoch": 0.07373410578587014,
      "grad_norm": 15.561528223892218,
      "learning_rate": 3.684210526315789e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -1.7109375,
      "logps/chosen": -176.0,
      "logps/rejected": -260.0,
      "loss": 0.9037,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.53125,
      "rewards/margins": 0.82421875,
      "rewards/rejected": -1.359375,
      "step": 980
    },
    {
      "epoch": 0.07448649462041983,
      "grad_norm": 14.282960703694673,
      "learning_rate": 3.7218045112781953e-07,
      "logits/chosen": -1.8984375,
      "logits/rejected": -1.65625,
      "logps/chosen": -165.0,
      "logps/rejected": -241.0,
      "loss": 0.9046,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.546875,
      "rewards/margins": 0.77734375,
      "rewards/rejected": -1.328125,
      "step": 990
    },
    {
      "epoch": 0.07523888345496953,
      "grad_norm": 15.155658640673064,
      "learning_rate": 3.759398496240601e-07,
      "logits/chosen": -1.7265625,
      "logits/rejected": -1.3203125,
      "logps/chosen": -183.0,
      "logps/rejected": -260.0,
      "loss": 0.9034,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.578125,
      "rewards/margins": 0.73046875,
      "rewards/rejected": -1.3125,
      "step": 1000
    },
    {
      "epoch": 0.07523888345496953,
      "eval_logits/chosen": -1.9453125,
      "eval_logits/rejected": -1.609375,
      "eval_logps/chosen": -186.0,
      "eval_logps/rejected": -254.0,
      "eval_loss": 0.4537469744682312,
      "eval_rewards/accuracies": 0.8118652701377869,
      "eval_rewards/chosen": -0.54296875,
      "eval_rewards/margins": 0.82421875,
      "eval_rewards/rejected": -1.3671875,
      "eval_runtime": 2670.1294,
      "eval_samples_per_second": 35.389,
      "eval_steps_per_second": 0.553,
      "step": 1000
    },
    {
      "epoch": 0.07599127228951923,
      "grad_norm": 13.94477905082793,
      "learning_rate": 3.796992481203007e-07,
      "logits/chosen": -1.9140625,
      "logits/rejected": -1.546875,
      "logps/chosen": -180.0,
      "logps/rejected": -260.0,
      "loss": 0.8747,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.53515625,
      "rewards/margins": 0.92578125,
      "rewards/rejected": -1.4609375,
      "step": 1010
    },
    {
      "epoch": 0.07674366112406891,
      "grad_norm": 14.738352019679763,
      "learning_rate": 3.8345864661654134e-07,
      "logits/chosen": -1.9609375,
      "logits/rejected": -1.6171875,
      "logps/chosen": -173.0,
      "logps/rejected": -264.0,
      "loss": 0.8653,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.486328125,
      "rewards/margins": 0.92578125,
      "rewards/rejected": -1.4140625,
      "step": 1020
    },
    {
      "epoch": 0.07749604995861861,
      "grad_norm": 14.799788450022126,
      "learning_rate": 3.8721804511278196e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.5546875,
      "logps/chosen": -167.0,
      "logps/rejected": -252.0,
      "loss": 0.8685,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.515625,
      "rewards/margins": 0.96484375,
      "rewards/rejected": -1.4765625,
      "step": 1030
    },
    {
      "epoch": 0.07824843879316831,
      "grad_norm": 22.35782954167918,
      "learning_rate": 3.909774436090225e-07,
      "logits/chosen": -1.8984375,
      "logits/rejected": -1.53125,
      "logps/chosen": -186.0,
      "logps/rejected": -262.0,
      "loss": 0.8866,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.609375,
      "rewards/margins": 0.78515625,
      "rewards/rejected": -1.3984375,
      "step": 1040
    },
    {
      "epoch": 0.079000827627718,
      "grad_norm": 13.38808270444351,
      "learning_rate": 3.9473684210526315e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -1.7890625,
      "logps/chosen": -190.0,
      "logps/rejected": -286.0,
      "loss": 0.8475,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.515625,
      "rewards/margins": 1.015625,
      "rewards/rejected": -1.5234375,
      "step": 1050
    },
    {
      "epoch": 0.0797532164622677,
      "grad_norm": 16.92345467659103,
      "learning_rate": 3.984962406015037e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.78125,
      "logps/chosen": -181.0,
      "logps/rejected": -262.0,
      "loss": 0.8592,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5390625,
      "rewards/margins": 0.94921875,
      "rewards/rejected": -1.4921875,
      "step": 1060
    },
    {
      "epoch": 0.0805056052968174,
      "grad_norm": 27.23539153662501,
      "learning_rate": 4.0225563909774433e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -1.90625,
      "logps/chosen": -183.0,
      "logps/rejected": -262.0,
      "loss": 0.8843,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.474609375,
      "rewards/margins": 0.9453125,
      "rewards/rejected": -1.421875,
      "step": 1070
    },
    {
      "epoch": 0.08125799413136708,
      "grad_norm": 16.198642386820726,
      "learning_rate": 4.060150375939849e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -1.828125,
      "logps/chosen": -191.0,
      "logps/rejected": -294.0,
      "loss": 0.8807,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.53125,
      "rewards/margins": 1.03125,
      "rewards/rejected": -1.5625,
      "step": 1080
    },
    {
      "epoch": 0.08201038296591678,
      "grad_norm": 12.90266802626918,
      "learning_rate": 4.0977443609022557e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.71875,
      "logps/chosen": -207.0,
      "logps/rejected": -294.0,
      "loss": 0.8581,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6484375,
      "rewards/margins": 0.984375,
      "rewards/rejected": -1.6328125,
      "step": 1090
    },
    {
      "epoch": 0.08276277180046648,
      "grad_norm": 14.647594208070746,
      "learning_rate": 4.1353383458646614e-07,
      "logits/chosen": -1.8828125,
      "logits/rejected": -1.5234375,
      "logps/chosen": -199.0,
      "logps/rejected": -272.0,
      "loss": 0.8419,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.6484375,
      "rewards/margins": 0.9609375,
      "rewards/rejected": -1.609375,
      "step": 1100
    },
    {
      "epoch": 0.08351516063501617,
      "grad_norm": 15.249707706731298,
      "learning_rate": 4.1729323308270676e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.6796875,
      "logps/chosen": -191.0,
      "logps/rejected": -268.0,
      "loss": 0.7998,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.65625,
      "rewards/margins": 1.0,
      "rewards/rejected": -1.65625,
      "step": 1110
    },
    {
      "epoch": 0.08426754946956587,
      "grad_norm": 17.598464162391597,
      "learning_rate": 4.2105263157894733e-07,
      "logits/chosen": -1.9921875,
      "logits/rejected": -1.6171875,
      "logps/chosen": -195.0,
      "logps/rejected": -278.0,
      "loss": 0.8729,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.69140625,
      "rewards/margins": 0.92578125,
      "rewards/rejected": -1.6171875,
      "step": 1120
    },
    {
      "epoch": 0.08501993830411557,
      "grad_norm": 16.90205587804178,
      "learning_rate": 4.2481203007518795e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.625,
      "logps/chosen": -204.0,
      "logps/rejected": -296.0,
      "loss": 0.8366,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.75390625,
      "rewards/margins": 0.98828125,
      "rewards/rejected": -1.7421875,
      "step": 1130
    },
    {
      "epoch": 0.08577232713866527,
      "grad_norm": 15.552984683141144,
      "learning_rate": 4.285714285714285e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.7109375,
      "logps/chosen": -196.0,
      "logps/rejected": -310.0,
      "loss": 0.8276,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.78515625,
      "rewards/margins": 1.046875,
      "rewards/rejected": -1.828125,
      "step": 1140
    },
    {
      "epoch": 0.08652471597321496,
      "grad_norm": 13.459117752253153,
      "learning_rate": 4.3233082706766913e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.59375,
      "logps/chosen": -172.0,
      "logps/rejected": -268.0,
      "loss": 0.8128,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.58984375,
      "rewards/margins": 0.98046875,
      "rewards/rejected": -1.5703125,
      "step": 1150
    },
    {
      "epoch": 0.08727710480776465,
      "grad_norm": 19.52274351726721,
      "learning_rate": 4.3609022556390975e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -1.875,
      "logps/chosen": -215.0,
      "logps/rejected": -302.0,
      "loss": 0.8437,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.80078125,
      "rewards/margins": 0.96484375,
      "rewards/rejected": -1.765625,
      "step": 1160
    },
    {
      "epoch": 0.08802949364231435,
      "grad_norm": 15.99140916600951,
      "learning_rate": 4.398496240601504e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.6875,
      "logps/chosen": -192.0,
      "logps/rejected": -276.0,
      "loss": 0.8598,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.80859375,
      "rewards/margins": 0.90234375,
      "rewards/rejected": -1.7109375,
      "step": 1170
    },
    {
      "epoch": 0.08878188247686404,
      "grad_norm": 14.893133334482847,
      "learning_rate": 4.4360902255639094e-07,
      "logits/chosen": -1.8828125,
      "logits/rejected": -1.6875,
      "logps/chosen": -196.0,
      "logps/rejected": -274.0,
      "loss": 0.8431,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.76953125,
      "rewards/margins": 0.8515625,
      "rewards/rejected": -1.625,
      "step": 1180
    },
    {
      "epoch": 0.08953427131141374,
      "grad_norm": 14.205367397216081,
      "learning_rate": 4.4736842105263156e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -1.9375,
      "logps/chosen": -196.0,
      "logps/rejected": -278.0,
      "loss": 0.8002,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.6796875,
      "rewards/margins": 1.0546875,
      "rewards/rejected": -1.734375,
      "step": 1190
    },
    {
      "epoch": 0.09028666014596344,
      "grad_norm": 19.154761844020502,
      "learning_rate": 4.5112781954887213e-07,
      "logits/chosen": -2.25,
      "logits/rejected": -1.9375,
      "logps/chosen": -195.0,
      "logps/rejected": -318.0,
      "loss": 0.8213,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.67578125,
      "rewards/margins": 1.140625,
      "rewards/rejected": -1.8203125,
      "step": 1200
    },
    {
      "epoch": 0.09103904898051313,
      "grad_norm": 16.5607930260631,
      "learning_rate": 4.5488721804511275e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -1.609375,
      "logps/chosen": -189.0,
      "logps/rejected": -308.0,
      "loss": 0.8089,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.6796875,
      "rewards/margins": 1.1875,
      "rewards/rejected": -1.8671875,
      "step": 1210
    },
    {
      "epoch": 0.09179143781506283,
      "grad_norm": 21.53670183419632,
      "learning_rate": 4.586466165413533e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.6328125,
      "logps/chosen": -186.0,
      "logps/rejected": -284.0,
      "loss": 0.8076,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.73828125,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -1.84375,
      "step": 1220
    },
    {
      "epoch": 0.09254382664961253,
      "grad_norm": 15.647557325663495,
      "learning_rate": 4.62406015037594e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.8359375,
      "logps/chosen": -205.0,
      "logps/rejected": -316.0,
      "loss": 0.8317,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.7578125,
      "rewards/margins": 1.125,
      "rewards/rejected": -1.8828125,
      "step": 1230
    },
    {
      "epoch": 0.09329621548416221,
      "grad_norm": 15.22172048148108,
      "learning_rate": 4.6616541353383456e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -1.9140625,
      "logps/chosen": -189.0,
      "logps/rejected": -292.0,
      "loss": 0.821,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.703125,
      "rewards/margins": 1.09375,
      "rewards/rejected": -1.7890625,
      "step": 1240
    },
    {
      "epoch": 0.09404860431871191,
      "grad_norm": 16.57451542944964,
      "learning_rate": 4.699248120300752e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -2.015625,
      "logps/chosen": -216.0,
      "logps/rejected": -292.0,
      "loss": 0.7934,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.6796875,
      "rewards/margins": 1.046875,
      "rewards/rejected": -1.7265625,
      "step": 1250
    },
    {
      "epoch": 0.09480099315326161,
      "grad_norm": 19.650533441252957,
      "learning_rate": 4.7368421052631574e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -1.953125,
      "logps/chosen": -193.0,
      "logps/rejected": -306.0,
      "loss": 0.8227,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.703125,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -1.9609375,
      "step": 1260
    },
    {
      "epoch": 0.0955533819878113,
      "grad_norm": 17.319369678698457,
      "learning_rate": 4.774436090225564e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -1.921875,
      "logps/chosen": -198.0,
      "logps/rejected": -346.0,
      "loss": 0.7551,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.79296875,
      "rewards/margins": 1.5,
      "rewards/rejected": -2.296875,
      "step": 1270
    },
    {
      "epoch": 0.096305770822361,
      "grad_norm": 15.864468918283583,
      "learning_rate": 4.812030075187969e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.890625,
      "logps/chosen": -191.0,
      "logps/rejected": -290.0,
      "loss": 0.8036,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.640625,
      "rewards/margins": 1.078125,
      "rewards/rejected": -1.7109375,
      "step": 1280
    },
    {
      "epoch": 0.0970581596569107,
      "grad_norm": 16.333285915031187,
      "learning_rate": 4.849624060150376e-07,
      "logits/chosen": -2.046875,
      "logits/rejected": -1.8359375,
      "logps/chosen": -195.0,
      "logps/rejected": -294.0,
      "loss": 0.7943,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.7109375,
      "rewards/margins": 1.21875,
      "rewards/rejected": -1.921875,
      "step": 1290
    },
    {
      "epoch": 0.09781054849146038,
      "grad_norm": 16.307381487800605,
      "learning_rate": 4.887218045112782e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.984375,
      "logps/chosen": -222.0,
      "logps/rejected": -328.0,
      "loss": 0.8063,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7890625,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -2.046875,
      "step": 1300
    },
    {
      "epoch": 0.09856293732601008,
      "grad_norm": 19.17029854466658,
      "learning_rate": 4.924812030075187e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.765625,
      "logps/chosen": -219.0,
      "logps/rejected": -302.0,
      "loss": 0.781,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.78515625,
      "rewards/margins": 1.109375,
      "rewards/rejected": -1.890625,
      "step": 1310
    },
    {
      "epoch": 0.09931532616055978,
      "grad_norm": 14.990309696860685,
      "learning_rate": 4.962406015037593e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -1.90625,
      "logps/chosen": -198.0,
      "logps/rejected": -288.0,
      "loss": 0.7915,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.79296875,
      "rewards/margins": 1.109375,
      "rewards/rejected": -1.8984375,
      "step": 1320
    },
    {
      "epoch": 0.10006771499510947,
      "grad_norm": 18.372579559117995,
      "learning_rate": 5e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.9140625,
      "logps/chosen": -180.0,
      "logps/rejected": -308.0,
      "loss": 0.7714,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.66015625,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -1.8671875,
      "step": 1330
    },
    {
      "epoch": 0.10082010382965917,
      "grad_norm": 16.1481309589053,
      "learning_rate": 4.999991376679495e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.8828125,
      "logps/chosen": -200.0,
      "logps/rejected": -334.0,
      "loss": 0.7482,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.73828125,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -2.109375,
      "step": 1340
    },
    {
      "epoch": 0.10157249266420887,
      "grad_norm": 21.102415835962333,
      "learning_rate": 4.999965506777466e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -1.90625,
      "logps/chosen": -216.0,
      "logps/rejected": -428.0,
      "loss": 0.7775,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.90234375,
      "rewards/margins": 2.0,
      "rewards/rejected": -2.90625,
      "step": 1350
    },
    {
      "epoch": 0.10232488149875855,
      "grad_norm": 15.959115626790151,
      "learning_rate": 4.999922390472384e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.953125,
      "logps/chosen": -207.0,
      "logps/rejected": -326.0,
      "loss": 0.773,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.7578125,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -2.0625,
      "step": 1360
    },
    {
      "epoch": 0.10307727033330825,
      "grad_norm": 15.634163728943385,
      "learning_rate": 4.999862028061692e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -1.828125,
      "logps/chosen": -209.0,
      "logps/rejected": -324.0,
      "loss": 0.7861,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.8359375,
      "rewards/margins": 1.234375,
      "rewards/rejected": -2.078125,
      "step": 1370
    },
    {
      "epoch": 0.10382965916785795,
      "grad_norm": 15.353863659650582,
      "learning_rate": 4.99978441996181e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -1.796875,
      "logps/chosen": -208.0,
      "logps/rejected": -342.0,
      "loss": 0.791,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.76953125,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -2.21875,
      "step": 1380
    },
    {
      "epoch": 0.10458204800240764,
      "grad_norm": 13.145206694307161,
      "learning_rate": 4.999689566708128e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -1.921875,
      "logps/chosen": -203.0,
      "logps/rejected": -320.0,
      "loss": 0.7863,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.77734375,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -2.0625,
      "step": 1390
    },
    {
      "epoch": 0.10533443683695734,
      "grad_norm": 16.609231616039995,
      "learning_rate": 4.999577468955008e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -1.8828125,
      "logps/chosen": -216.0,
      "logps/rejected": -308.0,
      "loss": 0.8025,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.87890625,
      "rewards/margins": 1.078125,
      "rewards/rejected": -1.953125,
      "step": 1400
    },
    {
      "epoch": 0.10608682567150704,
      "grad_norm": 13.42543477194773,
      "learning_rate": 4.999448127475773e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -1.8125,
      "logps/chosen": -204.0,
      "logps/rejected": -362.0,
      "loss": 0.759,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.8515625,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -2.53125,
      "step": 1410
    },
    {
      "epoch": 0.10683921450605673,
      "grad_norm": 20.327993271277418,
      "learning_rate": 4.999301543162706e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -1.875,
      "logps/chosen": -201.0,
      "logps/rejected": -336.0,
      "loss": 0.7374,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.73828125,
      "rewards/margins": 1.234375,
      "rewards/rejected": -1.9765625,
      "step": 1420
    },
    {
      "epoch": 0.10759160334060643,
      "grad_norm": 19.335577841018864,
      "learning_rate": 4.999137717027041e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.8046875,
      "logps/chosen": -188.0,
      "logps/rejected": -310.0,
      "loss": 0.749,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.79296875,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -1.984375,
      "step": 1430
    },
    {
      "epoch": 0.10834399217515613,
      "grad_norm": 26.673824134076316,
      "learning_rate": 4.998956650198959e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -1.890625,
      "logps/chosen": -195.0,
      "logps/rejected": -332.0,
      "loss": 0.7628,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.8046875,
      "rewards/margins": 1.390625,
      "rewards/rejected": -2.1875,
      "step": 1440
    },
    {
      "epoch": 0.10909638100970581,
      "grad_norm": 18.78422412831379,
      "learning_rate": 4.998758343927576e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -2.046875,
      "logps/chosen": -218.0,
      "logps/rejected": -332.0,
      "loss": 0.7159,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.9140625,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -2.125,
      "step": 1450
    },
    {
      "epoch": 0.10984876984425551,
      "grad_norm": 18.79182364606701,
      "learning_rate": 4.998542799580941e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -1.90625,
      "logps/chosen": -206.0,
      "logps/rejected": -326.0,
      "loss": 0.7463,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.80859375,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -2.1875,
      "step": 1460
    },
    {
      "epoch": 0.11060115867880521,
      "grad_norm": 22.813407084991223,
      "learning_rate": 4.998310018646021e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -2.03125,
      "logps/chosen": -206.0,
      "logps/rejected": -308.0,
      "loss": 0.7774,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.83984375,
      "rewards/margins": 1.203125,
      "rewards/rejected": -2.046875,
      "step": 1470
    },
    {
      "epoch": 0.1113535475133549,
      "grad_norm": 17.261823764752794,
      "learning_rate": 4.998060002728689e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -1.984375,
      "logps/chosen": -228.0,
      "logps/rejected": -348.0,
      "loss": 0.7258,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.8828125,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -2.28125,
      "step": 1480
    },
    {
      "epoch": 0.1121059363479046,
      "grad_norm": 16.935507339618148,
      "learning_rate": 4.99779275355372e-07,
      "logits/chosen": -2.25,
      "logits/rejected": -1.9375,
      "logps/chosen": -218.0,
      "logps/rejected": -358.0,
      "loss": 0.7636,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.79296875,
      "rewards/margins": 1.296875,
      "rewards/rejected": -2.09375,
      "step": 1490
    },
    {
      "epoch": 0.1128583251824543,
      "grad_norm": 19.99837834502528,
      "learning_rate": 4.997508272964775e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -1.7890625,
      "logps/chosen": -220.0,
      "logps/rejected": -366.0,
      "loss": 0.7268,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.88671875,
      "rewards/margins": 1.5,
      "rewards/rejected": -2.390625,
      "step": 1500
    },
    {
      "epoch": 0.11361071401700398,
      "grad_norm": 16.213868438747177,
      "learning_rate": 4.997206562924387e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -1.90625,
      "logps/chosen": -218.0,
      "logps/rejected": -338.0,
      "loss": 0.7137,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.81640625,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -2.171875,
      "step": 1510
    },
    {
      "epoch": 0.11436310285155368,
      "grad_norm": 17.046302395996115,
      "learning_rate": 4.99688762551395e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -1.8984375,
      "logps/chosen": -208.0,
      "logps/rejected": -318.0,
      "loss": 0.7397,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.90234375,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -2.171875,
      "step": 1520
    },
    {
      "epoch": 0.11511549168610338,
      "grad_norm": 19.91521093173485,
      "learning_rate": 4.996551462933705e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -1.765625,
      "logps/chosen": -224.0,
      "logps/rejected": -350.0,
      "loss": 0.7337,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.93359375,
      "rewards/margins": 1.40625,
      "rewards/rejected": -2.34375,
      "step": 1530
    },
    {
      "epoch": 0.11586788052065307,
      "grad_norm": 19.409758237021027,
      "learning_rate": 4.996198077502719e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -1.921875,
      "logps/chosen": -192.0,
      "logps/rejected": -316.0,
      "loss": 0.7251,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.75,
      "rewards/margins": 1.375,
      "rewards/rejected": -2.125,
      "step": 1540
    },
    {
      "epoch": 0.11662026935520277,
      "grad_norm": 14.00389135523563,
      "learning_rate": 4.99582747165888e-07,
      "logits/chosen": -2.25,
      "logits/rejected": -1.9453125,
      "logps/chosen": -215.0,
      "logps/rejected": -388.0,
      "loss": 0.715,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.91015625,
      "rewards/margins": 1.46875,
      "rewards/rejected": -2.375,
      "step": 1550
    },
    {
      "epoch": 0.11737265818975247,
      "grad_norm": 16.187186842782996,
      "learning_rate": 4.995439647958869e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.125,
      "logps/chosen": -220.0,
      "logps/rejected": -368.0,
      "loss": 0.6864,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.89453125,
      "rewards/margins": 1.546875,
      "rewards/rejected": -2.4375,
      "step": 1560
    },
    {
      "epoch": 0.11812504702430215,
      "grad_norm": 17.665692767876962,
      "learning_rate": 4.995034609078148e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.140625,
      "logps/chosen": -206.0,
      "logps/rejected": -360.0,
      "loss": 0.7253,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.88671875,
      "rewards/margins": 1.546875,
      "rewards/rejected": -2.4375,
      "step": 1570
    },
    {
      "epoch": 0.11887743585885185,
      "grad_norm": 16.355107188683863,
      "learning_rate": 4.994612357810942e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.203125,
      "logps/chosen": -252.0,
      "logps/rejected": -360.0,
      "loss": 0.6852,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.9765625,
      "rewards/margins": 1.4375,
      "rewards/rejected": -2.40625,
      "step": 1580
    },
    {
      "epoch": 0.11962982469340155,
      "grad_norm": 17.883786326206174,
      "learning_rate": 4.994172897070217e-07,
      "logits/chosen": -2.046875,
      "logits/rejected": -1.8125,
      "logps/chosen": -219.0,
      "logps/rejected": -376.0,
      "loss": 0.74,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.8671875,
      "rewards/margins": 1.640625,
      "rewards/rejected": -2.5,
      "step": 1590
    },
    {
      "epoch": 0.12038221352795124,
      "grad_norm": 17.8562980929471,
      "learning_rate": 4.99371622988766e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -2.0,
      "logps/chosen": -206.0,
      "logps/rejected": -344.0,
      "loss": 0.7079,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.9296875,
      "rewards/margins": 1.34375,
      "rewards/rejected": -2.28125,
      "step": 1600
    },
    {
      "epoch": 0.12113460236250094,
      "grad_norm": 18.79833682157474,
      "learning_rate": 4.993242359413664e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.09375,
      "logps/chosen": -217.0,
      "logps/rejected": -396.0,
      "loss": 0.7194,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.96484375,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -2.6875,
      "step": 1610
    },
    {
      "epoch": 0.12188699119705064,
      "grad_norm": 14.187291777751348,
      "learning_rate": 4.992751288917297e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.109375,
      "logps/chosen": -196.0,
      "logps/rejected": -352.0,
      "loss": 0.6876,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.75390625,
      "rewards/margins": 1.640625,
      "rewards/rejected": -2.390625,
      "step": 1620
    },
    {
      "epoch": 0.12263938003160033,
      "grad_norm": 13.677652735737842,
      "learning_rate": 4.992243021786285e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.109375,
      "logps/chosen": -204.0,
      "logps/rejected": -330.0,
      "loss": 0.6897,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.89453125,
      "rewards/margins": 1.21875,
      "rewards/rejected": -2.109375,
      "step": 1630
    },
    {
      "epoch": 0.12339176886615003,
      "grad_norm": 11.354806812856847,
      "learning_rate": 4.99171756152699e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -1.9921875,
      "logps/chosen": -242.0,
      "logps/rejected": -412.0,
      "loss": 0.6937,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.921875,
      "rewards/margins": 1.8671875,
      "rewards/rejected": -2.78125,
      "step": 1640
    },
    {
      "epoch": 0.12414415770069973,
      "grad_norm": 12.805555480706843,
      "learning_rate": 4.991174911764381e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.890625,
      "logps/chosen": -221.0,
      "logps/rejected": -360.0,
      "loss": 0.6998,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.9375,
      "rewards/margins": 1.359375,
      "rewards/rejected": -2.296875,
      "step": 1650
    },
    {
      "epoch": 0.12489654653524941,
      "grad_norm": 15.346865373908255,
      "learning_rate": 4.990615076242011e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.125,
      "logps/chosen": -233.0,
      "logps/rejected": -436.0,
      "loss": 0.7085,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8984375,
      "rewards/margins": 2.015625,
      "rewards/rejected": -2.90625,
      "step": 1660
    },
    {
      "epoch": 0.12564893536979913,
      "grad_norm": 16.30188472768267,
      "learning_rate": 4.990038058821995e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.046875,
      "logps/chosen": -213.0,
      "logps/rejected": -342.0,
      "loss": 0.683,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.94140625,
      "rewards/margins": 1.359375,
      "rewards/rejected": -2.296875,
      "step": 1670
    },
    {
      "epoch": 0.1264013242043488,
      "grad_norm": 17.280703886833347,
      "learning_rate": 4.989443863484976e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.125,
      "logps/chosen": -210.0,
      "logps/rejected": -342.0,
      "loss": 0.6685,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8984375,
      "rewards/margins": 1.453125,
      "rewards/rejected": -2.359375,
      "step": 1680
    },
    {
      "epoch": 0.1271537130388985,
      "grad_norm": 23.170708632461462,
      "learning_rate": 4.988832494330106e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -1.953125,
      "logps/chosen": -235.0,
      "logps/rejected": -392.0,
      "loss": 0.7086,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.078125,
      "rewards/margins": 1.390625,
      "rewards/rejected": -2.46875,
      "step": 1690
    },
    {
      "epoch": 0.1279061018734482,
      "grad_norm": 18.394127366286423,
      "learning_rate": 4.988203955575006e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -1.984375,
      "logps/chosen": -217.0,
      "logps/rejected": -396.0,
      "loss": 0.6603,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.84375,
      "rewards/margins": 1.8046875,
      "rewards/rejected": -2.65625,
      "step": 1700
    },
    {
      "epoch": 0.1286584907079979,
      "grad_norm": 19.796842740728817,
      "learning_rate": 4.987558251555755e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -2.109375,
      "logps/chosen": -214.0,
      "logps/rejected": -356.0,
      "loss": 0.6549,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.875,
      "rewards/margins": 1.4921875,
      "rewards/rejected": -2.375,
      "step": 1710
    },
    {
      "epoch": 0.1294108795425476,
      "grad_norm": 15.2812867213198,
      "learning_rate": 4.98689538672684e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -1.9453125,
      "logps/chosen": -218.0,
      "logps/rejected": -406.0,
      "loss": 0.6971,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.90234375,
      "rewards/margins": 2.109375,
      "rewards/rejected": -3.015625,
      "step": 1720
    },
    {
      "epoch": 0.1301632683770973,
      "grad_norm": 18.80087393302081,
      "learning_rate": 4.986215365661137e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -2.140625,
      "logps/chosen": -218.0,
      "logps/rejected": -386.0,
      "loss": 0.6496,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.92578125,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -2.5625,
      "step": 1730
    },
    {
      "epoch": 0.13091565721164697,
      "grad_norm": 17.254208781885957,
      "learning_rate": 4.985518193049879e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -1.9921875,
      "logps/chosen": -243.0,
      "logps/rejected": -372.0,
      "loss": 0.7074,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.97265625,
      "rewards/margins": 1.5,
      "rewards/rejected": -2.46875,
      "step": 1740
    },
    {
      "epoch": 0.13166804604619667,
      "grad_norm": 18.62600690542437,
      "learning_rate": 4.984803873702619e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.9921875,
      "logps/chosen": -236.0,
      "logps/rejected": -400.0,
      "loss": 0.7249,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.0703125,
      "rewards/margins": 1.6875,
      "rewards/rejected": -2.75,
      "step": 1750
    },
    {
      "epoch": 0.13242043488074637,
      "grad_norm": 13.72770738787324,
      "learning_rate": 4.984072412547202e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.9453125,
      "logps/chosen": -196.0,
      "logps/rejected": -366.0,
      "loss": 0.6907,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.83984375,
      "rewards/margins": 1.6796875,
      "rewards/rejected": -2.515625,
      "step": 1760
    },
    {
      "epoch": 0.13317282371529607,
      "grad_norm": 20.620906434719245,
      "learning_rate": 4.983323814629727e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.8515625,
      "logps/chosen": -220.0,
      "logps/rejected": -362.0,
      "loss": 0.728,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.953125,
      "rewards/margins": 1.4375,
      "rewards/rejected": -2.390625,
      "step": 1770
    },
    {
      "epoch": 0.13392521254984577,
      "grad_norm": 18.25104054990024,
      "learning_rate": 4.982558085114515e-07,
      "logits/chosen": -2.046875,
      "logits/rejected": -1.9765625,
      "logps/chosen": -231.0,
      "logps/rejected": -370.0,
      "loss": 0.7104,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.109375,
      "rewards/margins": 1.4375,
      "rewards/rejected": -2.546875,
      "step": 1780
    },
    {
      "epoch": 0.13467760138439547,
      "grad_norm": 25.07388247817432,
      "learning_rate": 4.981775229284068e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.9765625,
      "logps/chosen": -232.0,
      "logps/rejected": -380.0,
      "loss": 0.7268,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.03125,
      "rewards/margins": 1.5625,
      "rewards/rejected": -2.59375,
      "step": 1790
    },
    {
      "epoch": 0.13542999021894514,
      "grad_norm": 19.375898557242053,
      "learning_rate": 4.98097525253904e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.9453125,
      "logps/chosen": -224.0,
      "logps/rejected": -382.0,
      "loss": 0.7172,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.93359375,
      "rewards/margins": 1.65625,
      "rewards/rejected": -2.59375,
      "step": 1800
    },
    {
      "epoch": 0.13618237905349484,
      "grad_norm": 15.729867570005627,
      "learning_rate": 4.980158160398198e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -1.9609375,
      "logps/chosen": -209.0,
      "logps/rejected": -348.0,
      "loss": 0.6744,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.90625,
      "rewards/margins": 1.5078125,
      "rewards/rejected": -2.40625,
      "step": 1810
    },
    {
      "epoch": 0.13693476788804454,
      "grad_norm": 19.688743905890647,
      "learning_rate": 4.979323958498378e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.0625,
      "logps/chosen": -233.0,
      "logps/rejected": -380.0,
      "loss": 0.6778,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.0390625,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -2.859375,
      "step": 1820
    },
    {
      "epoch": 0.13768715672259424,
      "grad_norm": 14.606763103961597,
      "learning_rate": 4.978472652594453e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -1.984375,
      "logps/chosen": -218.0,
      "logps/rejected": -404.0,
      "loss": 0.6831,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.0703125,
      "rewards/margins": 1.859375,
      "rewards/rejected": -2.9375,
      "step": 1830
    },
    {
      "epoch": 0.13843954555714394,
      "grad_norm": 16.93878587897678,
      "learning_rate": 4.977604248559289e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -1.9609375,
      "logps/chosen": -215.0,
      "logps/rejected": -394.0,
      "loss": 0.6724,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.046875,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -2.8125,
      "step": 1840
    },
    {
      "epoch": 0.13919193439169364,
      "grad_norm": 24.41029323825072,
      "learning_rate": 4.976718752383709e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.0,
      "logps/chosen": -214.0,
      "logps/rejected": -390.0,
      "loss": 0.6733,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.96875,
      "rewards/margins": 1.765625,
      "rewards/rejected": -2.734375,
      "step": 1850
    },
    {
      "epoch": 0.1399443232262433,
      "grad_norm": 16.556214248268354,
      "learning_rate": 4.975816170176445e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -1.9609375,
      "logps/chosen": -234.0,
      "logps/rejected": -392.0,
      "loss": 0.6629,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.99609375,
      "rewards/margins": 1.71875,
      "rewards/rejected": -2.703125,
      "step": 1860
    },
    {
      "epoch": 0.140696712060793,
      "grad_norm": 22.14965658239816,
      "learning_rate": 4.974896508164101e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -1.90625,
      "logps/chosen": -209.0,
      "logps/rejected": -352.0,
      "loss": 0.6681,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.90625,
      "rewards/margins": 1.515625,
      "rewards/rejected": -2.421875,
      "step": 1870
    },
    {
      "epoch": 0.1414491008953427,
      "grad_norm": 16.182053739597375,
      "learning_rate": 4.973959772691112e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.734375,
      "logps/chosen": -216.0,
      "logps/rejected": -434.0,
      "loss": 0.6847,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.0078125,
      "rewards/margins": 2.015625,
      "rewards/rejected": -3.015625,
      "step": 1880
    },
    {
      "epoch": 0.1422014897298924,
      "grad_norm": 16.826255040456353,
      "learning_rate": 4.973005970219692e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.9609375,
      "logps/chosen": -241.0,
      "logps/rejected": -390.0,
      "loss": 0.6468,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0703125,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -2.796875,
      "step": 1890
    },
    {
      "epoch": 0.1429538785644421,
      "grad_norm": 17.889692882027187,
      "learning_rate": 4.972035107329796e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -1.9375,
      "logps/chosen": -244.0,
      "logps/rejected": -400.0,
      "loss": 0.6467,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1015625,
      "rewards/margins": 1.7265625,
      "rewards/rejected": -2.828125,
      "step": 1900
    },
    {
      "epoch": 0.1437062673989918,
      "grad_norm": 21.563989859911914,
      "learning_rate": 4.971047190719076e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.75,
      "logps/chosen": -196.0,
      "logps/rejected": -408.0,
      "loss": 0.6149,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.9453125,
      "rewards/margins": 1.75,
      "rewards/rejected": -2.703125,
      "step": 1910
    },
    {
      "epoch": 0.14445865623354148,
      "grad_norm": 83.62025132995463,
      "learning_rate": 4.970042227202828e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.0,
      "logps/chosen": -233.0,
      "logps/rejected": -434.0,
      "loss": 0.6625,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.234375,
      "rewards/margins": 1.9453125,
      "rewards/rejected": -3.1875,
      "step": 1920
    },
    {
      "epoch": 0.14521104506809118,
      "grad_norm": 21.27362515873193,
      "learning_rate": 4.969020223713948e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.8046875,
      "logps/chosen": -240.0,
      "logps/rejected": -434.0,
      "loss": 0.6598,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.1328125,
      "rewards/margins": 2.078125,
      "rewards/rejected": -3.203125,
      "step": 1930
    },
    {
      "epoch": 0.14596343390264088,
      "grad_norm": 22.997222157767474,
      "learning_rate": 4.967981187302889e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.109375,
      "logps/chosen": -244.0,
      "logps/rejected": -434.0,
      "loss": 0.6331,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.171875,
      "rewards/margins": 1.90625,
      "rewards/rejected": -3.078125,
      "step": 1940
    },
    {
      "epoch": 0.14671582273719058,
      "grad_norm": 16.661723870130817,
      "learning_rate": 4.966925125137607e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -1.8359375,
      "logps/chosen": -234.0,
      "logps/rejected": -412.0,
      "loss": 0.629,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.1484375,
      "rewards/margins": 1.8359375,
      "rewards/rejected": -2.984375,
      "step": 1950
    },
    {
      "epoch": 0.14746821157174028,
      "grad_norm": 23.190830565246937,
      "learning_rate": 4.965852044503512e-07,
      "logits/chosen": -1.9296875,
      "logits/rejected": -1.8828125,
      "logps/chosen": -219.0,
      "logps/rejected": -424.0,
      "loss": 0.6485,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.921875,
      "rewards/margins": 1.859375,
      "rewards/rejected": -2.78125,
      "step": 1960
    },
    {
      "epoch": 0.14822060040628998,
      "grad_norm": 17.52221454393284,
      "learning_rate": 4.964761952803416e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.921875,
      "logps/chosen": -236.0,
      "logps/rejected": -410.0,
      "loss": 0.691,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.2265625,
      "rewards/margins": 1.6875,
      "rewards/rejected": -2.90625,
      "step": 1970
    },
    {
      "epoch": 0.14897298924083965,
      "grad_norm": 21.062878758362963,
      "learning_rate": 4.963654857557488e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.890625,
      "logps/chosen": -224.0,
      "logps/rejected": -376.0,
      "loss": 0.6835,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.0078125,
      "rewards/margins": 1.6640625,
      "rewards/rejected": -2.671875,
      "step": 1980
    },
    {
      "epoch": 0.14972537807538935,
      "grad_norm": 11.91187961418117,
      "learning_rate": 4.962530766403199e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -1.9453125,
      "logps/chosen": -204.0,
      "logps/rejected": -396.0,
      "loss": 0.6324,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.91015625,
      "rewards/margins": 1.875,
      "rewards/rejected": -2.78125,
      "step": 1990
    },
    {
      "epoch": 0.15047776690993905,
      "grad_norm": 18.545202210499713,
      "learning_rate": 4.961389687095267e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -2.03125,
      "logps/chosen": -224.0,
      "logps/rejected": -384.0,
      "loss": 0.6343,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.98046875,
      "rewards/margins": 1.71875,
      "rewards/rejected": -2.703125,
      "step": 2000
    },
    {
      "epoch": 0.15047776690993905,
      "eval_logits/chosen": -2.109375,
      "eval_logits/rejected": -1.9609375,
      "eval_logps/chosen": -253.0,
      "eval_logps/rejected": -416.0,
      "eval_loss": 0.3413139283657074,
      "eval_rewards/accuracies": 0.8560426831245422,
      "eval_rewards/chosen": -1.2109375,
      "eval_rewards/margins": 1.78125,
      "eval_rewards/rejected": -2.984375,
      "eval_runtime": 2669.7803,
      "eval_samples_per_second": 35.394,
      "eval_steps_per_second": 0.553,
      "step": 2000
    },
    {
      "epoch": 0.15123015574448875,
      "grad_norm": 18.500865778378923,
      "learning_rate": 4.960231627505606e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.8359375,
      "logps/chosen": -246.0,
      "logps/rejected": -410.0,
      "loss": 0.6459,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.109375,
      "rewards/margins": 1.84375,
      "rewards/rejected": -2.953125,
      "step": 2010
    },
    {
      "epoch": 0.15198254457903845,
      "grad_norm": 19.425893564323463,
      "learning_rate": 4.959056595623271e-07,
      "logits/chosen": -1.9140625,
      "logits/rejected": -1.8515625,
      "logps/chosen": -253.0,
      "logps/rejected": -420.0,
      "loss": 0.6408,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.171875,
      "rewards/margins": 1.8359375,
      "rewards/rejected": -3.0,
      "step": 2020
    },
    {
      "epoch": 0.15273493341358815,
      "grad_norm": 16.414335557682115,
      "learning_rate": 4.957864599554404e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -1.578125,
      "logps/chosen": -224.0,
      "logps/rejected": -412.0,
      "loss": 0.6712,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.03125,
      "rewards/margins": 1.875,
      "rewards/rejected": -2.90625,
      "step": 2030
    },
    {
      "epoch": 0.15348732224813783,
      "grad_norm": 22.628673619514657,
      "learning_rate": 4.956655647522176e-07,
      "logits/chosen": -1.9296875,
      "logits/rejected": -1.8671875,
      "logps/chosen": -243.0,
      "logps/rejected": -420.0,
      "loss": 0.6004,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.0546875,
      "rewards/margins": 1.9921875,
      "rewards/rejected": -3.046875,
      "step": 2040
    },
    {
      "epoch": 0.15423971108268753,
      "grad_norm": 19.091247557214192,
      "learning_rate": 4.95542974786673e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.671875,
      "logps/chosen": -258.0,
      "logps/rejected": -416.0,
      "loss": 0.6111,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.28125,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -2.984375,
      "step": 2050
    },
    {
      "epoch": 0.15499209991723722,
      "grad_norm": 29.476674871286058,
      "learning_rate": 4.954186909045129e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -2.015625,
      "logps/chosen": -230.0,
      "logps/rejected": -400.0,
      "loss": 0.6631,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.0703125,
      "rewards/margins": 1.8125,
      "rewards/rejected": -2.890625,
      "step": 2060
    },
    {
      "epoch": 0.15574448875178692,
      "grad_norm": 15.801616733420543,
      "learning_rate": 4.95292713963129e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.046875,
      "logps/chosen": -226.0,
      "logps/rejected": -382.0,
      "loss": 0.6674,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.078125,
      "rewards/margins": 1.6953125,
      "rewards/rejected": -2.78125,
      "step": 2070
    },
    {
      "epoch": 0.15649687758633662,
      "grad_norm": 15.735575130067268,
      "learning_rate": 4.951650448315929e-07,
      "logits/chosen": -2.0,
      "logits/rejected": -1.8828125,
      "logps/chosen": -239.0,
      "logps/rejected": -388.0,
      "loss": 0.6233,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.1171875,
      "rewards/margins": 1.609375,
      "rewards/rejected": -2.734375,
      "step": 2080
    },
    {
      "epoch": 0.15724926642088632,
      "grad_norm": 12.797237972983407,
      "learning_rate": 4.950356843906501e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.0,
      "logps/chosen": -239.0,
      "logps/rejected": -416.0,
      "loss": 0.6717,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.21875,
      "rewards/margins": 1.78125,
      "rewards/rejected": -3.0,
      "step": 2090
    },
    {
      "epoch": 0.158001655255436,
      "grad_norm": 19.52640854559428,
      "learning_rate": 4.949046335327138e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -2.015625,
      "logps/chosen": -239.0,
      "logps/rejected": -430.0,
      "loss": 0.5828,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.078125,
      "rewards/margins": 2.0,
      "rewards/rejected": -3.09375,
      "step": 2100
    },
    {
      "epoch": 0.1587540440899857,
      "grad_norm": 15.19760254746959,
      "learning_rate": 4.94771893161859e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.828125,
      "logps/chosen": -240.0,
      "logps/rejected": -422.0,
      "loss": 0.6636,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.1171875,
      "rewards/margins": 1.921875,
      "rewards/rejected": -3.046875,
      "step": 2110
    },
    {
      "epoch": 0.1595064329245354,
      "grad_norm": 18.474204557560707,
      "learning_rate": 4.946374641938157e-07,
      "logits/chosen": -1.984375,
      "logits/rejected": -2.03125,
      "logps/chosen": -233.0,
      "logps/rejected": -384.0,
      "loss": 0.6513,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.1484375,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -2.734375,
      "step": 2120
    },
    {
      "epoch": 0.1602588217590851,
      "grad_norm": 17.55904381856254,
      "learning_rate": 4.945013475559632e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -2.015625,
      "logps/chosen": -253.0,
      "logps/rejected": -402.0,
      "loss": 0.6152,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1328125,
      "rewards/margins": 1.640625,
      "rewards/rejected": -2.765625,
      "step": 2130
    },
    {
      "epoch": 0.1610112105936348,
      "grad_norm": 21.606529007106413,
      "learning_rate": 4.943635441873235e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.921875,
      "logps/chosen": -237.0,
      "logps/rejected": -426.0,
      "loss": 0.6404,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.140625,
      "rewards/margins": 1.9453125,
      "rewards/rejected": -3.078125,
      "step": 2140
    },
    {
      "epoch": 0.1617635994281845,
      "grad_norm": 17.95977014546446,
      "learning_rate": 4.942240550385547e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -1.9765625,
      "logps/chosen": -232.0,
      "logps/rejected": -444.0,
      "loss": 0.5838,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.1328125,
      "rewards/margins": 2.109375,
      "rewards/rejected": -3.25,
      "step": 2150
    },
    {
      "epoch": 0.16251598826273417,
      "grad_norm": 21.84882361396256,
      "learning_rate": 4.940828810719444e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -1.953125,
      "logps/chosen": -226.0,
      "logps/rejected": -410.0,
      "loss": 0.6124,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.046875,
      "rewards/margins": 1.9140625,
      "rewards/rejected": -2.96875,
      "step": 2160
    },
    {
      "epoch": 0.16326837709728387,
      "grad_norm": 15.242777971436194,
      "learning_rate": 4.939400232614033e-07,
      "logits/chosen": -1.9140625,
      "logits/rejected": -1.9453125,
      "logps/chosen": -241.0,
      "logps/rejected": -528.0,
      "loss": 0.6696,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.1875,
      "rewards/margins": 2.40625,
      "rewards/rejected": -3.59375,
      "step": 2170
    },
    {
      "epoch": 0.16402076593183357,
      "grad_norm": 21.843895904473584,
      "learning_rate": 4.937954825924585e-07,
      "logits/chosen": -1.921875,
      "logits/rejected": -1.90625,
      "logps/chosen": -246.0,
      "logps/rejected": -390.0,
      "loss": 0.6206,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.28125,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -2.875,
      "step": 2180
    },
    {
      "epoch": 0.16477315476638327,
      "grad_norm": 17.231773230752005,
      "learning_rate": 4.936492600622464e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.921875,
      "logps/chosen": -260.0,
      "logps/rejected": -454.0,
      "loss": 0.6615,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.1796875,
      "rewards/margins": 2.203125,
      "rewards/rejected": -3.375,
      "step": 2190
    },
    {
      "epoch": 0.16552554360093297,
      "grad_norm": 16.604678065291914,
      "learning_rate": 4.935013566795058e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.890625,
      "logps/chosen": -228.0,
      "logps/rejected": -424.0,
      "loss": 0.5934,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.1484375,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -3.046875,
      "step": 2200
    },
    {
      "epoch": 0.16627793243548267,
      "grad_norm": 19.417492093191047,
      "learning_rate": 4.933517734645714e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -1.9296875,
      "logps/chosen": -209.0,
      "logps/rejected": -484.0,
      "loss": 0.6047,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.0546875,
      "rewards/margins": 2.546875,
      "rewards/rejected": -3.609375,
      "step": 2210
    },
    {
      "epoch": 0.16703032127003234,
      "grad_norm": 15.587567730538082,
      "learning_rate": 4.932005114493665e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -1.9375,
      "logps/chosen": -260.0,
      "logps/rejected": -492.0,
      "loss": 0.6043,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.359375,
      "rewards/margins": 2.359375,
      "rewards/rejected": -3.71875,
      "step": 2220
    },
    {
      "epoch": 0.16778271010458204,
      "grad_norm": 17.127811806923678,
      "learning_rate": 4.930475716773956e-07,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.9921875,
      "logps/chosen": -248.0,
      "logps/rejected": -452.0,
      "loss": 0.6239,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.2265625,
      "rewards/margins": 2.078125,
      "rewards/rejected": -3.3125,
      "step": 2230
    },
    {
      "epoch": 0.16853509893913174,
      "grad_norm": 15.59648731119962,
      "learning_rate": 4.928929552037378e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.9453125,
      "logps/chosen": -224.0,
      "logps/rejected": -396.0,
      "loss": 0.633,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.0625,
      "rewards/margins": 1.890625,
      "rewards/rejected": -2.953125,
      "step": 2240
    },
    {
      "epoch": 0.16928748777368144,
      "grad_norm": 17.64987216797727,
      "learning_rate": 4.927366630950389e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -2.078125,
      "logps/chosen": -220.0,
      "logps/rejected": -400.0,
      "loss": 0.5894,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.0078125,
      "rewards/margins": 1.9453125,
      "rewards/rejected": -2.953125,
      "step": 2250
    },
    {
      "epoch": 0.17003987660823114,
      "grad_norm": 18.998359427893455,
      "learning_rate": 4.925786964295046e-07,
      "logits/chosen": -1.9453125,
      "logits/rejected": -1.90625,
      "logps/chosen": -266.0,
      "logps/rejected": -440.0,
      "loss": 0.653,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.3828125,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -3.28125,
      "step": 2260
    },
    {
      "epoch": 0.17079226544278084,
      "grad_norm": 19.087053303607437,
      "learning_rate": 4.924190562968926e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.9453125,
      "logps/chosen": -264.0,
      "logps/rejected": -414.0,
      "loss": 0.6198,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3203125,
      "rewards/margins": 1.765625,
      "rewards/rejected": -3.078125,
      "step": 2270
    },
    {
      "epoch": 0.17154465427733054,
      "grad_norm": 17.495817983477522,
      "learning_rate": 4.922577437985052e-07,
      "logits/chosen": -2.046875,
      "logits/rejected": -1.8359375,
      "logps/chosen": -236.0,
      "logps/rejected": -420.0,
      "loss": 0.556,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.21875,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -3.03125,
      "step": 2280
    },
    {
      "epoch": 0.1722970431118802,
      "grad_norm": 18.375067862266864,
      "learning_rate": 4.920947600471821e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.015625,
      "logps/chosen": -239.0,
      "logps/rejected": -430.0,
      "loss": 0.601,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.21875,
      "rewards/margins": 1.953125,
      "rewards/rejected": -3.171875,
      "step": 2290
    },
    {
      "epoch": 0.1730494319464299,
      "grad_norm": 21.656375498567677,
      "learning_rate": 4.919301061672919e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -1.9921875,
      "logps/chosen": -244.0,
      "logps/rejected": -456.0,
      "loss": 0.6091,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.25,
      "rewards/margins": 2.140625,
      "rewards/rejected": -3.390625,
      "step": 2300
    },
    {
      "epoch": 0.1738018207809796,
      "grad_norm": 20.614196034562067,
      "learning_rate": 4.917637832947256e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -2.015625,
      "logps/chosen": -262.0,
      "logps/rejected": -444.0,
      "loss": 0.6283,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.2578125,
      "rewards/margins": 1.9453125,
      "rewards/rejected": -3.203125,
      "step": 2310
    },
    {
      "epoch": 0.1745542096155293,
      "grad_norm": 16.299509372405787,
      "learning_rate": 4.915957925768871e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -2.078125,
      "logps/chosen": -232.0,
      "logps/rejected": -424.0,
      "loss": 0.6035,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.25,
      "rewards/margins": 1.8828125,
      "rewards/rejected": -3.140625,
      "step": 2320
    },
    {
      "epoch": 0.175306598450079,
      "grad_norm": 23.198567813162285,
      "learning_rate": 4.914261351726868e-07,
      "logits/chosen": -1.9765625,
      "logits/rejected": -1.9453125,
      "logps/chosen": -262.0,
      "logps/rejected": -444.0,
      "loss": 0.6152,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.21875,
      "rewards/margins": 1.921875,
      "rewards/rejected": -3.140625,
      "step": 2330
    },
    {
      "epoch": 0.1760589872846287,
      "grad_norm": 15.239834297763418,
      "learning_rate": 4.91254812252533e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -1.734375,
      "logps/chosen": -244.0,
      "logps/rejected": -476.0,
      "loss": 0.5884,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.28125,
      "rewards/margins": 2.125,
      "rewards/rejected": -3.40625,
      "step": 2340
    },
    {
      "epoch": 0.17681137611917838,
      "grad_norm": 16.418336297068503,
      "learning_rate": 4.910818249983235e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.15625,
      "logps/chosen": -239.0,
      "logps/rejected": -442.0,
      "loss": 0.5876,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1796875,
      "rewards/margins": 2.0,
      "rewards/rejected": -3.1875,
      "step": 2350
    },
    {
      "epoch": 0.17756376495372808,
      "grad_norm": 19.466440581981193,
      "learning_rate": 4.909071746034379e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.015625,
      "logps/chosen": -268.0,
      "logps/rejected": -446.0,
      "loss": 0.6157,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.328125,
      "rewards/margins": 1.9296875,
      "rewards/rejected": -3.25,
      "step": 2360
    },
    {
      "epoch": 0.17831615378827778,
      "grad_norm": 16.982862861799973,
      "learning_rate": 4.907308622727293e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -1.875,
      "logps/chosen": -225.0,
      "logps/rejected": -444.0,
      "loss": 0.5933,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.109375,
      "rewards/margins": 2.234375,
      "rewards/rejected": -3.34375,
      "step": 2370
    },
    {
      "epoch": 0.17906854262282748,
      "grad_norm": 16.220644761292526,
      "learning_rate": 4.90552889222516e-07,
      "logits/chosen": -1.9375,
      "logits/rejected": -1.890625,
      "logps/chosen": -240.0,
      "logps/rejected": -474.0,
      "loss": 0.5675,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.046875,
      "rewards/margins": 2.234375,
      "rewards/rejected": -3.28125,
      "step": 2380
    },
    {
      "epoch": 0.17982093145737718,
      "grad_norm": 27.693509736403705,
      "learning_rate": 4.903732566805727e-07,
      "logits/chosen": -1.984375,
      "logits/rejected": -1.9453125,
      "logps/chosen": -268.0,
      "logps/rejected": -506.0,
      "loss": 0.6339,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.3984375,
      "rewards/margins": 2.4375,
      "rewards/rejected": -3.84375,
      "step": 2390
    },
    {
      "epoch": 0.18057332029192688,
      "grad_norm": 18.8185111440861,
      "learning_rate": 4.901919658861228e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.140625,
      "logps/chosen": -246.0,
      "logps/rejected": -400.0,
      "loss": 0.6429,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.15625,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -2.90625,
      "step": 2400
    },
    {
      "epoch": 0.18132570912647655,
      "grad_norm": 17.759673081777798,
      "learning_rate": 4.900090180898292e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -1.984375,
      "logps/chosen": -247.0,
      "logps/rejected": -430.0,
      "loss": 0.5932,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.171875,
      "rewards/margins": 1.9140625,
      "rewards/rejected": -3.078125,
      "step": 2410
    },
    {
      "epoch": 0.18207809796102625,
      "grad_norm": 13.815957012821045,
      "learning_rate": 4.898244145537857e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.0,
      "logps/chosen": -256.0,
      "logps/rejected": -442.0,
      "loss": 0.6068,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.3515625,
      "rewards/margins": 1.875,
      "rewards/rejected": -3.234375,
      "step": 2420
    },
    {
      "epoch": 0.18283048679557595,
      "grad_norm": 14.283477062432866,
      "learning_rate": 4.896381565515087e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -2.140625,
      "logps/chosen": -242.0,
      "logps/rejected": -408.0,
      "loss": 0.5578,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.1796875,
      "rewards/margins": 1.9140625,
      "rewards/rejected": -3.09375,
      "step": 2430
    },
    {
      "epoch": 0.18358287563012565,
      "grad_norm": 16.955738698704174,
      "learning_rate": 4.894502453679284e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -1.9453125,
      "logps/chosen": -237.0,
      "logps/rejected": -438.0,
      "loss": 0.5931,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.1015625,
      "rewards/margins": 2.125,
      "rewards/rejected": -3.234375,
      "step": 2440
    },
    {
      "epoch": 0.18433526446467535,
      "grad_norm": 24.1734142889167,
      "learning_rate": 4.892606822993793e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -2.0625,
      "logps/chosen": -239.0,
      "logps/rejected": -450.0,
      "loss": 0.628,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.171875,
      "rewards/margins": 2.0625,
      "rewards/rejected": -3.234375,
      "step": 2450
    },
    {
      "epoch": 0.18508765329922505,
      "grad_norm": 16.10350732531623,
      "learning_rate": 4.890694686535918e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.125,
      "logps/chosen": -268.0,
      "logps/rejected": -468.0,
      "loss": 0.5657,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.2578125,
      "rewards/margins": 2.15625,
      "rewards/rejected": -3.40625,
      "step": 2460
    },
    {
      "epoch": 0.18584004213377472,
      "grad_norm": 23.24254151424235,
      "learning_rate": 4.888766057496833e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -1.828125,
      "logps/chosen": -274.0,
      "logps/rejected": -488.0,
      "loss": 0.6112,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.609375,
      "rewards/margins": 2.0625,
      "rewards/rejected": -3.671875,
      "step": 2470
    },
    {
      "epoch": 0.18659243096832442,
      "grad_norm": 17.81492679666412,
      "learning_rate": 4.886820949181486e-07,
      "logits/chosen": -1.921875,
      "logits/rejected": -1.8203125,
      "logps/chosen": -236.0,
      "logps/rejected": -468.0,
      "loss": 0.5747,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.2421875,
      "rewards/margins": 2.3125,
      "rewards/rejected": -3.5625,
      "step": 2480
    },
    {
      "epoch": 0.18734481980287412,
      "grad_norm": 16.365045232734964,
      "learning_rate": 4.884859375008512e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -1.8203125,
      "logps/chosen": -255.0,
      "logps/rejected": -496.0,
      "loss": 0.6707,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.1875,
      "rewards/margins": 2.375,
      "rewards/rejected": -3.5625,
      "step": 2490
    },
    {
      "epoch": 0.18809720863742382,
      "grad_norm": 14.052969573746315,
      "learning_rate": 4.882881348510136e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -2.1875,
      "logps/chosen": -239.0,
      "logps/rejected": -468.0,
      "loss": 0.5603,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.125,
      "rewards/margins": 2.40625,
      "rewards/rejected": -3.53125,
      "step": 2500
    },
    {
      "epoch": 0.18884959747197352,
      "grad_norm": 15.981767924767054,
      "learning_rate": 4.880886883332083e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.9140625,
      "logps/chosen": -230.0,
      "logps/rejected": -500.0,
      "loss": 0.5487,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.03125,
      "rewards/margins": 2.6875,
      "rewards/rejected": -3.71875,
      "step": 2510
    },
    {
      "epoch": 0.18960198630652322,
      "grad_norm": 17.622155332496813,
      "learning_rate": 4.878875993233486e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -2.0,
      "logps/chosen": -258.0,
      "logps/rejected": -466.0,
      "loss": 0.5698,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.5,
      "rewards/margins": 1.9921875,
      "rewards/rejected": -3.484375,
      "step": 2520
    },
    {
      "epoch": 0.1903543751410729,
      "grad_norm": 20.12401769819481,
      "learning_rate": 4.876848692086782e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.046875,
      "logps/chosen": -276.0,
      "logps/rejected": -500.0,
      "loss": 0.5624,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.390625,
      "rewards/margins": 2.234375,
      "rewards/rejected": -3.609375,
      "step": 2530
    },
    {
      "epoch": 0.1911067639756226,
      "grad_norm": 23.350655720951597,
      "learning_rate": 4.874804993877625e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -2.109375,
      "logps/chosen": -243.0,
      "logps/rejected": -450.0,
      "loss": 0.6079,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.2890625,
      "rewards/margins": 2.09375,
      "rewards/rejected": -3.375,
      "step": 2540
    },
    {
      "epoch": 0.1918591528101723,
      "grad_norm": 16.09653231199627,
      "learning_rate": 4.872744912704788e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -1.9765625,
      "logps/chosen": -239.0,
      "logps/rejected": -458.0,
      "loss": 0.531,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.1796875,
      "rewards/margins": 2.34375,
      "rewards/rejected": -3.53125,
      "step": 2550
    },
    {
      "epoch": 0.192611541644722,
      "grad_norm": 17.613538807521895,
      "learning_rate": 4.870668462780062e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.0625,
      "logps/chosen": -260.0,
      "logps/rejected": -460.0,
      "loss": 0.5924,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.40625,
      "rewards/margins": 2.09375,
      "rewards/rejected": -3.5,
      "step": 2560
    },
    {
      "epoch": 0.1933639304792717,
      "grad_norm": 20.771809988245113,
      "learning_rate": 4.868575658428163e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.15625,
      "logps/chosen": -250.0,
      "logps/rejected": -442.0,
      "loss": 0.6004,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.328125,
      "rewards/margins": 2.0625,
      "rewards/rejected": -3.390625,
      "step": 2570
    },
    {
      "epoch": 0.1941163193138214,
      "grad_norm": 26.943046522573482,
      "learning_rate": 4.866466514086628e-07,
      "logits/chosen": -1.921875,
      "logits/rejected": -1.7421875,
      "logps/chosen": -270.0,
      "logps/rejected": -498.0,
      "loss": 0.5843,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.2890625,
      "rewards/margins": 2.5625,
      "rewards/rejected": -3.859375,
      "step": 2580
    },
    {
      "epoch": 0.19486870814837107,
      "grad_norm": 15.666571819738936,
      "learning_rate": 4.864341044305719e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -2.03125,
      "logps/chosen": -282.0,
      "logps/rejected": -452.0,
      "loss": 0.5785,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.4375,
      "rewards/margins": 2.03125,
      "rewards/rejected": -3.46875,
      "step": 2590
    },
    {
      "epoch": 0.19562109698292077,
      "grad_norm": 19.52713797872769,
      "learning_rate": 4.862199263748323e-07,
      "logits/chosen": -2.109375,
      "logits/rejected": -2.078125,
      "logps/chosen": -282.0,
      "logps/rejected": -482.0,
      "loss": 0.5803,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.609375,
      "rewards/margins": 2.21875,
      "rewards/rejected": -3.828125,
      "step": 2600
    },
    {
      "epoch": 0.19637348581747047,
      "grad_norm": 14.203879982252456,
      "learning_rate": 4.860041187189846e-07,
      "logits/chosen": -2.25,
      "logits/rejected": -2.21875,
      "logps/chosen": -239.0,
      "logps/rejected": -466.0,
      "loss": 0.6116,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.234375,
      "rewards/margins": 2.265625,
      "rewards/rejected": -3.5,
      "step": 2610
    },
    {
      "epoch": 0.19712587465202017,
      "grad_norm": 23.249551172710085,
      "learning_rate": 4.857866829518118e-07,
      "logits/chosen": -2.03125,
      "logits/rejected": -2.0625,
      "logps/chosen": -240.0,
      "logps/rejected": -498.0,
      "loss": 0.5656,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.2578125,
      "rewards/margins": 2.359375,
      "rewards/rejected": -3.609375,
      "step": 2620
    },
    {
      "epoch": 0.19787826348656987,
      "grad_norm": 22.32240998353166,
      "learning_rate": 4.855676205733287e-07,
      "logits/chosen": -2.0625,
      "logits/rejected": -1.8984375,
      "logps/chosen": -268.0,
      "logps/rejected": -466.0,
      "loss": 0.6494,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.375,
      "rewards/margins": 2.015625,
      "rewards/rejected": -3.390625,
      "step": 2630
    },
    {
      "epoch": 0.19863065232111957,
      "grad_norm": 20.249958335322624,
      "learning_rate": 4.853469330947712e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.25,
      "logps/chosen": -256.0,
      "logps/rejected": -462.0,
      "loss": 0.5673,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.1484375,
      "rewards/margins": 2.296875,
      "rewards/rejected": -3.4375,
      "step": 2640
    },
    {
      "epoch": 0.19938304115566924,
      "grad_norm": 22.560749343438296,
      "learning_rate": 4.851246220385862e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.140625,
      "logps/chosen": -236.0,
      "logps/rejected": -422.0,
      "loss": 0.5974,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.0703125,
      "rewards/margins": 2.046875,
      "rewards/rejected": -3.125,
      "step": 2650
    },
    {
      "epoch": 0.20013542999021894,
      "grad_norm": 12.869515514324851,
      "learning_rate": 4.849006889384217e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.296875,
      "logps/chosen": -254.0,
      "logps/rejected": -430.0,
      "loss": 0.5902,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.15625,
      "rewards/margins": 2.0,
      "rewards/rejected": -3.15625,
      "step": 2660
    },
    {
      "epoch": 0.20088781882476864,
      "grad_norm": 21.484571207146782,
      "learning_rate": 4.84675135339115e-07,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.9921875,
      "logps/chosen": -229.0,
      "logps/rejected": -460.0,
      "loss": 0.6057,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.109375,
      "rewards/margins": 2.390625,
      "rewards/rejected": -3.5,
      "step": 2670
    },
    {
      "epoch": 0.20164020765931834,
      "grad_norm": 14.074901227219774,
      "learning_rate": 4.84447962796683e-07,
      "logits/chosen": -1.9453125,
      "logits/rejected": -1.8125,
      "logps/chosen": -290.0,
      "logps/rejected": -492.0,
      "loss": 0.6213,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.453125,
      "rewards/margins": 2.203125,
      "rewards/rejected": -3.65625,
      "step": 2680
    },
    {
      "epoch": 0.20239259649386804,
      "grad_norm": 25.838686727852608,
      "learning_rate": 4.842191728783107e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.125,
      "logps/chosen": -254.0,
      "logps/rejected": -442.0,
      "loss": 0.6187,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.3046875,
      "rewards/margins": 1.921875,
      "rewards/rejected": -3.21875,
      "step": 2690
    },
    {
      "epoch": 0.20314498532841774,
      "grad_norm": 24.01302596739655,
      "learning_rate": 4.839887671623414e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.09375,
      "logps/chosen": -225.0,
      "logps/rejected": -444.0,
      "loss": 0.5522,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.0703125,
      "rewards/margins": 2.21875,
      "rewards/rejected": -3.28125,
      "step": 2700
    },
    {
      "epoch": 0.2038973741629674,
      "grad_norm": 35.03668301006675,
      "learning_rate": 4.837567472382651e-07,
      "logits/chosen": -2.09375,
      "logits/rejected": -1.984375,
      "logps/chosen": -253.0,
      "logps/rejected": -474.0,
      "loss": 0.6159,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.390625,
      "rewards/margins": 2.1875,
      "rewards/rejected": -3.578125,
      "step": 2710
    },
    {
      "epoch": 0.2046497629975171,
      "grad_norm": 21.25424587195176,
      "learning_rate": 4.835231147067072e-07,
      "logits/chosen": -2.046875,
      "logits/rejected": -2.0625,
      "logps/chosen": -238.0,
      "logps/rejected": -466.0,
      "loss": 0.5978,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.203125,
      "rewards/margins": 2.234375,
      "rewards/rejected": -3.4375,
      "step": 2720
    },
    {
      "epoch": 0.2054021518320668,
      "grad_norm": 17.59506129001128,
      "learning_rate": 4.832878711794185e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.0,
      "logps/chosen": -223.0,
      "logps/rejected": -510.0,
      "loss": 0.5677,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.015625,
      "rewards/margins": 2.8125,
      "rewards/rejected": -3.828125,
      "step": 2730
    },
    {
      "epoch": 0.2061545406666165,
      "grad_norm": 14.28920577509895,
      "learning_rate": 4.830510182792632e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.203125,
      "logps/chosen": -256.0,
      "logps/rejected": -492.0,
      "loss": 0.5188,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.375,
      "rewards/margins": 2.421875,
      "rewards/rejected": -3.796875,
      "step": 2740
    },
    {
      "epoch": 0.2069069295011662,
      "grad_norm": 25.492784035798838,
      "learning_rate": 4.82812557640208e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -2.453125,
      "logps/chosen": -292.0,
      "logps/rejected": -454.0,
      "loss": 0.5876,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.46875,
      "rewards/margins": 2.09375,
      "rewards/rejected": -3.578125,
      "step": 2750
    },
    {
      "epoch": 0.2076593183357159,
      "grad_norm": 20.377097986491428,
      "learning_rate": 4.82572490907311e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -2.078125,
      "logps/chosen": -268.0,
      "logps/rejected": -490.0,
      "loss": 0.5646,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.3125,
      "rewards/margins": 2.515625,
      "rewards/rejected": -3.828125,
      "step": 2760
    },
    {
      "epoch": 0.20841170717026558,
      "grad_norm": 16.695177535089368,
      "learning_rate": 4.8233081973671e-07,
      "logits/chosen": -2.125,
      "logits/rejected": -1.859375,
      "logps/chosen": -264.0,
      "logps/rejected": -484.0,
      "loss": 0.5406,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 2.171875,
      "rewards/rejected": -3.6875,
      "step": 2770
    },
    {
      "epoch": 0.20916409600481528,
      "grad_norm": 14.947335317025459,
      "learning_rate": 4.820875457956115e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -2.3125,
      "logps/chosen": -244.0,
      "logps/rejected": -478.0,
      "loss": 0.5824,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.21875,
      "rewards/margins": 2.390625,
      "rewards/rejected": -3.625,
      "step": 2780
    },
    {
      "epoch": 0.20991648483936498,
      "grad_norm": 18.230891389779174,
      "learning_rate": 4.818426707622788e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.1875,
      "logps/chosen": -268.0,
      "logps/rejected": -484.0,
      "loss": 0.5463,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.3671875,
      "rewards/margins": 2.3125,
      "rewards/rejected": -3.671875,
      "step": 2790
    },
    {
      "epoch": 0.21066887367391468,
      "grad_norm": 18.434072166259785,
      "learning_rate": 4.815961963260207e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.359375,
      "logps/chosen": -280.0,
      "logps/rejected": -516.0,
      "loss": 0.5421,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 2.5,
      "rewards/rejected": -4.0625,
      "step": 2800
    },
    {
      "epoch": 0.21142126250846438,
      "grad_norm": 19.372614078493147,
      "learning_rate": 4.813481241871794e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.171875,
      "logps/chosen": -266.0,
      "logps/rejected": -524.0,
      "loss": 0.5625,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 2.515625,
      "rewards/rejected": -3.953125,
      "step": 2810
    },
    {
      "epoch": 0.21217365134301408,
      "grad_norm": 25.07095714814959,
      "learning_rate": 4.810984560571195e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.0625,
      "logps/chosen": -245.0,
      "logps/rejected": -502.0,
      "loss": 0.5674,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.3515625,
      "rewards/margins": 2.734375,
      "rewards/rejected": -4.09375,
      "step": 2820
    },
    {
      "epoch": 0.21292604017756375,
      "grad_norm": 19.712629910655593,
      "learning_rate": 4.808471936582156e-07,
      "logits/chosen": -2.140625,
      "logits/rejected": -2.15625,
      "logps/chosen": -239.0,
      "logps/rejected": -454.0,
      "loss": 0.5349,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.2109375,
      "rewards/margins": 2.1875,
      "rewards/rejected": -3.40625,
      "step": 2830
    },
    {
      "epoch": 0.21367842901211345,
      "grad_norm": 22.365780994293157,
      "learning_rate": 4.805943387238409e-07,
      "logits/chosen": -2.15625,
      "logits/rejected": -2.125,
      "logps/chosen": -264.0,
      "logps/rejected": -488.0,
      "loss": 0.53,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.375,
      "rewards/margins": 2.40625,
      "rewards/rejected": -3.78125,
      "step": 2840
    },
    {
      "epoch": 0.21443081784666315,
      "grad_norm": 22.69894367876453,
      "learning_rate": 4.803398929983543e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.109375,
      "logps/chosen": -260.0,
      "logps/rejected": -512.0,
      "loss": 0.6074,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.4375,
      "rewards/margins": 2.59375,
      "rewards/rejected": -4.03125,
      "step": 2850
    },
    {
      "epoch": 0.21518320668121285,
      "grad_norm": 26.562818352749527,
      "learning_rate": 4.800838582370898e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.15625,
      "logps/chosen": -272.0,
      "logps/rejected": -462.0,
      "loss": 0.6057,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.578125,
      "rewards/margins": 1.890625,
      "rewards/rejected": -3.46875,
      "step": 2860
    },
    {
      "epoch": 0.21593559551576255,
      "grad_norm": 18.65311295870827,
      "learning_rate": 4.79826236206343e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.015625,
      "logps/chosen": -270.0,
      "logps/rejected": -506.0,
      "loss": 0.5466,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.4921875,
      "rewards/margins": 2.34375,
      "rewards/rejected": -3.84375,
      "step": 2870
    },
    {
      "epoch": 0.21668798435031225,
      "grad_norm": 23.12253184249722,
      "learning_rate": 4.795670286833599e-07,
      "logits/chosen": -2.25,
      "logits/rejected": -2.203125,
      "logps/chosen": -268.0,
      "logps/rejected": -490.0,
      "loss": 0.5404,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.296875,
      "rewards/margins": 2.328125,
      "rewards/rejected": -3.625,
      "step": 2880
    },
    {
      "epoch": 0.21744037318486192,
      "grad_norm": 19.528320984783537,
      "learning_rate": 4.79306237456324e-07,
      "logits/chosen": -2.171875,
      "logits/rejected": -2.140625,
      "logps/chosen": -262.0,
      "logps/rejected": -478.0,
      "loss": 0.556,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.453125,
      "rewards/margins": 2.125,
      "rewards/rejected": -3.578125,
      "step": 2890
    },
    {
      "epoch": 0.21819276201941162,
      "grad_norm": 24.271743697153067,
      "learning_rate": 4.790438643243447e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.359375,
      "logps/chosen": -255.0,
      "logps/rejected": -480.0,
      "loss": 0.5741,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.1640625,
      "rewards/margins": 2.453125,
      "rewards/rejected": -3.609375,
      "step": 2900
    },
    {
      "epoch": 0.21894515085396132,
      "grad_norm": 18.316831884088554,
      "learning_rate": 4.787799110974436e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.25,
      "logps/chosen": -262.0,
      "logps/rejected": -492.0,
      "loss": 0.5602,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.4296875,
      "rewards/margins": 2.453125,
      "rewards/rejected": -3.875,
      "step": 2910
    },
    {
      "epoch": 0.21969753968851102,
      "grad_norm": 23.70378706577178,
      "learning_rate": 4.785143795965437e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.328125,
      "logps/chosen": -266.0,
      "logps/rejected": -468.0,
      "loss": 0.5666,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 2.03125,
      "rewards/rejected": -3.484375,
      "step": 2920
    },
    {
      "epoch": 0.22044992852306072,
      "grad_norm": 16.70152046927947,
      "learning_rate": 4.782472716534554e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.09375,
      "logps/chosen": -286.0,
      "logps/rejected": -524.0,
      "loss": 0.5108,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.4609375,
      "rewards/margins": 2.359375,
      "rewards/rejected": -3.828125,
      "step": 2930
    },
    {
      "epoch": 0.22120231735761042,
      "grad_norm": 21.508285399012113,
      "learning_rate": 4.779785891108647e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.1875,
      "logps/chosen": -274.0,
      "logps/rejected": -484.0,
      "loss": 0.5933,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.4765625,
      "rewards/margins": 2.203125,
      "rewards/rejected": -3.6875,
      "step": 2940
    },
    {
      "epoch": 0.2219547061921601,
      "grad_norm": 24.739045285619603,
      "learning_rate": 4.777083338223202e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.359375,
      "logps/chosen": -266.0,
      "logps/rejected": -502.0,
      "loss": 0.538,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 2.328125,
      "rewards/rejected": -3.859375,
      "step": 2950
    },
    {
      "epoch": 0.2227070950267098,
      "grad_norm": 31.004603768524817,
      "learning_rate": 4.774365076522202e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.421875,
      "logps/chosen": -300.0,
      "logps/rejected": -486.0,
      "loss": 0.5457,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.5,
      "rewards/margins": 2.234375,
      "rewards/rejected": -3.734375,
      "step": 2960
    },
    {
      "epoch": 0.2234594838612595,
      "grad_norm": 16.307156425827937,
      "learning_rate": 4.771631124758e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -2.171875,
      "logps/chosen": -254.0,
      "logps/rejected": -528.0,
      "loss": 0.5484,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.4296875,
      "rewards/margins": 2.671875,
      "rewards/rejected": -4.09375,
      "step": 2970
    },
    {
      "epoch": 0.2242118726958092,
      "grad_norm": 20.027864615108168,
      "learning_rate": 4.76888150179119e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.25,
      "logps/chosen": -247.0,
      "logps/rejected": -528.0,
      "loss": 0.5272,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.2265625,
      "rewards/margins": 2.828125,
      "rewards/rejected": -4.0625,
      "step": 2980
    },
    {
      "epoch": 0.2249642615303589,
      "grad_norm": 24.75903304408932,
      "learning_rate": 4.7661162265904773e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.390625,
      "logps/chosen": -272.0,
      "logps/rejected": -508.0,
      "loss": 0.5534,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.3671875,
      "rewards/margins": 2.546875,
      "rewards/rejected": -3.921875,
      "step": 2990
    },
    {
      "epoch": 0.2257166503649086,
      "grad_norm": 17.70102224517484,
      "learning_rate": 4.7633353182325443e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -2.140625,
      "logps/chosen": -248.0,
      "logps/rejected": -536.0,
      "loss": 0.5173,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.3828125,
      "rewards/margins": 2.6875,
      "rewards/rejected": -4.0625,
      "step": 3000
    },
    {
      "epoch": 0.2257166503649086,
      "eval_logits/chosen": -2.34375,
      "eval_logits/rejected": -2.328125,
      "eval_logps/chosen": -278.0,
      "eval_logps/rejected": -504.0,
      "eval_loss": 0.30194997787475586,
      "eval_rewards/accuracies": 0.870853066444397,
      "eval_rewards/chosen": -1.4609375,
      "eval_rewards/margins": 2.40625,
      "eval_rewards/rejected": -3.859375,
      "eval_runtime": 2669.471,
      "eval_samples_per_second": 35.398,
      "eval_steps_per_second": 0.553,
      "step": 3000
    },
    {
      "epoch": 0.22646903919945827,
      "grad_norm": 20.39406410714397,
      "learning_rate": 4.760538795901923e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.28125,
      "logps/chosen": -268.0,
      "logps/rejected": -520.0,
      "loss": 0.5781,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.4609375,
      "rewards/margins": 2.65625,
      "rewards/rejected": -4.125,
      "step": 3010
    },
    {
      "epoch": 0.22722142803400797,
      "grad_norm": 16.761372829462697,
      "learning_rate": 4.757726678890859e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.453125,
      "logps/chosen": -252.0,
      "logps/rejected": -600.0,
      "loss": 0.5596,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.2421875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -4.4375,
      "step": 3020
    },
    {
      "epoch": 0.22797381686855767,
      "grad_norm": 16.967322787983278,
      "learning_rate": 4.754898986599182e-07,
      "logits/chosen": -2.25,
      "logits/rejected": -2.359375,
      "logps/chosen": -260.0,
      "logps/rejected": -446.0,
      "loss": 0.5666,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.40625,
      "rewards/margins": 1.9375,
      "rewards/rejected": -3.34375,
      "step": 3030
    },
    {
      "epoch": 0.22872620570310737,
      "grad_norm": 30.537187077757903,
      "learning_rate": 4.75205573853417e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.171875,
      "logps/chosen": -260.0,
      "logps/rejected": -470.0,
      "loss": 0.585,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.390625,
      "rewards/margins": 2.28125,
      "rewards/rejected": -3.671875,
      "step": 3040
    },
    {
      "epoch": 0.22947859453765707,
      "grad_norm": 20.59582694720253,
      "learning_rate": 4.749196954310414e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.375,
      "logps/chosen": -264.0,
      "logps/rejected": -512.0,
      "loss": 0.5513,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.3828125,
      "rewards/margins": 2.578125,
      "rewards/rejected": -3.953125,
      "step": 3050
    },
    {
      "epoch": 0.23023098337220677,
      "grad_norm": 18.111494072081317,
      "learning_rate": 4.746322653649683e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.421875,
      "logps/chosen": -284.0,
      "logps/rejected": -536.0,
      "loss": 0.556,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.671875,
      "rewards/margins": 2.515625,
      "rewards/rejected": -4.1875,
      "step": 3060
    },
    {
      "epoch": 0.23098337220675647,
      "grad_norm": 14.735055056223025,
      "learning_rate": 4.7434328563807913e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.453125,
      "logps/chosen": -241.0,
      "logps/rejected": -540.0,
      "loss": 0.5297,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.25,
      "rewards/margins": 2.796875,
      "rewards/rejected": -4.03125,
      "step": 3070
    },
    {
      "epoch": 0.23173576104130614,
      "grad_norm": 25.078126350285537,
      "learning_rate": 4.7405275824394574e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.25,
      "logps/chosen": -282.0,
      "logps/rejected": -464.0,
      "loss": 0.5384,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 1.96875,
      "rewards/rejected": -3.515625,
      "step": 3080
    },
    {
      "epoch": 0.23248814987585584,
      "grad_norm": 18.670783364890944,
      "learning_rate": 4.737606851868167e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.4375,
      "logps/chosen": -304.0,
      "logps/rejected": -536.0,
      "loss": 0.5262,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.625,
      "rewards/margins": 2.703125,
      "rewards/rejected": -4.34375,
      "step": 3090
    },
    {
      "epoch": 0.23324053871040554,
      "grad_norm": 17.084820169228372,
      "learning_rate": 4.734670684816037e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.25,
      "logps/chosen": -294.0,
      "logps/rejected": -564.0,
      "loss": 0.522,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.4140625,
      "rewards/margins": 2.734375,
      "rewards/rejected": -4.125,
      "step": 3100
    },
    {
      "epoch": 0.23399292754495524,
      "grad_norm": 16.508187022356626,
      "learning_rate": 4.7317191015386744e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.578125,
      "logps/chosen": -247.0,
      "logps/rejected": -486.0,
      "loss": 0.5397,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.3984375,
      "rewards/margins": 2.390625,
      "rewards/rejected": -3.796875,
      "step": 3110
    },
    {
      "epoch": 0.23474531637950494,
      "grad_norm": 22.81112843690957,
      "learning_rate": 4.7287521223980395e-07,
      "logits/chosen": -2.1875,
      "logits/rejected": -2.375,
      "logps/chosen": -272.0,
      "logps/rejected": -516.0,
      "loss": 0.5663,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.3359375,
      "rewards/margins": 2.59375,
      "rewards/rejected": -3.9375,
      "step": 3120
    },
    {
      "epoch": 0.23549770521405464,
      "grad_norm": 14.137736470289017,
      "learning_rate": 4.725769767862301e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.390625,
      "logps/chosen": -266.0,
      "logps/rejected": -464.0,
      "loss": 0.5185,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.4609375,
      "rewards/margins": 2.09375,
      "rewards/rejected": -3.546875,
      "step": 3130
    },
    {
      "epoch": 0.2362500940486043,
      "grad_norm": 17.708104747597037,
      "learning_rate": 4.7227720585056986e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.40625,
      "logps/chosen": -274.0,
      "logps/rejected": -500.0,
      "loss": 0.5044,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.3828125,
      "rewards/margins": 2.671875,
      "rewards/rejected": -4.0625,
      "step": 3140
    },
    {
      "epoch": 0.237002482883154,
      "grad_norm": 21.857508406971537,
      "learning_rate": 4.7197590150083986e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.359375,
      "logps/chosen": -276.0,
      "logps/rejected": -564.0,
      "loss": 0.5423,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.4296875,
      "rewards/margins": 2.9375,
      "rewards/rejected": -4.375,
      "step": 3150
    },
    {
      "epoch": 0.2377548717177037,
      "grad_norm": 30.144439359706897,
      "learning_rate": 4.716730658156354e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.4375,
      "logps/chosen": -276.0,
      "logps/rejected": -564.0,
      "loss": 0.5212,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 2.953125,
      "rewards/rejected": -4.5,
      "step": 3160
    },
    {
      "epoch": 0.2385072605522534,
      "grad_norm": 20.301753028417583,
      "learning_rate": 4.7136870088411564e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.53125,
      "logps/chosen": -276.0,
      "logps/rejected": -544.0,
      "loss": 0.5021,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.5625,
      "rewards/margins": 2.828125,
      "rewards/rejected": -4.375,
      "step": 3170
    },
    {
      "epoch": 0.2392596493868031,
      "grad_norm": 21.739282302934885,
      "learning_rate": 4.710628088059898e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.34375,
      "logps/chosen": -252.0,
      "logps/rejected": -510.0,
      "loss": 0.5064,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.3046875,
      "rewards/margins": 2.53125,
      "rewards/rejected": -3.828125,
      "step": 3180
    },
    {
      "epoch": 0.2400120382213528,
      "grad_norm": 26.049130161240477,
      "learning_rate": 4.707553916915022e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.484375,
      "logps/chosen": -268.0,
      "logps/rejected": -500.0,
      "loss": 0.5723,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 2.375,
      "rewards/rejected": -3.984375,
      "step": 3190
    },
    {
      "epoch": 0.24076442705590248,
      "grad_norm": 20.773764406599536,
      "learning_rate": 4.704464516614178e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.328125,
      "logps/chosen": -262.0,
      "logps/rejected": -544.0,
      "loss": 0.5439,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.4296875,
      "rewards/margins": 2.8125,
      "rewards/rejected": -4.21875,
      "step": 3200
    },
    {
      "epoch": 0.24151681589045218,
      "grad_norm": 23.350184605630393,
      "learning_rate": 4.7013599084700787e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.359375,
      "logps/chosen": -270.0,
      "logps/rejected": -502.0,
      "loss": 0.5709,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 2.46875,
      "rewards/rejected": -3.96875,
      "step": 3210
    },
    {
      "epoch": 0.24226920472500188,
      "grad_norm": 25.201870382018328,
      "learning_rate": 4.698240113900348e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.359375,
      "logps/chosen": -239.0,
      "logps/rejected": -540.0,
      "loss": 0.5307,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.3046875,
      "rewards/margins": 2.953125,
      "rewards/rejected": -4.25,
      "step": 3220
    },
    {
      "epoch": 0.24302159355955158,
      "grad_norm": 20.34420841939156,
      "learning_rate": 4.6951051544273765e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.1875,
      "logps/chosen": -244.0,
      "logps/rejected": -596.0,
      "loss": 0.5065,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1484375,
      "rewards/margins": 3.40625,
      "rewards/rejected": -4.5625,
      "step": 3230
    },
    {
      "epoch": 0.24377398239410128,
      "grad_norm": 17.1843569078629,
      "learning_rate": 4.6919550516781723e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.46875,
      "logps/chosen": -240.0,
      "logps/rejected": -508.0,
      "loss": 0.518,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.2265625,
      "rewards/margins": 2.6875,
      "rewards/rejected": -3.921875,
      "step": 3240
    },
    {
      "epoch": 0.24452637122865098,
      "grad_norm": 27.57780798833985,
      "learning_rate": 4.688789827384213e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.453125,
      "logps/chosen": -292.0,
      "logps/rejected": -506.0,
      "loss": 0.563,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 2.28125,
      "rewards/rejected": -3.875,
      "step": 3250
    },
    {
      "epoch": 0.24527876006320065,
      "grad_norm": 24.50557701822332,
      "learning_rate": 4.6856095033812927e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.28125,
      "logps/chosen": -239.0,
      "logps/rejected": -508.0,
      "loss": 0.5226,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.1484375,
      "rewards/margins": 2.75,
      "rewards/rejected": -3.90625,
      "step": 3260
    },
    {
      "epoch": 0.24603114889775035,
      "grad_norm": 19.5339534815997,
      "learning_rate": 4.682414101609374e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.25,
      "logps/chosen": -251.0,
      "logps/rejected": -496.0,
      "loss": 0.54,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.359375,
      "rewards/margins": 2.40625,
      "rewards/rejected": -3.765625,
      "step": 3270
    },
    {
      "epoch": 0.24678353773230005,
      "grad_norm": 16.86000922234752,
      "learning_rate": 4.679203644112436e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.40625,
      "logps/chosen": -268.0,
      "logps/rejected": -548.0,
      "loss": 0.535,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.46875,
      "rewards/margins": 2.734375,
      "rewards/rejected": -4.21875,
      "step": 3280
    },
    {
      "epoch": 0.24753592656684975,
      "grad_norm": 21.174658676159325,
      "learning_rate": 4.6759781530383214e-07,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.453125,
      "logps/chosen": -268.0,
      "logps/rejected": -544.0,
      "loss": 0.5829,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5625,
      "rewards/margins": 2.828125,
      "rewards/rejected": -4.40625,
      "step": 3290
    },
    {
      "epoch": 0.24828831540139945,
      "grad_norm": 24.451503092807013,
      "learning_rate": 4.672737650638586e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.40625,
      "logps/chosen": -282.0,
      "logps/rejected": -500.0,
      "loss": 0.535,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 2.40625,
      "rewards/rejected": -3.9375,
      "step": 3300
    },
    {
      "epoch": 0.24904070423594915,
      "grad_norm": 16.69602647933983,
      "learning_rate": 4.669482159268341e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.359375,
      "logps/chosen": -240.0,
      "logps/rejected": -484.0,
      "loss": 0.5327,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.28125,
      "rewards/margins": 2.484375,
      "rewards/rejected": -3.765625,
      "step": 3310
    },
    {
      "epoch": 0.24979309307049882,
      "grad_norm": 23.783726560370386,
      "learning_rate": 4.666211701386103e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.484375,
      "logps/chosen": -268.0,
      "logps/rejected": -494.0,
      "loss": 0.5283,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.515625,
      "rewards/margins": 2.46875,
      "rewards/rejected": -3.984375,
      "step": 3320
    },
    {
      "epoch": 0.25054548190504855,
      "grad_norm": 21.780517565226354,
      "learning_rate": 4.662926299553638e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.578125,
      "logps/chosen": -274.0,
      "logps/rejected": -540.0,
      "loss": 0.5086,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.53125,
      "rewards/margins": 2.734375,
      "rewards/rejected": -4.28125,
      "step": 3330
    },
    {
      "epoch": 0.25129787073959825,
      "grad_norm": 20.24696352967672,
      "learning_rate": 4.6596259764358037e-07,
      "logits/chosen": -2.28125,
      "logits/rejected": -2.234375,
      "logps/chosen": -270.0,
      "logps/rejected": -596.0,
      "loss": 0.4834,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.484375,
      "rewards/margins": 3.0625,
      "rewards/rejected": -4.5625,
      "step": 3340
    },
    {
      "epoch": 0.2520502595741479,
      "grad_norm": 24.210212416481845,
      "learning_rate": 4.6563107548003967e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.421875,
      "logps/chosen": -262.0,
      "logps/rejected": -536.0,
      "loss": 0.544,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.5234375,
      "rewards/margins": 2.71875,
      "rewards/rejected": -4.21875,
      "step": 3350
    },
    {
      "epoch": 0.2528026484086976,
      "grad_norm": 19.034818639283614,
      "learning_rate": 4.6529806575179895e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.28125,
      "logps/chosen": -243.0,
      "logps/rejected": -472.0,
      "loss": 0.5451,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3359375,
      "rewards/margins": 2.359375,
      "rewards/rejected": -3.6875,
      "step": 3360
    },
    {
      "epoch": 0.2535550372432473,
      "grad_norm": 23.2464110618845,
      "learning_rate": 4.6496357075617816e-07,
      "logits/chosen": -2.203125,
      "logits/rejected": -2.25,
      "logps/chosen": -276.0,
      "logps/rejected": -552.0,
      "loss": 0.4945,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.3359375,
      "rewards/margins": 2.921875,
      "rewards/rejected": -4.25,
      "step": 3370
    },
    {
      "epoch": 0.254307426077797,
      "grad_norm": 16.869696023995385,
      "learning_rate": 4.646275928007431e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.546875,
      "logps/chosen": -286.0,
      "logps/rejected": -532.0,
      "loss": 0.5126,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 2.484375,
      "rewards/rejected": -4.1875,
      "step": 3380
    },
    {
      "epoch": 0.2550598149123467,
      "grad_norm": 23.84393739649861,
      "learning_rate": 4.642901342032905e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.53125,
      "logps/chosen": -288.0,
      "logps/rejected": -556.0,
      "loss": 0.5401,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 2.75,
      "rewards/rejected": -4.46875,
      "step": 3390
    },
    {
      "epoch": 0.2558122037468964,
      "grad_norm": 23.88254024886985,
      "learning_rate": 4.639511972918311e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.671875,
      "logps/chosen": -282.0,
      "logps/rejected": -536.0,
      "loss": 0.5175,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.765625,
      "rewards/margins": 2.5625,
      "rewards/rejected": -4.3125,
      "step": 3400
    },
    {
      "epoch": 0.2565645925814461,
      "grad_norm": 27.877451137096493,
      "learning_rate": 4.636107844045742e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.390625,
      "logps/chosen": -260.0,
      "logps/rejected": -564.0,
      "loss": 0.5137,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.3984375,
      "rewards/margins": 2.953125,
      "rewards/rejected": -4.34375,
      "step": 3410
    },
    {
      "epoch": 0.2573169814159958,
      "grad_norm": 22.159509162829103,
      "learning_rate": 4.632688978899114e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.359375,
      "logps/chosen": -264.0,
      "logps/rejected": -552.0,
      "loss": 0.5052,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.40625,
      "rewards/margins": 2.796875,
      "rewards/rejected": -4.21875,
      "step": 3420
    },
    {
      "epoch": 0.2580693702505455,
      "grad_norm": 19.495230152239074,
      "learning_rate": 4.629255401064004e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.421875,
      "logps/chosen": -294.0,
      "logps/rejected": -532.0,
      "loss": 0.478,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 2.53125,
      "rewards/rejected": -4.21875,
      "step": 3430
    },
    {
      "epoch": 0.2588217590850952,
      "grad_norm": 20.05000500692216,
      "learning_rate": 4.625807134227483e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.546875,
      "logps/chosen": -274.0,
      "logps/rejected": -512.0,
      "loss": 0.5546,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.546875,
      "rewards/margins": 2.484375,
      "rewards/rejected": -4.03125,
      "step": 3440
    },
    {
      "epoch": 0.2595741479196449,
      "grad_norm": 13.506950771795047,
      "learning_rate": 4.622344202177961e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.203125,
      "logps/chosen": -270.0,
      "logps/rejected": -540.0,
      "loss": 0.5005,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.40625,
      "rewards/margins": 2.609375,
      "rewards/rejected": -4.03125,
      "step": 3450
    },
    {
      "epoch": 0.2603265367541946,
      "grad_norm": 30.286858370348586,
      "learning_rate": 4.6188666288050163e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.484375,
      "logps/chosen": -251.0,
      "logps/rejected": -520.0,
      "loss": 0.5569,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.2578125,
      "rewards/margins": 2.828125,
      "rewards/rejected": -4.09375,
      "step": 3460
    },
    {
      "epoch": 0.26107892558874424,
      "grad_norm": 17.12227948776187,
      "learning_rate": 4.615374438099232e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.515625,
      "logps/chosen": -266.0,
      "logps/rejected": -474.0,
      "loss": 0.5613,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.4375,
      "rewards/margins": 2.140625,
      "rewards/rejected": -3.59375,
      "step": 3470
    },
    {
      "epoch": 0.26183131442329394,
      "grad_norm": 23.498034409382743,
      "learning_rate": 4.611867654152033e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -2.359375,
      "logps/chosen": -290.0,
      "logps/rejected": -532.0,
      "loss": 0.5346,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.71875,
      "rewards/margins": 2.40625,
      "rewards/rejected": -4.125,
      "step": 3480
    },
    {
      "epoch": 0.26258370325784364,
      "grad_norm": 21.720323648948117,
      "learning_rate": 4.608346301155516e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.296875,
      "logps/chosen": -286.0,
      "logps/rejected": -552.0,
      "loss": 0.5299,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.5,
      "rewards/margins": 2.734375,
      "rewards/rejected": -4.21875,
      "step": 3490
    },
    {
      "epoch": 0.26333609209239334,
      "grad_norm": 16.817551627016755,
      "learning_rate": 4.604810403402285e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.484375,
      "logps/chosen": -264.0,
      "logps/rejected": -510.0,
      "loss": 0.5517,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 2.609375,
      "rewards/rejected": -4.15625,
      "step": 3500
    },
    {
      "epoch": 0.26408848092694304,
      "grad_norm": 18.927597237938723,
      "learning_rate": 4.601259985285284e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.46875,
      "logps/chosen": -294.0,
      "logps/rejected": -572.0,
      "loss": 0.4933,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 2.875,
      "rewards/rejected": -4.5,
      "step": 3510
    },
    {
      "epoch": 0.26484086976149274,
      "grad_norm": 16.61803029465999,
      "learning_rate": 4.5976950712976286e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.453125,
      "logps/chosen": -266.0,
      "logps/rejected": -532.0,
      "loss": 0.5159,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.484375,
      "rewards/margins": 2.671875,
      "rewards/rejected": -4.15625,
      "step": 3520
    },
    {
      "epoch": 0.26559325859604244,
      "grad_norm": 25.22207198099473,
      "learning_rate": 4.5941156860324345e-07,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.296875,
      "logps/chosen": -302.0,
      "logps/rejected": -628.0,
      "loss": 0.5518,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.609375,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.125,
      "step": 3530
    },
    {
      "epoch": 0.26634564743059214,
      "grad_norm": 19.603222270583007,
      "learning_rate": 4.590521854182651e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.59375,
      "logps/chosen": -296.0,
      "logps/rejected": -580.0,
      "loss": 0.4858,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.5,
      "rewards/margins": 3.046875,
      "rewards/rejected": -4.5625,
      "step": 3540
    },
    {
      "epoch": 0.26709803626514184,
      "grad_norm": 16.722830074613817,
      "learning_rate": 4.5869136005408893e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.578125,
      "logps/chosen": -304.0,
      "logps/rejected": -584.0,
      "loss": 0.5167,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.453125,
      "rewards/margins": 3.140625,
      "rewards/rejected": -4.59375,
      "step": 3550
    },
    {
      "epoch": 0.26785042509969154,
      "grad_norm": 22.41384747714571,
      "learning_rate": 4.5832909499992514e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.484375,
      "logps/chosen": -268.0,
      "logps/rejected": -568.0,
      "loss": 0.5355,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 3.046875,
      "rewards/rejected": -4.5625,
      "step": 3560
    },
    {
      "epoch": 0.26860281393424124,
      "grad_norm": 17.857363995652346,
      "learning_rate": 4.579653927549159e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.4375,
      "logps/chosen": -270.0,
      "logps/rejected": -540.0,
      "loss": 0.5296,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.46875,
      "rewards/margins": 2.921875,
      "rewards/rejected": -4.40625,
      "step": 3570
    },
    {
      "epoch": 0.26935520276879094,
      "grad_norm": 25.10604815877877,
      "learning_rate": 4.57600255828118e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.421875,
      "logps/chosen": -262.0,
      "logps/rejected": -584.0,
      "loss": 0.5683,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.4375,
      "rewards/margins": 3.125,
      "rewards/rejected": -4.5625,
      "step": 3580
    },
    {
      "epoch": 0.2701075916033406,
      "grad_norm": 18.015978292722778,
      "learning_rate": 4.572336867384856e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.4375,
      "logps/chosen": -288.0,
      "logps/rejected": -536.0,
      "loss": 0.5385,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 2.640625,
      "rewards/rejected": -4.21875,
      "step": 3590
    },
    {
      "epoch": 0.2708599804378903,
      "grad_norm": 18.914664206496923,
      "learning_rate": 4.5686568801485306e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.5625,
      "logps/chosen": -251.0,
      "logps/rejected": -502.0,
      "loss": 0.4532,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.34375,
      "rewards/margins": 2.640625,
      "rewards/rejected": -3.984375,
      "step": 3600
    },
    {
      "epoch": 0.27161236927244,
      "grad_norm": 30.290981489865846,
      "learning_rate": 4.5649626219591685e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.359375,
      "logps/chosen": -284.0,
      "logps/rejected": -572.0,
      "loss": 0.5669,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.65625,
      "rewards/margins": 2.875,
      "rewards/rejected": -4.53125,
      "step": 3610
    },
    {
      "epoch": 0.2723647581069897,
      "grad_norm": 18.95137244140273,
      "learning_rate": 4.5612541183021905e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.375,
      "logps/chosen": -264.0,
      "logps/rejected": -576.0,
      "loss": 0.4924,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.515625,
      "rewards/margins": 3.1875,
      "rewards/rejected": -4.6875,
      "step": 3620
    },
    {
      "epoch": 0.2731171469415394,
      "grad_norm": 27.72559343531231,
      "learning_rate": 4.5575313947612875e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.46875,
      "logps/chosen": -270.0,
      "logps/rejected": -576.0,
      "loss": 0.5339,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 3.03125,
      "rewards/rejected": -4.59375,
      "step": 3630
    },
    {
      "epoch": 0.2738695357760891,
      "grad_norm": 28.298684907631202,
      "learning_rate": 4.553794477018251e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.390625,
      "logps/chosen": -284.0,
      "logps/rejected": -548.0,
      "loss": 0.5538,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.5625,
      "rewards/margins": 2.734375,
      "rewards/rejected": -4.3125,
      "step": 3640
    },
    {
      "epoch": 0.2746219246106388,
      "grad_norm": 19.282224685813993,
      "learning_rate": 4.550043390852791e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.4375,
      "logps/chosen": -274.0,
      "logps/rejected": -556.0,
      "loss": 0.4888,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.578125,
      "rewards/margins": 2.765625,
      "rewards/rejected": -4.34375,
      "step": 3650
    },
    {
      "epoch": 0.2753743134451885,
      "grad_norm": 23.040642126309983,
      "learning_rate": 4.546278162142364e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.71875,
      "logps/chosen": -262.0,
      "logps/rejected": -504.0,
      "loss": 0.5067,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.4609375,
      "rewards/margins": 2.625,
      "rewards/rejected": -4.0625,
      "step": 3660
    },
    {
      "epoch": 0.2761267022797382,
      "grad_norm": 20.107922947295997,
      "learning_rate": 4.542498816861988e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.390625,
      "logps/chosen": -253.0,
      "logps/rejected": -496.0,
      "loss": 0.486,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.3203125,
      "rewards/margins": 2.578125,
      "rewards/rejected": -3.90625,
      "step": 3670
    },
    {
      "epoch": 0.2768790911142879,
      "grad_norm": 19.865023501989672,
      "learning_rate": 4.538705381084067e-07,
      "logits/chosen": -2.296875,
      "logits/rejected": -2.46875,
      "logps/chosen": -260.0,
      "logps/rejected": -488.0,
      "loss": 0.5343,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4921875,
      "rewards/margins": 2.4375,
      "rewards/rejected": -3.9375,
      "step": 3680
    },
    {
      "epoch": 0.2776314799488376,
      "grad_norm": 27.86578574563935,
      "learning_rate": 4.5348978809782123e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.296875,
      "logps/chosen": -282.0,
      "logps/rejected": -528.0,
      "loss": 0.5365,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.53125,
      "rewards/margins": 2.625,
      "rewards/rejected": -4.15625,
      "step": 3690
    },
    {
      "epoch": 0.2783838687833873,
      "grad_norm": 19.52815555113833,
      "learning_rate": 4.531076342811059e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.484375,
      "logps/chosen": -274.0,
      "logps/rejected": -564.0,
      "loss": 0.4505,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.3828125,
      "rewards/margins": 2.984375,
      "rewards/rejected": -4.375,
      "step": 3700
    },
    {
      "epoch": 0.2791362576179369,
      "grad_norm": 24.00158905821956,
      "learning_rate": 4.5272407929460846e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.40625,
      "logps/chosen": -294.0,
      "logps/rejected": -568.0,
      "loss": 0.5315,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.609375,
      "rewards/margins": 3.046875,
      "rewards/rejected": -4.65625,
      "step": 3710
    },
    {
      "epoch": 0.2798886464524866,
      "grad_norm": 17.113013086561917,
      "learning_rate": 4.5233912578434297e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.5,
      "logps/chosen": -302.0,
      "logps/rejected": -588.0,
      "loss": 0.5074,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.734375,
      "rewards/margins": 2.75,
      "rewards/rejected": -4.5,
      "step": 3720
    },
    {
      "epoch": 0.2806410352870363,
      "grad_norm": 27.149988685132236,
      "learning_rate": 4.5195277640597165e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.484375,
      "logps/chosen": -288.0,
      "logps/rejected": -556.0,
      "loss": 0.5334,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.625,
      "rewards/margins": 2.6875,
      "rewards/rejected": -4.3125,
      "step": 3730
    },
    {
      "epoch": 0.281393424121586,
      "grad_norm": 27.06881082673993,
      "learning_rate": 4.5156503382478583e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.3125,
      "logps/chosen": -292.0,
      "logps/rejected": -536.0,
      "loss": 0.5176,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.5625,
      "rewards/margins": 2.703125,
      "rewards/rejected": -4.28125,
      "step": 3740
    },
    {
      "epoch": 0.2821458129561357,
      "grad_norm": 21.715697785271068,
      "learning_rate": 4.511759007156886e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.5,
      "logps/chosen": -278.0,
      "logps/rejected": -540.0,
      "loss": 0.497,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.59375,
      "rewards/margins": 2.65625,
      "rewards/rejected": -4.25,
      "step": 3750
    },
    {
      "epoch": 0.2828982017906854,
      "grad_norm": 19.76074820354843,
      "learning_rate": 4.507853797631753e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.484375,
      "logps/chosen": -280.0,
      "logps/rejected": -516.0,
      "loss": 0.4955,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 2.4375,
      "rewards/rejected": -3.96875,
      "step": 3760
    },
    {
      "epoch": 0.2836505906252351,
      "grad_norm": 18.892363105567146,
      "learning_rate": 4.503934736613161e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.609375,
      "logps/chosen": -294.0,
      "logps/rejected": -572.0,
      "loss": 0.473,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.6875,
      "rewards/margins": 2.984375,
      "rewards/rejected": -4.65625,
      "step": 3770
    },
    {
      "epoch": 0.2844029794597848,
      "grad_norm": 20.575328323002775,
      "learning_rate": 4.500001851137363e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.484375,
      "logps/chosen": -255.0,
      "logps/rejected": -536.0,
      "loss": 0.5179,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.453125,
      "rewards/margins": 2.9375,
      "rewards/rejected": -4.375,
      "step": 3780
    },
    {
      "epoch": 0.2851553682943345,
      "grad_norm": 24.320248086311537,
      "learning_rate": 4.496055168335986e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.421875,
      "logps/chosen": -272.0,
      "logps/rejected": -556.0,
      "loss": 0.5402,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 2.65625,
      "rewards/rejected": -4.21875,
      "step": 3790
    },
    {
      "epoch": 0.2859077571288842,
      "grad_norm": 27.16462407495915,
      "learning_rate": 4.4920947154358384e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.453125,
      "logps/chosen": -286.0,
      "logps/rejected": -544.0,
      "loss": 0.5177,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 2.625,
      "rewards/rejected": -4.125,
      "step": 3800
    },
    {
      "epoch": 0.2866601459634339,
      "grad_norm": 18.61468237865215,
      "learning_rate": 4.4881205197587236e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.453125,
      "logps/chosen": -280.0,
      "logps/rejected": -520.0,
      "loss": 0.4803,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 2.578125,
      "rewards/rejected": -4.09375,
      "step": 3810
    },
    {
      "epoch": 0.2874125347979836,
      "grad_norm": 18.685617289971038,
      "learning_rate": 4.484132608721252e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.546875,
      "logps/chosen": -260.0,
      "logps/rejected": -536.0,
      "loss": 0.5089,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 2.6875,
      "rewards/rejected": -4.25,
      "step": 3820
    },
    {
      "epoch": 0.2881649236325333,
      "grad_norm": 29.703167001382056,
      "learning_rate": 4.480131009834651e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.34375,
      "logps/chosen": -298.0,
      "logps/rejected": -608.0,
      "loss": 0.5336,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.828125,
      "rewards/margins": 3.125,
      "rewards/rejected": -4.9375,
      "step": 3830
    },
    {
      "epoch": 0.28891731246708297,
      "grad_norm": 17.175627635715113,
      "learning_rate": 4.476115750704578e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.546875,
      "logps/chosen": -298.0,
      "logps/rejected": -552.0,
      "loss": 0.5429,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.671875,
      "rewards/margins": 2.78125,
      "rewards/rejected": -4.4375,
      "step": 3840
    },
    {
      "epoch": 0.28966970130163266,
      "grad_norm": 20.261928779606702,
      "learning_rate": 4.472086859030926e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.53125,
      "logps/chosen": -270.0,
      "logps/rejected": -568.0,
      "loss": 0.4723,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.640625,
      "rewards/margins": 2.859375,
      "rewards/rejected": -4.5,
      "step": 3850
    },
    {
      "epoch": 0.29042209013618236,
      "grad_norm": 18.33477341582105,
      "learning_rate": 4.468044362607633e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.375,
      "logps/chosen": -274.0,
      "logps/rejected": -584.0,
      "loss": 0.486,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.4765625,
      "rewards/margins": 3.015625,
      "rewards/rejected": -4.5,
      "step": 3860
    },
    {
      "epoch": 0.29117447897073206,
      "grad_norm": 24.90247102050078,
      "learning_rate": 4.4639882893224924e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.34375,
      "logps/chosen": -306.0,
      "logps/rejected": -564.0,
      "loss": 0.4872,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.765625,
      "rewards/margins": 2.71875,
      "rewards/rejected": -4.5,
      "step": 3870
    },
    {
      "epoch": 0.29192686780528176,
      "grad_norm": 27.925371412853583,
      "learning_rate": 4.4599186671569623e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.5,
      "logps/chosen": -302.0,
      "logps/rejected": -552.0,
      "loss": 0.4989,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 2.578125,
      "rewards/rejected": -4.3125,
      "step": 3880
    },
    {
      "epoch": 0.29267925663983146,
      "grad_norm": 21.547925586562368,
      "learning_rate": 4.4558355241859655e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.6875,
      "logps/chosen": -239.0,
      "logps/rejected": -516.0,
      "loss": 0.5172,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.3671875,
      "rewards/margins": 2.78125,
      "rewards/rejected": -4.15625,
      "step": 3890
    },
    {
      "epoch": 0.29343164547438116,
      "grad_norm": 26.245129523633555,
      "learning_rate": 4.451738888577705e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.484375,
      "logps/chosen": -270.0,
      "logps/rejected": -520.0,
      "loss": 0.5138,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.4609375,
      "rewards/margins": 2.59375,
      "rewards/rejected": -4.0625,
      "step": 3900
    },
    {
      "epoch": 0.29418403430893086,
      "grad_norm": 27.905382588551653,
      "learning_rate": 4.44762878859346e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.671875,
      "logps/chosen": -290.0,
      "logps/rejected": -516.0,
      "loss": 0.5086,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 2.46875,
      "rewards/rejected": -4.03125,
      "step": 3910
    },
    {
      "epoch": 0.29493642314348056,
      "grad_norm": 17.390882795992297,
      "learning_rate": 4.443505252587399e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.8125,
      "logps/chosen": -300.0,
      "logps/rejected": -596.0,
      "loss": 0.5196,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.125,
      "rewards/rejected": -4.875,
      "step": 3920
    },
    {
      "epoch": 0.29568881197803026,
      "grad_norm": 24.866209824761608,
      "learning_rate": 4.43936830900638e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.46875,
      "logps/chosen": -244.0,
      "logps/rejected": -556.0,
      "loss": 0.4753,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.484375,
      "rewards/margins": 2.96875,
      "rewards/rejected": -4.46875,
      "step": 3930
    },
    {
      "epoch": 0.29644120081257996,
      "grad_norm": 27.1926932271372,
      "learning_rate": 4.4352179863897556e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.5625,
      "logps/chosen": -276.0,
      "logps/rejected": -500.0,
      "loss": 0.5193,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.5234375,
      "rewards/margins": 2.421875,
      "rewards/rejected": -3.953125,
      "step": 3940
    },
    {
      "epoch": 0.29719358964712966,
      "grad_norm": 19.93421607384319,
      "learning_rate": 4.4310543133691746e-07,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.53125,
      "logps/chosen": -312.0,
      "logps/rejected": -600.0,
      "loss": 0.5141,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.65625,
      "rewards/margins": 3.0625,
      "rewards/rejected": -4.71875,
      "step": 3950
    },
    {
      "epoch": 0.2979459784816793,
      "grad_norm": 21.93471668197706,
      "learning_rate": 4.426877318668387e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.78125,
      "logps/chosen": -272.0,
      "logps/rejected": -576.0,
      "loss": 0.4937,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.59375,
      "rewards/margins": 3.078125,
      "rewards/rejected": -4.6875,
      "step": 3960
    },
    {
      "epoch": 0.298698367316229,
      "grad_norm": 17.1738612198678,
      "learning_rate": 4.422687031103045e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.671875,
      "logps/chosen": -308.0,
      "logps/rejected": -552.0,
      "loss": 0.5494,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 2.734375,
      "rewards/rejected": -4.40625,
      "step": 3970
    },
    {
      "epoch": 0.2994507561507787,
      "grad_norm": 20.73012490360146,
      "learning_rate": 4.4184834795805016e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.515625,
      "logps/chosen": -232.0,
      "logps/rejected": -506.0,
      "loss": 0.4731,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.2734375,
      "rewards/margins": 2.71875,
      "rewards/rejected": -3.984375,
      "step": 3980
    },
    {
      "epoch": 0.3002031449853284,
      "grad_norm": 20.535235211533763,
      "learning_rate": 4.414266693099615e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.671875,
      "logps/chosen": -288.0,
      "logps/rejected": -532.0,
      "loss": 0.5261,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.46875,
      "rewards/margins": 2.671875,
      "rewards/rejected": -4.15625,
      "step": 3990
    },
    {
      "epoch": 0.3009555338198781,
      "grad_norm": 20.414162518264927,
      "learning_rate": 4.4100367007505455e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.71875,
      "logps/chosen": -278.0,
      "logps/rejected": -580.0,
      "loss": 0.4817,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.625,
      "rewards/margins": 3.0,
      "rewards/rejected": -4.625,
      "step": 4000
    },
    {
      "epoch": 0.3009555338198781,
      "eval_logits/chosen": -2.515625,
      "eval_logits/rejected": -2.609375,
      "eval_logps/chosen": -290.0,
      "eval_logps/rejected": -544.0,
      "eval_loss": 0.28693443536758423,
      "eval_rewards/accuracies": 0.8762694597244263,
      "eval_rewards/chosen": -1.5859375,
      "eval_rewards/margins": 2.671875,
      "eval_rewards/rejected": -4.28125,
      "eval_runtime": 2668.9078,
      "eval_samples_per_second": 35.405,
      "eval_steps_per_second": 0.553,
      "step": 4000
    },
    {
      "epoch": 0.3017079226544278,
      "grad_norm": 21.378542333579798,
      "learning_rate": 4.405793531714558e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.796875,
      "logps/chosen": -258.0,
      "logps/rejected": -584.0,
      "loss": 0.4688,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.3984375,
      "rewards/margins": 3.3125,
      "rewards/rejected": -4.71875,
      "step": 4010
    },
    {
      "epoch": 0.3024603114889775,
      "grad_norm": 24.358947783310157,
      "learning_rate": 4.401537215263816e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.640625,
      "logps/chosen": -326.0,
      "logps/rejected": -600.0,
      "loss": 0.4647,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.8984375,
      "rewards/margins": 2.921875,
      "rewards/rejected": -4.8125,
      "step": 4020
    },
    {
      "epoch": 0.3032127003235272,
      "grad_norm": 17.974985125523347,
      "learning_rate": 4.3972677807611866e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5625,
      "logps/chosen": -298.0,
      "logps/rejected": -612.0,
      "loss": 0.4977,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.671875,
      "rewards/margins": 3.0,
      "rewards/rejected": -4.65625,
      "step": 4030
    },
    {
      "epoch": 0.3039650891580769,
      "grad_norm": 21.680134107383743,
      "learning_rate": 4.39298525766003e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.765625,
      "logps/chosen": -288.0,
      "logps/rejected": -564.0,
      "loss": 0.4683,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.59375,
      "rewards/margins": 2.9375,
      "rewards/rejected": -4.53125,
      "step": 4040
    },
    {
      "epoch": 0.3047174779926266,
      "grad_norm": 25.295201177739706,
      "learning_rate": 4.388689675504002e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.734375,
      "logps/chosen": -314.0,
      "logps/rejected": -596.0,
      "loss": 0.5418,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.71875,
      "rewards/margins": 2.921875,
      "rewards/rejected": -4.65625,
      "step": 4050
    },
    {
      "epoch": 0.3054698668271763,
      "grad_norm": 23.57937458288293,
      "learning_rate": 4.3843810639268475e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.640625,
      "logps/chosen": -288.0,
      "logps/rejected": -580.0,
      "loss": 0.5008,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 3.171875,
      "rewards/rejected": -4.75,
      "step": 4060
    },
    {
      "epoch": 0.306222255661726,
      "grad_norm": 20.079632092734936,
      "learning_rate": 4.3800594526521983e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.609375,
      "logps/chosen": -252.0,
      "logps/rejected": -528.0,
      "loss": 0.4879,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.4296875,
      "rewards/margins": 2.765625,
      "rewards/rejected": -4.1875,
      "step": 4070
    },
    {
      "epoch": 0.30697464449627565,
      "grad_norm": 20.954809179366347,
      "learning_rate": 4.375724871493365e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.609375,
      "logps/chosen": -264.0,
      "logps/rejected": -532.0,
      "loss": 0.5231,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.3984375,
      "rewards/margins": 2.859375,
      "rewards/rejected": -4.25,
      "step": 4080
    },
    {
      "epoch": 0.30772703333082535,
      "grad_norm": 18.139631925482703,
      "learning_rate": 4.371377350353134e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.5625,
      "logps/chosen": -266.0,
      "logps/rejected": -532.0,
      "loss": 0.505,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.515625,
      "rewards/margins": 2.734375,
      "rewards/rejected": -4.25,
      "step": 4090
    },
    {
      "epoch": 0.30847942216537505,
      "grad_norm": 29.303601603084495,
      "learning_rate": 4.3670169192235586e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.625,
      "logps/chosen": -266.0,
      "logps/rejected": -504.0,
      "loss": 0.5307,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.5234375,
      "rewards/margins": 2.4375,
      "rewards/rejected": -3.96875,
      "step": 4100
    },
    {
      "epoch": 0.30923181099992475,
      "grad_norm": 19.466582796291934,
      "learning_rate": 4.362643608185757e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.515625,
      "logps/chosen": -294.0,
      "logps/rejected": -540.0,
      "loss": 0.5253,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.484375,
      "rewards/margins": 2.578125,
      "rewards/rejected": -4.0625,
      "step": 4110
    },
    {
      "epoch": 0.30998419983447445,
      "grad_norm": 25.436331142190024,
      "learning_rate": 4.358257447409698e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.65625,
      "logps/chosen": -280.0,
      "logps/rejected": -576.0,
      "loss": 0.4844,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.625,
      "rewards/margins": 2.921875,
      "rewards/rejected": -4.5625,
      "step": 4120
    },
    {
      "epoch": 0.31073658866902415,
      "grad_norm": 17.941621637224088,
      "learning_rate": 4.353858467153998e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.609375,
      "logps/chosen": -280.0,
      "logps/rejected": -616.0,
      "loss": 0.5212,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 3.46875,
      "rewards/rejected": -5.09375,
      "step": 4130
    },
    {
      "epoch": 0.31148897750357385,
      "grad_norm": 18.456515618542245,
      "learning_rate": 4.349446697765711e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.765625,
      "logps/chosen": -270.0,
      "logps/rejected": -544.0,
      "loss": 0.5135,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.4921875,
      "rewards/margins": 2.921875,
      "rewards/rejected": -4.40625,
      "step": 4140
    },
    {
      "epoch": 0.31224136633812355,
      "grad_norm": 20.881902874729686,
      "learning_rate": 4.345022169680117e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5625,
      "logps/chosen": -242.0,
      "logps/rejected": -528.0,
      "loss": 0.4627,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.3515625,
      "rewards/margins": 2.828125,
      "rewards/rejected": -4.1875,
      "step": 4150
    },
    {
      "epoch": 0.31299375517267325,
      "grad_norm": 17.33859044942403,
      "learning_rate": 4.3405849134205164e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.703125,
      "logps/chosen": -288.0,
      "logps/rejected": -568.0,
      "loss": 0.5167,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 2.765625,
      "rewards/rejected": -4.4375,
      "step": 4160
    },
    {
      "epoch": 0.31374614400722295,
      "grad_norm": 26.44811003085519,
      "learning_rate": 4.3361349595980136e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.671875,
      "logps/chosen": -294.0,
      "logps/rejected": -636.0,
      "loss": 0.4541,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.640625,
      "rewards/margins": 3.421875,
      "rewards/rejected": -5.0625,
      "step": 4170
    },
    {
      "epoch": 0.31449853284177265,
      "grad_norm": 23.075606871180597,
      "learning_rate": 4.331672338911313e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.59375,
      "logps/chosen": -290.0,
      "logps/rejected": -604.0,
      "loss": 0.4914,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 3.21875,
      "rewards/rejected": -4.90625,
      "step": 4180
    },
    {
      "epoch": 0.31525092167632235,
      "grad_norm": 27.46128965425931,
      "learning_rate": 4.3271970821465e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.65625,
      "logps/chosen": -294.0,
      "logps/rejected": -564.0,
      "loss": 0.5046,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.609375,
      "rewards/margins": 2.734375,
      "rewards/rejected": -4.34375,
      "step": 4190
    },
    {
      "epoch": 0.316003310510872,
      "grad_norm": 23.231004121861403,
      "learning_rate": 4.3227092201768346e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.53125,
      "logps/chosen": -274.0,
      "logps/rejected": -532.0,
      "loss": 0.5164,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 2.640625,
      "rewards/rejected": -4.25,
      "step": 4200
    },
    {
      "epoch": 0.3167556993454217,
      "grad_norm": 20.175969546787517,
      "learning_rate": 4.3182087839625326e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.484375,
      "logps/chosen": -290.0,
      "logps/rejected": -572.0,
      "loss": 0.5024,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 2.828125,
      "rewards/rejected": -4.53125,
      "step": 4210
    },
    {
      "epoch": 0.3175080881799714,
      "grad_norm": 17.601388622050404,
      "learning_rate": 4.313695804550559e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.640625,
      "logps/chosen": -284.0,
      "logps/rejected": -584.0,
      "loss": 0.4604,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 2.859375,
      "rewards/rejected": -4.46875,
      "step": 4220
    },
    {
      "epoch": 0.3182604770145211,
      "grad_norm": 19.917898839893788,
      "learning_rate": 4.309170313074408e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.53125,
      "logps/chosen": -294.0,
      "logps/rejected": -588.0,
      "loss": 0.5314,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 2.75,
      "rewards/rejected": -4.46875,
      "step": 4230
    },
    {
      "epoch": 0.3190128658490708,
      "grad_norm": 17.83008301543401,
      "learning_rate": 4.304632340753889e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.578125,
      "logps/chosen": -266.0,
      "logps/rejected": -624.0,
      "loss": 0.4978,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.4765625,
      "rewards/margins": 3.484375,
      "rewards/rejected": -4.9375,
      "step": 4240
    },
    {
      "epoch": 0.3197652546836205,
      "grad_norm": 18.86518679780084,
      "learning_rate": 4.3000819188949145e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.734375,
      "logps/chosen": -280.0,
      "logps/rejected": -500.0,
      "loss": 0.4899,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 2.359375,
      "rewards/rejected": -4.0,
      "step": 4250
    },
    {
      "epoch": 0.3205176435181702,
      "grad_norm": 20.146066202963087,
      "learning_rate": 4.2955190788892827e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.734375,
      "logps/chosen": -276.0,
      "logps/rejected": -524.0,
      "loss": 0.5114,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.53125,
      "rewards/margins": 2.609375,
      "rewards/rejected": -4.125,
      "step": 4260
    },
    {
      "epoch": 0.3212700323527199,
      "grad_norm": 17.61802525059847,
      "learning_rate": 4.2909438522144566e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.46875,
      "logps/chosen": -264.0,
      "logps/rejected": -560.0,
      "loss": 0.4945,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4921875,
      "rewards/margins": 2.859375,
      "rewards/rejected": -4.34375,
      "step": 4270
    },
    {
      "epoch": 0.3220224211872696,
      "grad_norm": 20.861445779388053,
      "learning_rate": 4.2863562704333545e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.625,
      "logps/chosen": -272.0,
      "logps/rejected": -544.0,
      "loss": 0.4563,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 2.828125,
      "rewards/rejected": -4.28125,
      "step": 4280
    },
    {
      "epoch": 0.3227748100218193,
      "grad_norm": 24.89199118014932,
      "learning_rate": 4.2817563651941263e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.671875,
      "logps/chosen": -276.0,
      "logps/rejected": -548.0,
      "loss": 0.4852,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 2.8125,
      "rewards/rejected": -4.40625,
      "step": 4290
    },
    {
      "epoch": 0.323527198856369,
      "grad_norm": 23.01643360779954,
      "learning_rate": 4.277144168229939e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.4375,
      "logps/chosen": -266.0,
      "logps/rejected": -552.0,
      "loss": 0.5267,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.484375,
      "rewards/margins": 2.828125,
      "rewards/rejected": -4.3125,
      "step": 4300
    },
    {
      "epoch": 0.3242795876909187,
      "grad_norm": 14.901466107553938,
      "learning_rate": 4.272519711358753e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.609375,
      "logps/chosen": -247.0,
      "logps/rejected": -524.0,
      "loss": 0.4313,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.328125,
      "rewards/margins": 2.796875,
      "rewards/rejected": -4.125,
      "step": 4310
    },
    {
      "epoch": 0.32503197652546834,
      "grad_norm": 24.253979897740447,
      "learning_rate": 4.2678830264831077e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.671875,
      "logps/chosen": -282.0,
      "logps/rejected": -572.0,
      "loss": 0.4875,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.4765625,
      "rewards/margins": 2.96875,
      "rewards/rejected": -4.4375,
      "step": 4320
    },
    {
      "epoch": 0.32578436536001804,
      "grad_norm": 21.620370572175357,
      "learning_rate": 4.2632341455899e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.703125,
      "logps/chosen": -298.0,
      "logps/rejected": -580.0,
      "loss": 0.4715,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 2.875,
      "rewards/rejected": -4.71875,
      "step": 4330
    },
    {
      "epoch": 0.32653675419456774,
      "grad_norm": 18.80473208594618,
      "learning_rate": 4.25857310075016e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.71875,
      "logps/chosen": -272.0,
      "logps/rejected": -560.0,
      "loss": 0.5102,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.46875,
      "rewards/margins": 2.96875,
      "rewards/rejected": -4.4375,
      "step": 4340
    },
    {
      "epoch": 0.32728914302911744,
      "grad_norm": 23.438339173330064,
      "learning_rate": 4.2538999241188374e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.4375,
      "logps/chosen": -280.0,
      "logps/rejected": -544.0,
      "loss": 0.4514,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 2.875,
      "rewards/rejected": -4.5,
      "step": 4350
    },
    {
      "epoch": 0.32804153186366714,
      "grad_norm": 20.775746782576963,
      "learning_rate": 4.2492146479345693e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.359375,
      "logps/chosen": -292.0,
      "logps/rejected": -612.0,
      "loss": 0.4631,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.7578125,
      "rewards/margins": 3.1875,
      "rewards/rejected": -4.9375,
      "step": 4360
    },
    {
      "epoch": 0.32879392069821683,
      "grad_norm": 22.2374083545146,
      "learning_rate": 4.244517304519467e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.546875,
      "logps/chosen": -270.0,
      "logps/rejected": -624.0,
      "loss": 0.4952,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 3.328125,
      "rewards/rejected": -4.96875,
      "step": 4370
    },
    {
      "epoch": 0.32954630953276653,
      "grad_norm": 22.255023136667525,
      "learning_rate": 4.2398079262788893e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.671875,
      "logps/chosen": -274.0,
      "logps/rejected": -556.0,
      "loss": 0.4569,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 2.828125,
      "rewards/rejected": -4.4375,
      "step": 4380
    },
    {
      "epoch": 0.33029869836731623,
      "grad_norm": 19.890278499605557,
      "learning_rate": 4.2350865457012184e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -258.0,
      "logps/rejected": -564.0,
      "loss": 0.4631,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.390625,
      "rewards/margins": 3.140625,
      "rewards/rejected": -4.53125,
      "step": 4390
    },
    {
      "epoch": 0.33105108720186593,
      "grad_norm": 21.531081628130337,
      "learning_rate": 4.2303531953576366e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.546875,
      "logps/chosen": -274.0,
      "logps/rejected": -600.0,
      "loss": 0.4757,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.453125,
      "rewards/margins": 3.28125,
      "rewards/rejected": -4.71875,
      "step": 4400
    },
    {
      "epoch": 0.33180347603641563,
      "grad_norm": 23.27833028506438,
      "learning_rate": 4.2256079079019015e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.703125,
      "logps/chosen": -262.0,
      "logps/rejected": -576.0,
      "loss": 0.53,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 3.125,
      "rewards/rejected": -4.65625,
      "step": 4410
    },
    {
      "epoch": 0.33255586487096533,
      "grad_norm": 19.992555736521695,
      "learning_rate": 4.220850716070121e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.703125,
      "logps/chosen": -274.0,
      "logps/rejected": -548.0,
      "loss": 0.4696,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.5234375,
      "rewards/margins": 2.9375,
      "rewards/rejected": -4.46875,
      "step": 4420
    },
    {
      "epoch": 0.33330825370551503,
      "grad_norm": 21.564549875247046,
      "learning_rate": 4.2160816526805267e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.515625,
      "logps/chosen": -262.0,
      "logps/rejected": -580.0,
      "loss": 0.4804,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 3.125,
      "rewards/rejected": -4.65625,
      "step": 4430
    },
    {
      "epoch": 0.3340606425400647,
      "grad_norm": 26.551180226982073,
      "learning_rate": 4.211300750633248e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.515625,
      "logps/chosen": -298.0,
      "logps/rejected": -596.0,
      "loss": 0.4578,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.78125,
      "rewards/margins": 2.984375,
      "rewards/rejected": -4.75,
      "step": 4440
    },
    {
      "epoch": 0.3348130313746144,
      "grad_norm": 18.092058679497374,
      "learning_rate": 4.2065080429100865e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.5,
      "logps/chosen": -316.0,
      "logps/rejected": -628.0,
      "loss": 0.4887,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.921875,
      "rewards/margins": 3.078125,
      "rewards/rejected": -5.0,
      "step": 4450
    },
    {
      "epoch": 0.3355654202091641,
      "grad_norm": 20.590357972470507,
      "learning_rate": 4.2017035625742846e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.5625,
      "logps/chosen": -302.0,
      "logps/rejected": -616.0,
      "loss": 0.4588,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.09375,
      "rewards/rejected": -4.96875,
      "step": 4460
    },
    {
      "epoch": 0.3363178090437138,
      "grad_norm": 17.15219860330965,
      "learning_rate": 4.196887342770302e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.796875,
      "logps/chosen": -306.0,
      "logps/rejected": -612.0,
      "loss": 0.4502,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.671875,
      "rewards/margins": 3.265625,
      "rewards/rejected": -4.9375,
      "step": 4470
    },
    {
      "epoch": 0.3370701978782635,
      "grad_norm": 18.53794528398302,
      "learning_rate": 4.1920594167235845e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.53125,
      "logps/chosen": -296.0,
      "logps/rejected": -568.0,
      "loss": 0.534,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 2.90625,
      "rewards/rejected": -4.5625,
      "step": 4480
    },
    {
      "epoch": 0.3378225867128132,
      "grad_norm": 22.38361022724724,
      "learning_rate": 4.187219817740336e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.625,
      "logps/chosen": -280.0,
      "logps/rejected": -572.0,
      "loss": 0.4849,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 2.84375,
      "rewards/rejected": -4.375,
      "step": 4490
    },
    {
      "epoch": 0.3385749755473629,
      "grad_norm": 18.857971639717416,
      "learning_rate": 4.182368579207285e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.3125,
      "logps/chosen": -318.0,
      "logps/rejected": -608.0,
      "loss": 0.5074,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.015625,
      "rewards/rejected": -4.875,
      "step": 4500
    },
    {
      "epoch": 0.3393273643819126,
      "grad_norm": 24.122394882362272,
      "learning_rate": 4.177505734591461e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.796875,
      "logps/chosen": -278.0,
      "logps/rejected": -568.0,
      "loss": 0.4798,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.4609375,
      "rewards/margins": 3.0,
      "rewards/rejected": -4.46875,
      "step": 4510
    },
    {
      "epoch": 0.3400797532164623,
      "grad_norm": 22.963215626235648,
      "learning_rate": 4.172631317439956e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.46875,
      "logps/chosen": -278.0,
      "logps/rejected": -568.0,
      "loss": 0.4971,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.421875,
      "rewards/margins": 2.875,
      "rewards/rejected": -4.3125,
      "step": 4520
    },
    {
      "epoch": 0.340832142051012,
      "grad_norm": 25.29841943977269,
      "learning_rate": 4.167745361379702e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.65625,
      "logps/chosen": -288.0,
      "logps/rejected": -564.0,
      "loss": 0.4618,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 2.796875,
      "rewards/rejected": -4.5,
      "step": 4530
    },
    {
      "epoch": 0.3415845308855617,
      "grad_norm": 21.477934813704486,
      "learning_rate": 4.162847900117229e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.3125,
      "logps/chosen": -286.0,
      "logps/rejected": -600.0,
      "loss": 0.4721,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 2.96875,
      "rewards/rejected": -4.625,
      "step": 4540
    },
    {
      "epoch": 0.3423369197201114,
      "grad_norm": 19.84585598462481,
      "learning_rate": 4.1579389674384394e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.625,
      "logps/chosen": -292.0,
      "logps/rejected": -540.0,
      "loss": 0.4774,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 2.65625,
      "rewards/rejected": -4.34375,
      "step": 4550
    },
    {
      "epoch": 0.3430893085546611,
      "grad_norm": 16.285968113146385,
      "learning_rate": 4.153018597208374e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.796875,
      "logps/chosen": -262.0,
      "logps/rejected": -564.0,
      "loss": 0.4505,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.484375,
      "rewards/margins": 3.046875,
      "rewards/rejected": -4.53125,
      "step": 4560
    },
    {
      "epoch": 0.3438416973892107,
      "grad_norm": 25.428967203867806,
      "learning_rate": 4.1480868233709765e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.765625,
      "logps/chosen": -322.0,
      "logps/rejected": -648.0,
      "loss": 0.4531,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 3.453125,
      "rewards/rejected": -5.25,
      "step": 4570
    },
    {
      "epoch": 0.3445940862237604,
      "grad_norm": 29.961551781367056,
      "learning_rate": 4.1431436799488606e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.796875,
      "logps/chosen": -316.0,
      "logps/rejected": -640.0,
      "loss": 0.4757,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 3.328125,
      "rewards/rejected": -5.25,
      "step": 4580
    },
    {
      "epoch": 0.3453464750583101,
      "grad_norm": 18.107768001184667,
      "learning_rate": 4.1381892010430717e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.46875,
      "logps/chosen": -296.0,
      "logps/rejected": -588.0,
      "loss": 0.5065,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 3.0,
      "rewards/rejected": -4.6875,
      "step": 4590
    },
    {
      "epoch": 0.3460988638928598,
      "grad_norm": 23.009212255848063,
      "learning_rate": 4.13322342083286e-07,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.53125,
      "logps/chosen": -332.0,
      "logps/rejected": -620.0,
      "loss": 0.4918,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 3.140625,
      "rewards/rejected": -4.9375,
      "step": 4600
    },
    {
      "epoch": 0.3468512527274095,
      "grad_norm": 28.580472541805158,
      "learning_rate": 4.1282463735754356e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.734375,
      "logps/chosen": -306.0,
      "logps/rejected": -592.0,
      "loss": 0.5218,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 2.875,
      "rewards/rejected": -4.6875,
      "step": 4610
    },
    {
      "epoch": 0.3476036415619592,
      "grad_norm": 24.40461967455721,
      "learning_rate": 4.123258093605739e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.53125,
      "logps/chosen": -312.0,
      "logps/rejected": -624.0,
      "loss": 0.4941,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.328125,
      "rewards/rejected": -5.09375,
      "step": 4620
    },
    {
      "epoch": 0.3483560303965089,
      "grad_norm": 19.880481080284294,
      "learning_rate": 4.118258615336199e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.578125,
      "logps/chosen": -290.0,
      "logps/rejected": -560.0,
      "loss": 0.509,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 2.765625,
      "rewards/rejected": -4.34375,
      "step": 4630
    },
    {
      "epoch": 0.3491084192310586,
      "grad_norm": 14.947995562055405,
      "learning_rate": 4.1132479732564964e-07,
      "logits/chosen": -2.375,
      "logits/rejected": -2.4375,
      "logps/chosen": -282.0,
      "logps/rejected": -604.0,
      "loss": 0.4201,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 3.078125,
      "rewards/rejected": -4.625,
      "step": 4640
    },
    {
      "epoch": 0.3498608080656083,
      "grad_norm": 27.222540074282293,
      "learning_rate": 4.1082262019333315e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.421875,
      "logps/chosen": -268.0,
      "logps/rejected": -604.0,
      "loss": 0.4796,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.484375,
      "rewards/margins": 3.203125,
      "rewards/rejected": -4.6875,
      "step": 4650
    },
    {
      "epoch": 0.350613196900158,
      "grad_norm": 20.935329162549365,
      "learning_rate": 4.103193336010179e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.765625,
      "logps/chosen": -300.0,
      "logps/rejected": -572.0,
      "loss": 0.4624,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.609375,
      "rewards/margins": 3.09375,
      "rewards/rejected": -4.71875,
      "step": 4660
    },
    {
      "epoch": 0.3513655857347077,
      "grad_norm": 19.39470631380493,
      "learning_rate": 4.098149410207051e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.578125,
      "logps/chosen": -274.0,
      "logps/rejected": -588.0,
      "loss": 0.4798,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.59375,
      "rewards/margins": 3.0625,
      "rewards/rejected": -4.65625,
      "step": 4670
    },
    {
      "epoch": 0.3521179745692574,
      "grad_norm": 19.831740214848217,
      "learning_rate": 4.09309445932026e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -302.0,
      "logps/rejected": -580.0,
      "loss": 0.5193,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8984375,
      "rewards/margins": 2.75,
      "rewards/rejected": -4.65625,
      "step": 4680
    },
    {
      "epoch": 0.35287036340380706,
      "grad_norm": 20.269846575264374,
      "learning_rate": 4.088028518222173e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.65625,
      "logps/chosen": -276.0,
      "logps/rejected": -564.0,
      "loss": 0.4798,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.5,
      "rewards/margins": 3.109375,
      "rewards/rejected": -4.625,
      "step": 4690
    },
    {
      "epoch": 0.35362275223835676,
      "grad_norm": 17.99579848930179,
      "learning_rate": 4.0829516218609785e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.59375,
      "logps/chosen": -300.0,
      "logps/rejected": -640.0,
      "loss": 0.4684,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.640625,
      "rewards/margins": 3.53125,
      "rewards/rejected": -5.15625,
      "step": 4700
    },
    {
      "epoch": 0.35437514107290646,
      "grad_norm": 19.018471752495422,
      "learning_rate": 4.077863805260439e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.828125,
      "logps/chosen": -310.0,
      "logps/rejected": -580.0,
      "loss": 0.4723,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.609375,
      "rewards/margins": 3.046875,
      "rewards/rejected": -4.65625,
      "step": 4710
    },
    {
      "epoch": 0.35512752990745616,
      "grad_norm": 18.489926097237685,
      "learning_rate": 4.0727651035196545e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.484375,
      "logps/chosen": -284.0,
      "logps/rejected": -680.0,
      "loss": 0.4715,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.625,
      "rewards/margins": 3.609375,
      "rewards/rejected": -5.25,
      "step": 4720
    },
    {
      "epoch": 0.35587991874200586,
      "grad_norm": 25.141327195133307,
      "learning_rate": 4.0676555518128157e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.671875,
      "logps/chosen": -318.0,
      "logps/rejected": -580.0,
      "loss": 0.5321,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 2.703125,
      "rewards/rejected": -4.625,
      "step": 4730
    },
    {
      "epoch": 0.35663230757655556,
      "grad_norm": 29.504900491559653,
      "learning_rate": 4.062535185388964e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.78125,
      "logps/chosen": -268.0,
      "logps/rejected": -544.0,
      "loss": 0.525,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 2.875,
      "rewards/rejected": -4.3125,
      "step": 4740
    },
    {
      "epoch": 0.35738469641110526,
      "grad_norm": 19.79113081144493,
      "learning_rate": 4.0574040395717493e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.6875,
      "logps/chosen": -292.0,
      "logps/rejected": -612.0,
      "loss": 0.4476,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.59375,
      "rewards/margins": 3.21875,
      "rewards/rejected": -4.8125,
      "step": 4750
    },
    {
      "epoch": 0.35813708524565496,
      "grad_norm": 19.130666803773927,
      "learning_rate": 4.0522621497591813e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.765625,
      "logps/chosen": -258.0,
      "logps/rejected": -568.0,
      "loss": 0.4957,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.4609375,
      "rewards/margins": 3.125,
      "rewards/rejected": -4.59375,
      "step": 4760
    },
    {
      "epoch": 0.35888947408020466,
      "grad_norm": 19.545461988917094,
      "learning_rate": 4.0471095514233933e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.484375,
      "logps/chosen": -280.0,
      "logps/rejected": -600.0,
      "loss": 0.4776,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.640625,
      "rewards/margins": 3.09375,
      "rewards/rejected": -4.75,
      "step": 4770
    },
    {
      "epoch": 0.35964186291475436,
      "grad_norm": 19.56988594200999,
      "learning_rate": 4.041946280110389e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.65625,
      "logps/chosen": -272.0,
      "logps/rejected": -592.0,
      "loss": 0.4457,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.578125,
      "rewards/margins": 3.265625,
      "rewards/rejected": -4.84375,
      "step": 4780
    },
    {
      "epoch": 0.36039425174930406,
      "grad_norm": 19.44126708157583,
      "learning_rate": 4.0367723714398047e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.8125,
      "logps/chosen": -300.0,
      "logps/rejected": -712.0,
      "loss": 0.4393,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -5.84375,
      "step": 4790
    },
    {
      "epoch": 0.36114664058385376,
      "grad_norm": 22.795735777319713,
      "learning_rate": 4.031587861104657e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.859375,
      "logps/chosen": -312.0,
      "logps/rejected": -632.0,
      "loss": 0.4962,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.90625,
      "rewards/margins": 3.265625,
      "rewards/rejected": -5.15625,
      "step": 4800
    },
    {
      "epoch": 0.3618990294184034,
      "grad_norm": 20.66774968855472,
      "learning_rate": 4.0263927848711026e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.703125,
      "logps/chosen": -298.0,
      "logps/rejected": -608.0,
      "loss": 0.5118,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 3.3125,
      "rewards/rejected": -5.0,
      "step": 4810
    },
    {
      "epoch": 0.3626514182529531,
      "grad_norm": 20.152432187791934,
      "learning_rate": 4.021187178578187e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.75,
      "logps/chosen": -276.0,
      "logps/rejected": -600.0,
      "loss": 0.4398,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.578125,
      "rewards/margins": 3.25,
      "rewards/rejected": -4.84375,
      "step": 4820
    },
    {
      "epoch": 0.3634038070875028,
      "grad_norm": 21.73376167276531,
      "learning_rate": 4.0159710781375994e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.734375,
      "logps/chosen": -310.0,
      "logps/rejected": -576.0,
      "loss": 0.487,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.90625,
      "rewards/margins": 2.703125,
      "rewards/rejected": -4.59375,
      "step": 4830
    },
    {
      "epoch": 0.3641561959220525,
      "grad_norm": 25.685488060312174,
      "learning_rate": 4.0107445195334244e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.84375,
      "logps/chosen": -326.0,
      "logps/rejected": -628.0,
      "loss": 0.4629,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 3.234375,
      "rewards/rejected": -5.15625,
      "step": 4840
    },
    {
      "epoch": 0.3649085847566022,
      "grad_norm": 17.25280951339148,
      "learning_rate": 4.005507538821894e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.59375,
      "logps/chosen": -288.0,
      "logps/rejected": -648.0,
      "loss": 0.4554,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.4375,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.125,
      "step": 4850
    },
    {
      "epoch": 0.3656609735911519,
      "grad_norm": 19.055701026186203,
      "learning_rate": 4.0002601721311393e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.515625,
      "logps/chosen": -286.0,
      "logps/rejected": -604.0,
      "loss": 0.4397,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 3.359375,
      "rewards/rejected": -4.9375,
      "step": 4860
    },
    {
      "epoch": 0.3664133624257016,
      "grad_norm": 25.50324409323137,
      "learning_rate": 3.995002455660939e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.859375,
      "logps/chosen": -264.0,
      "logps/rejected": -608.0,
      "loss": 0.4313,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.4296875,
      "rewards/margins": 3.5,
      "rewards/rejected": -4.9375,
      "step": 4870
    },
    {
      "epoch": 0.3671657512602513,
      "grad_norm": 23.62316726532893,
      "learning_rate": 3.989734425682473e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.9375,
      "logps/chosen": -296.0,
      "logps/rejected": -608.0,
      "loss": 0.4537,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.21875,
      "rewards/rejected": -5.03125,
      "step": 4880
    },
    {
      "epoch": 0.367918140094801,
      "grad_norm": 20.023091746603825,
      "learning_rate": 3.984456118538071e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.75,
      "logps/chosen": -294.0,
      "logps/rejected": -636.0,
      "loss": 0.4608,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.7578125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -5.09375,
      "step": 4890
    },
    {
      "epoch": 0.3686705289293507,
      "grad_norm": 22.483992055099264,
      "learning_rate": 3.9791675706409593e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.6875,
      "logps/chosen": -276.0,
      "logps/rejected": -680.0,
      "loss": 0.4931,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -5.625,
      "step": 4900
    },
    {
      "epoch": 0.3694229177639004,
      "grad_norm": 23.417557416767927,
      "learning_rate": 3.9738688184750125e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.75,
      "logps/chosen": -302.0,
      "logps/rejected": -604.0,
      "loss": 0.4561,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 3.125,
      "rewards/rejected": -4.75,
      "step": 4910
    },
    {
      "epoch": 0.3701753065984501,
      "grad_norm": 21.690395280725046,
      "learning_rate": 3.968559898594502e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -3.046875,
      "logps/chosen": -242.0,
      "logps/rejected": -532.0,
      "loss": 0.4947,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4140625,
      "rewards/margins": 3.015625,
      "rewards/rejected": -4.4375,
      "step": 4920
    },
    {
      "epoch": 0.37092769543299975,
      "grad_norm": 16.078703965976786,
      "learning_rate": 3.9632408476238416e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.515625,
      "logps/chosen": -292.0,
      "logps/rejected": -544.0,
      "loss": 0.4573,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 2.53125,
      "rewards/rejected": -4.25,
      "step": 4930
    },
    {
      "epoch": 0.37168008426754945,
      "grad_norm": 24.789464985858757,
      "learning_rate": 3.957911702257337e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.71875,
      "logps/chosen": -292.0,
      "logps/rejected": -636.0,
      "loss": 0.4597,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 3.515625,
      "rewards/rejected": -5.03125,
      "step": 4940
    },
    {
      "epoch": 0.37243247310209915,
      "grad_norm": 21.96309849468352,
      "learning_rate": 3.952572499258929e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.78125,
      "logps/chosen": -288.0,
      "logps/rejected": -592.0,
      "loss": 0.4696,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 3.203125,
      "rewards/rejected": -4.875,
      "step": 4950
    },
    {
      "epoch": 0.37318486193664885,
      "grad_norm": 20.01542824526465,
      "learning_rate": 3.9472232754619474e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.703125,
      "logps/chosen": -292.0,
      "logps/rejected": -588.0,
      "loss": 0.464,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.578125,
      "rewards/margins": 3.203125,
      "rewards/rejected": -4.78125,
      "step": 4960
    },
    {
      "epoch": 0.37393725077119855,
      "grad_norm": 18.59159801592601,
      "learning_rate": 3.9418640677688464e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.515625,
      "logps/chosen": -304.0,
      "logps/rejected": -544.0,
      "loss": 0.4993,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.65625,
      "rewards/margins": 2.609375,
      "rewards/rejected": -4.28125,
      "step": 4970
    },
    {
      "epoch": 0.37468963960574825,
      "grad_norm": 28.664682350781625,
      "learning_rate": 3.936494913150961e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.46875,
      "logps/chosen": -290.0,
      "logps/rejected": -528.0,
      "loss": 0.449,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.578125,
      "rewards/margins": 2.578125,
      "rewards/rejected": -4.15625,
      "step": 4980
    },
    {
      "epoch": 0.37544202844029795,
      "grad_norm": 20.551864722466945,
      "learning_rate": 3.931115848648242e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.859375,
      "logps/chosen": -282.0,
      "logps/rejected": -600.0,
      "loss": 0.4381,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.625,
      "rewards/margins": 3.3125,
      "rewards/rejected": -4.9375,
      "step": 4990
    },
    {
      "epoch": 0.37619441727484765,
      "grad_norm": 20.33578038413793,
      "learning_rate": 3.925726911369008e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.921875,
      "logps/chosen": -302.0,
      "logps/rejected": -596.0,
      "loss": 0.46,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 3.078125,
      "rewards/rejected": -4.8125,
      "step": 5000
    },
    {
      "epoch": 0.37619441727484765,
      "eval_logits/chosen": -2.671875,
      "eval_logits/rejected": -2.765625,
      "eval_logps/chosen": -330.0,
      "eval_logps/rejected": -620.0,
      "eval_loss": 0.2789464294910431,
      "eval_rewards/accuracies": 0.8784698843955994,
      "eval_rewards/chosen": -1.9765625,
      "eval_rewards/margins": 3.046875,
      "eval_rewards/rejected": -5.03125,
      "eval_runtime": 2668.066,
      "eval_samples_per_second": 35.416,
      "eval_steps_per_second": 0.554,
      "step": 5000
    },
    {
      "epoch": 0.37694680610939735,
      "grad_norm": 19.08794989080311,
      "learning_rate": 3.9203281384896856e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.796875,
      "logps/chosen": -294.0,
      "logps/rejected": -572.0,
      "loss": 0.4756,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 2.90625,
      "rewards/rejected": -4.71875,
      "step": 5010
    },
    {
      "epoch": 0.37769919494394705,
      "grad_norm": 28.375865434506924,
      "learning_rate": 3.9149195672545547e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.4375,
      "logps/chosen": -296.0,
      "logps/rejected": -588.0,
      "loss": 0.4964,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.734375,
      "rewards/margins": 2.921875,
      "rewards/rejected": -4.65625,
      "step": 5020
    },
    {
      "epoch": 0.37845158377849675,
      "grad_norm": 16.748857617307703,
      "learning_rate": 3.9095012349754897e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.546875,
      "logps/chosen": -280.0,
      "logps/rejected": -580.0,
      "loss": 0.4668,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 3.15625,
      "rewards/rejected": -4.71875,
      "step": 5030
    },
    {
      "epoch": 0.37920397261304645,
      "grad_norm": 19.68051630474057,
      "learning_rate": 3.904073179031702e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.546875,
      "logps/chosen": -278.0,
      "logps/rejected": -600.0,
      "loss": 0.4284,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.4375,
      "rewards/margins": 3.203125,
      "rewards/rejected": -4.625,
      "step": 5040
    },
    {
      "epoch": 0.3799563614475961,
      "grad_norm": 18.911126910009518,
      "learning_rate": 3.898635436869485e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.734375,
      "logps/chosen": -290.0,
      "logps/rejected": -592.0,
      "loss": 0.4629,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.125,
      "rewards/rejected": -4.875,
      "step": 5050
    },
    {
      "epoch": 0.3807087502821458,
      "grad_norm": 34.88019788014345,
      "learning_rate": 3.8931880460019537e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.8125,
      "logps/chosen": -286.0,
      "logps/rejected": -540.0,
      "loss": 0.484,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.71875,
      "rewards/margins": 2.625,
      "rewards/rejected": -4.34375,
      "step": 5060
    },
    {
      "epoch": 0.3814611391166955,
      "grad_norm": 20.981980522767312,
      "learning_rate": 3.887731044008787e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.65625,
      "logps/chosen": -296.0,
      "logps/rejected": -588.0,
      "loss": 0.4358,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.7578125,
      "rewards/margins": 2.859375,
      "rewards/rejected": -4.625,
      "step": 5070
    },
    {
      "epoch": 0.3822135279512452,
      "grad_norm": 23.986480403807217,
      "learning_rate": 3.8822644685359655e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.8125,
      "logps/chosen": -294.0,
      "logps/rejected": -640.0,
      "loss": 0.5079,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.703125,
      "rewards/margins": 3.4375,
      "rewards/rejected": -5.15625,
      "step": 5080
    },
    {
      "epoch": 0.3829659167857949,
      "grad_norm": 25.733823977911168,
      "learning_rate": 3.876788357295516e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -278.0,
      "logps/rejected": -592.0,
      "loss": 0.4493,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 3.125,
      "rewards/rejected": -4.78125,
      "step": 5090
    },
    {
      "epoch": 0.3837183056203446,
      "grad_norm": 21.58237053292917,
      "learning_rate": 3.871302748065246e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.671875,
      "logps/chosen": -248.0,
      "logps/rejected": -632.0,
      "loss": 0.439,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.4765625,
      "rewards/margins": 3.453125,
      "rewards/rejected": -4.90625,
      "step": 5100
    },
    {
      "epoch": 0.3844706944548943,
      "grad_norm": 24.104499838599086,
      "learning_rate": 3.8658076786884917e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.96875,
      "logps/chosen": -284.0,
      "logps/rejected": -640.0,
      "loss": 0.4356,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -5.25,
      "step": 5110
    },
    {
      "epoch": 0.385223083289444,
      "grad_norm": 24.545168773967603,
      "learning_rate": 3.860303187073848e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.796875,
      "logps/chosen": -306.0,
      "logps/rejected": -668.0,
      "loss": 0.4186,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.53125,
      "step": 5120
    },
    {
      "epoch": 0.3859754721239937,
      "grad_norm": 21.151626469886512,
      "learning_rate": 3.854789311194909e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.734375,
      "logps/chosen": -282.0,
      "logps/rejected": -540.0,
      "loss": 0.46,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.625,
      "rewards/margins": 2.75,
      "rewards/rejected": -4.375,
      "step": 5130
    },
    {
      "epoch": 0.3867278609585434,
      "grad_norm": 21.751334881010287,
      "learning_rate": 3.849266089090013e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.953125,
      "logps/chosen": -282.0,
      "logps/rejected": -596.0,
      "loss": 0.4456,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 3.140625,
      "rewards/rejected": -4.71875,
      "step": 5140
    },
    {
      "epoch": 0.3874802497930931,
      "grad_norm": 21.49022175784225,
      "learning_rate": 3.843733558861969e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -286.0,
      "logps/rejected": -584.0,
      "loss": 0.4438,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 2.96875,
      "rewards/rejected": -4.625,
      "step": 5150
    },
    {
      "epoch": 0.3882326386276428,
      "grad_norm": 19.464208454739303,
      "learning_rate": 3.8381917586778036e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.65625,
      "logps/chosen": -266.0,
      "logps/rejected": -552.0,
      "loss": 0.4725,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.484375,
      "rewards/margins": 2.953125,
      "rewards/rejected": -4.4375,
      "step": 5160
    },
    {
      "epoch": 0.38898502746219243,
      "grad_norm": 14.960364490773049,
      "learning_rate": 3.8326407267684916e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.6875,
      "logps/chosen": -284.0,
      "logps/rejected": -652.0,
      "loss": 0.4378,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.484375,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.21875,
      "step": 5170
    },
    {
      "epoch": 0.38973741629674213,
      "grad_norm": 25.201931144917946,
      "learning_rate": 3.827080501428695e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.78125,
      "logps/chosen": -290.0,
      "logps/rejected": -600.0,
      "loss": 0.4531,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.703125,
      "rewards/margins": 3.171875,
      "rewards/rejected": -4.875,
      "step": 5180
    },
    {
      "epoch": 0.39048980513129183,
      "grad_norm": 18.986679580762782,
      "learning_rate": 3.821511121016498e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.875,
      "logps/chosen": -266.0,
      "logps/rejected": -568.0,
      "loss": 0.4575,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.578125,
      "rewards/margins": 3.015625,
      "rewards/rejected": -4.59375,
      "step": 5190
    },
    {
      "epoch": 0.39124219396584153,
      "grad_norm": 21.70004155008451,
      "learning_rate": 3.815932623953142e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.65625,
      "logps/chosen": -282.0,
      "logps/rejected": -568.0,
      "loss": 0.435,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.53125,
      "rewards/margins": 2.96875,
      "rewards/rejected": -4.5,
      "step": 5200
    },
    {
      "epoch": 0.39199458280039123,
      "grad_norm": 26.589801472050745,
      "learning_rate": 3.810345048722762e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.84375,
      "logps/chosen": -290.0,
      "logps/rejected": -616.0,
      "loss": 0.431,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 3.34375,
      "rewards/rejected": -5.0,
      "step": 5210
    },
    {
      "epoch": 0.39274697163494093,
      "grad_norm": 19.676728734894844,
      "learning_rate": 3.804748433872119e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.46875,
      "logps/chosen": -316.0,
      "logps/rejected": -592.0,
      "loss": 0.4361,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.921875,
      "rewards/margins": 2.875,
      "rewards/rejected": -4.8125,
      "step": 5220
    },
    {
      "epoch": 0.39349936046949063,
      "grad_norm": 21.112479477471545,
      "learning_rate": 3.7991428180103375e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.84375,
      "logps/chosen": -298.0,
      "logps/rejected": -616.0,
      "loss": 0.4669,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.609375,
      "rewards/margins": 3.328125,
      "rewards/rejected": -4.9375,
      "step": 5230
    },
    {
      "epoch": 0.39425174930404033,
      "grad_norm": 26.503333281078653,
      "learning_rate": 3.7935282398086335e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.625,
      "logps/chosen": -286.0,
      "logps/rejected": -572.0,
      "loss": 0.5049,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 3.109375,
      "rewards/rejected": -4.625,
      "step": 5240
    },
    {
      "epoch": 0.39500413813859003,
      "grad_norm": 18.911872868992525,
      "learning_rate": 3.787904738000053e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.65625,
      "logps/chosen": -282.0,
      "logps/rejected": -576.0,
      "loss": 0.4734,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.5234375,
      "rewards/margins": 2.875,
      "rewards/rejected": -4.40625,
      "step": 5250
    },
    {
      "epoch": 0.39575652697313973,
      "grad_norm": 23.695696828614565,
      "learning_rate": 3.7822723513792036e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.8125,
      "logps/chosen": -270.0,
      "logps/rejected": -636.0,
      "loss": 0.4131,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.4375,
      "rewards/margins": 3.34375,
      "rewards/rejected": -4.78125,
      "step": 5260
    },
    {
      "epoch": 0.39650891580768943,
      "grad_norm": 19.800253653720628,
      "learning_rate": 3.776631118801985e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.609375,
      "logps/chosen": -298.0,
      "logps/rejected": -604.0,
      "loss": 0.4259,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.140625,
      "rewards/rejected": -4.9375,
      "step": 5270
    },
    {
      "epoch": 0.39726130464223913,
      "grad_norm": 24.98609737091108,
      "learning_rate": 3.7709810791853224e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.859375,
      "logps/chosen": -318.0,
      "logps/rejected": -680.0,
      "loss": 0.4689,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.625,
      "step": 5280
    },
    {
      "epoch": 0.3980136934767888,
      "grad_norm": 25.487976129034067,
      "learning_rate": 3.765322271506898e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.890625,
      "logps/chosen": -276.0,
      "logps/rejected": -608.0,
      "loss": 0.4277,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 3.40625,
      "rewards/rejected": -5.0625,
      "step": 5290
    },
    {
      "epoch": 0.3987660823113385,
      "grad_norm": 25.754713973323625,
      "learning_rate": 3.7596547348048806e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.71875,
      "logps/chosen": -282.0,
      "logps/rejected": -596.0,
      "loss": 0.4329,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.71875,
      "rewards/margins": 3.015625,
      "rewards/rejected": -4.71875,
      "step": 5300
    },
    {
      "epoch": 0.3995184711458882,
      "grad_norm": 23.34140697593013,
      "learning_rate": 3.753978508177661e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.734375,
      "logps/chosen": -312.0,
      "logps/rejected": -648.0,
      "loss": 0.4361,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 3.375,
      "rewards/rejected": -5.1875,
      "step": 5310
    },
    {
      "epoch": 0.4002708599804379,
      "grad_norm": 18.815859451091132,
      "learning_rate": 3.7482936307835737e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.953125,
      "logps/chosen": -300.0,
      "logps/rejected": -612.0,
      "loss": 0.4268,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 3.265625,
      "rewards/rejected": -5.0,
      "step": 5320
    },
    {
      "epoch": 0.4010232488149876,
      "grad_norm": 24.111518147401178,
      "learning_rate": 3.7426001418406356e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.578125,
      "logps/chosen": -276.0,
      "logps/rejected": -620.0,
      "loss": 0.4385,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.515625,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.03125,
      "step": 5330
    },
    {
      "epoch": 0.4017756376495373,
      "grad_norm": 22.386243955446595,
      "learning_rate": 3.7368980806262707e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.9375,
      "logps/chosen": -282.0,
      "logps/rejected": -592.0,
      "loss": 0.4192,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.7578125,
      "rewards/margins": 3.15625,
      "rewards/rejected": -4.90625,
      "step": 5340
    },
    {
      "epoch": 0.402528026484087,
      "grad_norm": 22.68077523332008,
      "learning_rate": 3.73118748647704e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.890625,
      "logps/chosen": -306.0,
      "logps/rejected": -608.0,
      "loss": 0.4886,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 3.109375,
      "rewards/rejected": -4.9375,
      "step": 5350
    },
    {
      "epoch": 0.4032804153186367,
      "grad_norm": 24.335689386584164,
      "learning_rate": 3.725468398788372e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -302.0,
      "logps/rejected": -588.0,
      "loss": 0.4656,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.8125,
      "rewards/margins": 2.96875,
      "rewards/rejected": -4.78125,
      "step": 5360
    },
    {
      "epoch": 0.4040328041531864,
      "grad_norm": 22.13750764646052,
      "learning_rate": 3.719740857014284e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.84375,
      "logps/chosen": -310.0,
      "logps/rejected": -620.0,
      "loss": 0.4676,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 3.1875,
      "rewards/rejected": -5.0,
      "step": 5370
    },
    {
      "epoch": 0.4047851929877361,
      "grad_norm": 25.061692643039255,
      "learning_rate": 3.714004900667122e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.828125,
      "logps/chosen": -280.0,
      "logps/rejected": -588.0,
      "loss": 0.4707,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 3.25,
      "rewards/rejected": -4.875,
      "step": 5380
    },
    {
      "epoch": 0.4055375818222858,
      "grad_norm": 17.137378124461794,
      "learning_rate": 3.708260569317277e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.734375,
      "logps/chosen": -272.0,
      "logps/rejected": -568.0,
      "loss": 0.4607,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 3.03125,
      "rewards/rejected": -4.59375,
      "step": 5390
    },
    {
      "epoch": 0.4062899706568355,
      "grad_norm": 15.330476020204397,
      "learning_rate": 3.702507902592917e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.6875,
      "logps/chosen": -264.0,
      "logps/rejected": -608.0,
      "loss": 0.4256,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 3.4375,
      "rewards/rejected": -4.9375,
      "step": 5400
    },
    {
      "epoch": 0.4070423594913852,
      "grad_norm": 19.07843022940658,
      "learning_rate": 3.696746940179713e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.75,
      "logps/chosen": -312.0,
      "logps/rejected": -628.0,
      "loss": 0.4633,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.84375,
      "rewards/margins": 3.140625,
      "rewards/rejected": -5.0,
      "step": 5410
    },
    {
      "epoch": 0.4077947483259348,
      "grad_norm": 21.53196525371359,
      "learning_rate": 3.690977721820565e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.578125,
      "logps/chosen": -306.0,
      "logps/rejected": -588.0,
      "loss": 0.4533,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.65625,
      "rewards/margins": 2.984375,
      "rewards/rejected": -4.65625,
      "step": 5420
    },
    {
      "epoch": 0.4085471371604845,
      "grad_norm": 27.03385743903897,
      "learning_rate": 3.6852002873153293e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.828125,
      "logps/chosen": -290.0,
      "logps/rejected": -580.0,
      "loss": 0.4269,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.7734375,
      "rewards/margins": 2.921875,
      "rewards/rejected": -4.6875,
      "step": 5430
    },
    {
      "epoch": 0.4092995259950342,
      "grad_norm": 20.46994341027527,
      "learning_rate": 3.679414676520541e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.765625,
      "logps/chosen": -300.0,
      "logps/rejected": -636.0,
      "loss": 0.4571,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 3.390625,
      "rewards/rejected": -5.125,
      "step": 5440
    },
    {
      "epoch": 0.4100519148295839,
      "grad_norm": 18.953653817532654,
      "learning_rate": 3.673620929349141e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.515625,
      "logps/chosen": -280.0,
      "logps/rejected": -652.0,
      "loss": 0.4082,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.6875,
      "rewards/margins": 3.734375,
      "rewards/rejected": -5.40625,
      "step": 5450
    },
    {
      "epoch": 0.4108043036641336,
      "grad_norm": 24.05988107248726,
      "learning_rate": 3.6678190857701996e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.828125,
      "logps/chosen": -304.0,
      "logps/rejected": -604.0,
      "loss": 0.4673,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 3.078125,
      "rewards/rejected": -4.8125,
      "step": 5460
    },
    {
      "epoch": 0.4115566924986833,
      "grad_norm": 22.70221171553333,
      "learning_rate": 3.6620091858086426e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.75,
      "logps/chosen": -294.0,
      "logps/rejected": -628.0,
      "loss": 0.4328,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.65625,
      "rewards/margins": 3.25,
      "rewards/rejected": -4.90625,
      "step": 5470
    },
    {
      "epoch": 0.412309081333233,
      "grad_norm": 17.66500953887493,
      "learning_rate": 3.6561912695449747e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.65625,
      "logps/chosen": -294.0,
      "logps/rejected": -636.0,
      "loss": 0.4602,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 3.375,
      "rewards/rejected": -5.0,
      "step": 5480
    },
    {
      "epoch": 0.4130614701677827,
      "grad_norm": 18.743178460648398,
      "learning_rate": 3.6503653771150003e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.78125,
      "logps/chosen": -272.0,
      "logps/rejected": -588.0,
      "loss": 0.4171,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.40625,
      "rewards/margins": 3.28125,
      "rewards/rejected": -4.6875,
      "step": 5490
    },
    {
      "epoch": 0.4138138590023324,
      "grad_norm": 28.26801505173643,
      "learning_rate": 3.6445315487095487e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.78125,
      "logps/chosen": -294.0,
      "logps/rejected": -572.0,
      "loss": 0.4362,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 3.03125,
      "rewards/rejected": -4.75,
      "step": 5500
    },
    {
      "epoch": 0.4145662478368821,
      "grad_norm": 25.313509420056977,
      "learning_rate": 3.6386898245741987e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.625,
      "logps/chosen": -276.0,
      "logps/rejected": -684.0,
      "loss": 0.4636,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.578125,
      "rewards/margins": 4.0,
      "rewards/rejected": -5.59375,
      "step": 5510
    },
    {
      "epoch": 0.4153186366714318,
      "grad_norm": 20.6494638868617,
      "learning_rate": 3.632840245008998e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.8125,
      "logps/chosen": -278.0,
      "logps/rejected": -608.0,
      "loss": 0.4069,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.59375,
      "rewards/margins": 3.3125,
      "rewards/rejected": -4.90625,
      "step": 5520
    },
    {
      "epoch": 0.4160710255059815,
      "grad_norm": 25.492085415169498,
      "learning_rate": 3.626982850368185e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.59375,
      "logps/chosen": -304.0,
      "logps/rejected": -652.0,
      "loss": 0.4368,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.90625,
      "rewards/margins": 3.4375,
      "rewards/rejected": -5.34375,
      "step": 5530
    },
    {
      "epoch": 0.41682341434053116,
      "grad_norm": 21.159472122140233,
      "learning_rate": 3.621117681059914e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.734375,
      "logps/chosen": -324.0,
      "logps/rejected": -704.0,
      "loss": 0.4406,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.75,
      "step": 5540
    },
    {
      "epoch": 0.41757580317508086,
      "grad_norm": 21.814940084343988,
      "learning_rate": 3.6152447775459715e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.859375,
      "logps/chosen": -288.0,
      "logps/rejected": -576.0,
      "loss": 0.4527,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.03125,
      "rewards/rejected": -4.84375,
      "step": 5550
    },
    {
      "epoch": 0.41832819200963056,
      "grad_norm": 19.323966509166393,
      "learning_rate": 3.609364180341503e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.71875,
      "logps/chosen": -286.0,
      "logps/rejected": -592.0,
      "loss": 0.4114,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 3.171875,
      "rewards/rejected": -4.78125,
      "step": 5560
    },
    {
      "epoch": 0.41908058084418026,
      "grad_norm": 25.072257576801707,
      "learning_rate": 3.603475930014726e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.6875,
      "logps/chosen": -304.0,
      "logps/rejected": -608.0,
      "loss": 0.4217,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.734375,
      "rewards/margins": 3.25,
      "rewards/rejected": -5.0,
      "step": 5570
    },
    {
      "epoch": 0.41983296967872996,
      "grad_norm": 25.312542235082756,
      "learning_rate": 3.5975800671866575e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.84375,
      "logps/chosen": -312.0,
      "logps/rejected": -648.0,
      "loss": 0.4635,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 3.390625,
      "rewards/rejected": -5.25,
      "step": 5580
    },
    {
      "epoch": 0.42058535851327966,
      "grad_norm": 19.03313326732141,
      "learning_rate": 3.591676632530829e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.84375,
      "logps/chosen": -278.0,
      "logps/rejected": -624.0,
      "loss": 0.4103,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.609375,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.15625,
      "step": 5590
    },
    {
      "epoch": 0.42133774734782936,
      "grad_norm": 19.677669436298892,
      "learning_rate": 3.5857656667730074e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.546875,
      "logps/chosen": -288.0,
      "logps/rejected": -656.0,
      "loss": 0.4104,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.3125,
      "step": 5600
    },
    {
      "epoch": 0.42209013618237906,
      "grad_norm": 24.639178084289835,
      "learning_rate": 3.579847210690915e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.828125,
      "logps/chosen": -336.0,
      "logps/rejected": -624.0,
      "loss": 0.4311,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -5.15625,
      "step": 5610
    },
    {
      "epoch": 0.42284252501692876,
      "grad_norm": 20.569453910901515,
      "learning_rate": 3.573921305113947e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.71875,
      "logps/chosen": -282.0,
      "logps/rejected": -608.0,
      "loss": 0.3921,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.703125,
      "rewards/margins": 3.28125,
      "rewards/rejected": -4.96875,
      "step": 5620
    },
    {
      "epoch": 0.42359491385147846,
      "grad_norm": 16.255566623292225,
      "learning_rate": 3.5679879909228895e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.9375,
      "logps/chosen": -300.0,
      "logps/rejected": -608.0,
      "loss": 0.4807,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.828125,
      "rewards/margins": 3.109375,
      "rewards/rejected": -4.9375,
      "step": 5630
    },
    {
      "epoch": 0.42434730268602816,
      "grad_norm": 26.741503201116586,
      "learning_rate": 3.562047309049638e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.828125,
      "logps/chosen": -294.0,
      "logps/rejected": -640.0,
      "loss": 0.4497,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.515625,
      "rewards/rejected": -5.3125,
      "step": 5640
    },
    {
      "epoch": 0.42509969152057786,
      "grad_norm": 18.84615137974081,
      "learning_rate": 3.5560993004769156e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.640625,
      "logps/chosen": -278.0,
      "logps/rejected": -572.0,
      "loss": 0.4647,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.59375,
      "rewards/margins": 2.96875,
      "rewards/rejected": -4.5625,
      "step": 5650
    },
    {
      "epoch": 0.4258520803551275,
      "grad_norm": 31.559337985305778,
      "learning_rate": 3.550144006237991e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.640625,
      "logps/chosen": -284.0,
      "logps/rejected": -580.0,
      "loss": 0.4634,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.578125,
      "rewards/margins": 3.0625,
      "rewards/rejected": -4.625,
      "step": 5660
    },
    {
      "epoch": 0.4266044691896772,
      "grad_norm": 30.532355982933204,
      "learning_rate": 3.5441814674163914e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.765625,
      "logps/chosen": -296.0,
      "logps/rejected": -632.0,
      "loss": 0.4124,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 3.546875,
      "rewards/rejected": -5.09375,
      "step": 5670
    },
    {
      "epoch": 0.4273568580242269,
      "grad_norm": 14.685119090932437,
      "learning_rate": 3.5382117251456245e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.65625,
      "logps/chosen": -314.0,
      "logps/rejected": -632.0,
      "loss": 0.412,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.84375,
      "rewards/margins": 3.34375,
      "rewards/rejected": -5.1875,
      "step": 5680
    },
    {
      "epoch": 0.4281092468587766,
      "grad_norm": 24.80955507671821,
      "learning_rate": 3.5322348206088903e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.890625,
      "logps/chosen": -272.0,
      "logps/rejected": -676.0,
      "loss": 0.3923,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.34375,
      "step": 5690
    },
    {
      "epoch": 0.4288616356933263,
      "grad_norm": 18.884527630846662,
      "learning_rate": 3.526250795038801e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.796875,
      "logps/chosen": -286.0,
      "logps/rejected": -624.0,
      "loss": 0.4317,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.703125,
      "rewards/margins": 3.390625,
      "rewards/rejected": -5.09375,
      "step": 5700
    },
    {
      "epoch": 0.429614024527876,
      "grad_norm": 25.112398346417393,
      "learning_rate": 3.520259689717091e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -3.03125,
      "logps/chosen": -318.0,
      "logps/rejected": -648.0,
      "loss": 0.4705,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.0,
      "rewards/margins": 3.265625,
      "rewards/rejected": -5.28125,
      "step": 5710
    },
    {
      "epoch": 0.4303664133624257,
      "grad_norm": 26.425738528547175,
      "learning_rate": 3.514261545974338e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.671875,
      "logps/chosen": -332.0,
      "logps/rejected": -680.0,
      "loss": 0.4491,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.78125,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.53125,
      "step": 5720
    },
    {
      "epoch": 0.4311188021969754,
      "grad_norm": 18.06739470492072,
      "learning_rate": 3.508256405189675e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.96875,
      "logps/chosen": -324.0,
      "logps/rejected": -572.0,
      "loss": 0.4534,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.875,
      "rewards/margins": 2.84375,
      "rewards/rejected": -4.71875,
      "step": 5730
    },
    {
      "epoch": 0.4318711910315251,
      "grad_norm": 22.310883587293837,
      "learning_rate": 3.502244308790506e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.734375,
      "logps/chosen": -306.0,
      "logps/rejected": -624.0,
      "loss": 0.4138,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.25,
      "rewards/rejected": -5.0,
      "step": 5740
    },
    {
      "epoch": 0.4326235798660748,
      "grad_norm": 24.640562675879135,
      "learning_rate": 3.496225298252216e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.734375,
      "logps/chosen": -308.0,
      "logps/rejected": -612.0,
      "loss": 0.4561,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.84375,
      "rewards/margins": 3.109375,
      "rewards/rejected": -4.9375,
      "step": 5750
    },
    {
      "epoch": 0.4333759687006245,
      "grad_norm": 22.828618403349946,
      "learning_rate": 3.490199415097892e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.921875,
      "logps/chosen": -300.0,
      "logps/rejected": -668.0,
      "loss": 0.4375,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.65625,
      "step": 5760
    },
    {
      "epoch": 0.4341283575351742,
      "grad_norm": 22.58571678924633,
      "learning_rate": 3.4841667008980316e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.71875,
      "logps/chosen": -280.0,
      "logps/rejected": -652.0,
      "loss": 0.4083,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.46875,
      "step": 5770
    },
    {
      "epoch": 0.43488074636972385,
      "grad_norm": 18.053819620839125,
      "learning_rate": 3.478127197270257e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.578125,
      "logps/chosen": -296.0,
      "logps/rejected": -656.0,
      "loss": 0.4092,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8125,
      "rewards/margins": 3.609375,
      "rewards/rejected": -5.40625,
      "step": 5780
    },
    {
      "epoch": 0.43563313520427355,
      "grad_norm": 17.18053963944144,
      "learning_rate": 3.4720809458790277e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.609375,
      "logps/chosen": -314.0,
      "logps/rejected": -748.0,
      "loss": 0.4821,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 4.34375,
      "rewards/rejected": -6.1875,
      "step": 5790
    },
    {
      "epoch": 0.43638552403882325,
      "grad_norm": 20.527952294966013,
      "learning_rate": 3.466027988435356e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.78125,
      "logps/chosen": -282.0,
      "logps/rejected": -672.0,
      "loss": 0.4226,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.640625,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.40625,
      "step": 5800
    },
    {
      "epoch": 0.43713791287337295,
      "grad_norm": 24.87196024758967,
      "learning_rate": 3.459968366696515e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.828125,
      "logps/chosen": -276.0,
      "logps/rejected": -640.0,
      "loss": 0.4249,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.53125,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.3125,
      "step": 5810
    },
    {
      "epoch": 0.43789030170792265,
      "grad_norm": 20.52978062404546,
      "learning_rate": 3.453902122465753e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.9375,
      "logps/chosen": -306.0,
      "logps/rejected": -612.0,
      "loss": 0.4611,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -5.0625,
      "step": 5820
    },
    {
      "epoch": 0.43864269054247235,
      "grad_norm": 19.43655575384478,
      "learning_rate": 3.4478292975920043e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.84375,
      "logps/chosen": -276.0,
      "logps/rejected": -588.0,
      "loss": 0.4444,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.578125,
      "rewards/margins": 3.265625,
      "rewards/rejected": -4.84375,
      "step": 5830
    },
    {
      "epoch": 0.43939507937702205,
      "grad_norm": 25.82158387722616,
      "learning_rate": 3.441749933969601e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.875,
      "logps/chosen": -294.0,
      "logps/rejected": -600.0,
      "loss": 0.406,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -4.96875,
      "step": 5840
    },
    {
      "epoch": 0.44014746821157175,
      "grad_norm": 19.244472988117682,
      "learning_rate": 3.435664073537985e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.90625,
      "logps/chosen": -326.0,
      "logps/rejected": -612.0,
      "loss": 0.4335,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 3.03125,
      "rewards/rejected": -5.0,
      "step": 5850
    },
    {
      "epoch": 0.44089985704612145,
      "grad_norm": 28.335619668795633,
      "learning_rate": 3.4295717582814143e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.65625,
      "logps/chosen": -328.0,
      "logps/rejected": -616.0,
      "loss": 0.4343,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 2.984375,
      "rewards/rejected": -4.96875,
      "step": 5860
    },
    {
      "epoch": 0.44165224588067115,
      "grad_norm": 20.02234991956254,
      "learning_rate": 3.4234730302286814e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.953125,
      "logps/chosen": -296.0,
      "logps/rejected": -616.0,
      "loss": 0.3897,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.34375,
      "rewards/rejected": -5.15625,
      "step": 5870
    },
    {
      "epoch": 0.44240463471522085,
      "grad_norm": 35.326088993218896,
      "learning_rate": 3.417367931452813e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.515625,
      "logps/chosen": -332.0,
      "logps/rejected": -756.0,
      "loss": 0.4271,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.90625,
      "rewards/margins": 4.21875,
      "rewards/rejected": -6.125,
      "step": 5880
    },
    {
      "epoch": 0.44315702354977055,
      "grad_norm": 21.48670936462512,
      "learning_rate": 3.4112565040707893e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.8125,
      "logps/chosen": -304.0,
      "logps/rejected": -604.0,
      "loss": 0.4398,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 3.015625,
      "rewards/rejected": -4.9375,
      "step": 5890
    },
    {
      "epoch": 0.4439094123843202,
      "grad_norm": 20.010824304649756,
      "learning_rate": 3.405138790243248e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.75,
      "logps/chosen": -310.0,
      "logps/rejected": -600.0,
      "loss": 0.4371,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.046875,
      "rewards/rejected": -4.84375,
      "step": 5900
    },
    {
      "epoch": 0.4446618012188699,
      "grad_norm": 26.5897268899179,
      "learning_rate": 3.399014832174194e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.703125,
      "logps/chosen": -286.0,
      "logps/rejected": -600.0,
      "loss": 0.4353,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 3.1875,
      "rewards/rejected": -4.90625,
      "step": 5910
    },
    {
      "epoch": 0.4454141900534196,
      "grad_norm": 30.71944122545211,
      "learning_rate": 3.3928846721107104e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.703125,
      "logps/chosen": -314.0,
      "logps/rejected": -676.0,
      "loss": 0.4394,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.53125,
      "step": 5920
    },
    {
      "epoch": 0.4461665788879693,
      "grad_norm": 21.172568888692055,
      "learning_rate": 3.386748352342664e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.875,
      "logps/chosen": -306.0,
      "logps/rejected": -608.0,
      "loss": 0.4162,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 3.171875,
      "rewards/rejected": -5.0,
      "step": 5930
    },
    {
      "epoch": 0.446918967722519,
      "grad_norm": 28.368085784741385,
      "learning_rate": 3.380605915202419e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.8125,
      "logps/chosen": -314.0,
      "logps/rejected": -624.0,
      "loss": 0.4651,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8984375,
      "rewards/margins": 3.25,
      "rewards/rejected": -5.15625,
      "step": 5940
    },
    {
      "epoch": 0.4476713565570687,
      "grad_norm": 17.461218075835788,
      "learning_rate": 3.3744574030645366e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.546875,
      "logps/chosen": -300.0,
      "logps/rejected": -560.0,
      "loss": 0.4701,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.734375,
      "rewards/margins": 2.71875,
      "rewards/rejected": -4.46875,
      "step": 5950
    },
    {
      "epoch": 0.4484237453916184,
      "grad_norm": 20.488566738765062,
      "learning_rate": 3.3683028583454896e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.609375,
      "logps/chosen": -282.0,
      "logps/rejected": -588.0,
      "loss": 0.4233,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.515625,
      "rewards/margins": 3.09375,
      "rewards/rejected": -4.59375,
      "step": 5960
    },
    {
      "epoch": 0.4491761342261681,
      "grad_norm": 23.45382095842351,
      "learning_rate": 3.3621423235033687e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.703125,
      "logps/chosen": -304.0,
      "logps/rejected": -608.0,
      "loss": 0.4222,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.78125,
      "rewards/margins": 3.1875,
      "rewards/rejected": -4.96875,
      "step": 5970
    },
    {
      "epoch": 0.4499285230607178,
      "grad_norm": 30.213714940639587,
      "learning_rate": 3.355975841037585e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.6875,
      "logps/chosen": -288.0,
      "logps/rejected": -616.0,
      "loss": 0.4345,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.7734375,
      "rewards/margins": 3.25,
      "rewards/rejected": -5.03125,
      "step": 5980
    },
    {
      "epoch": 0.4506809118952675,
      "grad_norm": 22.867987796490574,
      "learning_rate": 3.3498034534885844e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.90625,
      "logps/chosen": -288.0,
      "logps/rejected": -604.0,
      "loss": 0.4064,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.28125,
      "rewards/rejected": -5.03125,
      "step": 5990
    },
    {
      "epoch": 0.4514333007298172,
      "grad_norm": 18.04536696251433,
      "learning_rate": 3.343625203437547e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.765625,
      "logps/chosen": -274.0,
      "logps/rejected": -600.0,
      "loss": 0.4115,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 3.375,
      "rewards/rejected": -4.96875,
      "step": 6000
    },
    {
      "epoch": 0.4514333007298172,
      "eval_logits/chosen": -2.625,
      "eval_logits/rejected": -2.734375,
      "eval_logps/chosen": -318.0,
      "eval_logps/rejected": -604.0,
      "eval_loss": 0.2707754671573639,
      "eval_rewards/accuracies": 0.8798239827156067,
      "eval_rewards/chosen": -1.8671875,
      "eval_rewards/margins": 3.015625,
      "eval_rewards/rejected": -4.875,
      "eval_runtime": 2668.43,
      "eval_samples_per_second": 35.411,
      "eval_steps_per_second": 0.554,
      "step": 6000
    },
    {
      "epoch": 0.4521856895643669,
      "grad_norm": 19.710252089930982,
      "learning_rate": 3.337441133506096e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.765625,
      "logps/chosen": -304.0,
      "logps/rejected": -632.0,
      "loss": 0.473,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 3.328125,
      "rewards/rejected": -5.1875,
      "step": 6010
    },
    {
      "epoch": 0.45293807839891653,
      "grad_norm": 26.895326588781515,
      "learning_rate": 3.3312512863560053e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.71875,
      "logps/chosen": -302.0,
      "logps/rejected": -652.0,
      "loss": 0.4497,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 3.515625,
      "rewards/rejected": -5.46875,
      "step": 6020
    },
    {
      "epoch": 0.45369046723346623,
      "grad_norm": 15.26628491302783,
      "learning_rate": 3.325055704688906e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.65625,
      "logps/chosen": -278.0,
      "logps/rejected": -564.0,
      "loss": 0.4228,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 3.015625,
      "rewards/rejected": -4.65625,
      "step": 6030
    },
    {
      "epoch": 0.45444285606801593,
      "grad_norm": 23.418224812556527,
      "learning_rate": 3.318854431245984e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.453125,
      "logps/chosen": -266.0,
      "logps/rejected": -604.0,
      "loss": 0.4062,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4765625,
      "rewards/margins": 3.515625,
      "rewards/rejected": -5.0,
      "step": 6040
    },
    {
      "epoch": 0.45519524490256563,
      "grad_norm": 22.37538243719865,
      "learning_rate": 3.312647508807696e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.5625,
      "logps/chosen": -304.0,
      "logps/rejected": -592.0,
      "loss": 0.457,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.84375,
      "rewards/margins": 2.8125,
      "rewards/rejected": -4.65625,
      "step": 6050
    },
    {
      "epoch": 0.45594763373711533,
      "grad_norm": 28.036684781042563,
      "learning_rate": 3.3064349801934664e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.421875,
      "logps/chosen": -288.0,
      "logps/rejected": -636.0,
      "loss": 0.3917,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.625,
      "rewards/margins": 3.546875,
      "rewards/rejected": -5.1875,
      "step": 6060
    },
    {
      "epoch": 0.45670002257166503,
      "grad_norm": 16.668258952245495,
      "learning_rate": 3.300216888261396e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.671875,
      "logps/chosen": -348.0,
      "logps/rejected": -704.0,
      "loss": 0.3938,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.109375,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.78125,
      "step": 6070
    },
    {
      "epoch": 0.45745241140621473,
      "grad_norm": 26.62796462778445,
      "learning_rate": 3.2939932759079655e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.8125,
      "logps/chosen": -316.0,
      "logps/rejected": -656.0,
      "loss": 0.4273,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.53125,
      "rewards/rejected": -5.40625,
      "step": 6080
    },
    {
      "epoch": 0.45820480024076443,
      "grad_norm": 21.229428163972646,
      "learning_rate": 3.2877641860677357e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.640625,
      "logps/chosen": -292.0,
      "logps/rejected": -676.0,
      "loss": 0.3759,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.015625,
      "rewards/margins": 3.546875,
      "rewards/rejected": -5.5625,
      "step": 6090
    },
    {
      "epoch": 0.45895718907531413,
      "grad_norm": 24.121166254288948,
      "learning_rate": 3.2815296617130585e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.453125,
      "logps/chosen": -314.0,
      "logps/rejected": -644.0,
      "loss": 0.4317,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.203125,
      "rewards/rejected": -5.125,
      "step": 6100
    },
    {
      "epoch": 0.45970957790986383,
      "grad_norm": 23.241630950267492,
      "learning_rate": 3.275289745853775e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.6875,
      "logps/chosen": -306.0,
      "logps/rejected": -660.0,
      "loss": 0.3957,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 3.65625,
      "rewards/rejected": -5.375,
      "step": 6110
    },
    {
      "epoch": 0.46046196674441353,
      "grad_norm": 17.857464329915143,
      "learning_rate": 3.269044481536921e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.984375,
      "logps/chosen": -296.0,
      "logps/rejected": -660.0,
      "loss": 0.4458,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.734375,
      "rewards/margins": 3.71875,
      "rewards/rejected": -5.4375,
      "step": 6120
    },
    {
      "epoch": 0.46121435557896323,
      "grad_norm": 24.347402615117446,
      "learning_rate": 3.2627939118464296e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.828125,
      "logps/chosen": -276.0,
      "logps/rejected": -608.0,
      "loss": 0.4282,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.65625,
      "rewards/margins": 3.328125,
      "rewards/rejected": -4.96875,
      "step": 6130
    },
    {
      "epoch": 0.46196674441351293,
      "grad_norm": 20.509826636441183,
      "learning_rate": 3.256538079902833e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.703125,
      "logps/chosen": -294.0,
      "logps/rejected": -672.0,
      "loss": 0.4564,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.75,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.53125,
      "step": 6140
    },
    {
      "epoch": 0.4627191332480626,
      "grad_norm": 25.567724593745158,
      "learning_rate": 3.2502770288629655e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.796875,
      "logps/chosen": -302.0,
      "logps/rejected": -640.0,
      "loss": 0.434,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.1875,
      "step": 6150
    },
    {
      "epoch": 0.4634715220826123,
      "grad_norm": 20.959777846149574,
      "learning_rate": 3.244010801919668e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.765625,
      "logps/chosen": -316.0,
      "logps/rejected": -648.0,
      "loss": 0.4125,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.890625,
      "rewards/margins": 3.53125,
      "rewards/rejected": -5.4375,
      "step": 6160
    },
    {
      "epoch": 0.464223910917162,
      "grad_norm": 22.76491997911319,
      "learning_rate": 3.237739442301487e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.90625,
      "logps/chosen": -306.0,
      "logps/rejected": -712.0,
      "loss": 0.4177,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.796875,
      "rewards/margins": 4.25,
      "rewards/rejected": -6.0625,
      "step": 6170
    },
    {
      "epoch": 0.4649762997517117,
      "grad_norm": 20.77758132858046,
      "learning_rate": 3.231462993272377e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.875,
      "logps/chosen": -328.0,
      "logps/rejected": -672.0,
      "loss": 0.4347,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.046875,
      "rewards/margins": 3.4375,
      "rewards/rejected": -5.5,
      "step": 6180
    },
    {
      "epoch": 0.4657286885862614,
      "grad_norm": 22.360517284137295,
      "learning_rate": 3.225181498131404e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.59375,
      "logps/chosen": -288.0,
      "logps/rejected": -660.0,
      "loss": 0.4022,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.65625,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.34375,
      "step": 6190
    },
    {
      "epoch": 0.4664810774208111,
      "grad_norm": 25.25164370608612,
      "learning_rate": 3.218895000212445e-07,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.59375,
      "logps/chosen": -286.0,
      "logps/rejected": -620.0,
      "loss": 0.4659,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 3.234375,
      "rewards/rejected": -4.875,
      "step": 6200
    },
    {
      "epoch": 0.4672334662553608,
      "grad_norm": 14.902979043447347,
      "learning_rate": 3.2126035428838874e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.71875,
      "logps/chosen": -320.0,
      "logps/rejected": -608.0,
      "loss": 0.4151,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.765625,
      "rewards/margins": 2.96875,
      "rewards/rejected": -4.75,
      "step": 6210
    },
    {
      "epoch": 0.4679858550899105,
      "grad_norm": 29.568526508666785,
      "learning_rate": 3.2063071695483354e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.84375,
      "logps/chosen": -298.0,
      "logps/rejected": -620.0,
      "loss": 0.4135,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 3.0625,
      "rewards/rejected": -4.9375,
      "step": 6220
    },
    {
      "epoch": 0.4687382439244602,
      "grad_norm": 20.81173984165901,
      "learning_rate": 3.200005923642305e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.765625,
      "logps/chosen": -292.0,
      "logps/rejected": -636.0,
      "loss": 0.416,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.375,
      "rewards/rejected": -5.125,
      "step": 6230
    },
    {
      "epoch": 0.4694906327590099,
      "grad_norm": 22.340298368924735,
      "learning_rate": 3.193699848635925e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.703125,
      "logps/chosen": -310.0,
      "logps/rejected": -628.0,
      "loss": 0.4119,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 3.25,
      "rewards/rejected": -5.125,
      "step": 6240
    },
    {
      "epoch": 0.4702430215935596,
      "grad_norm": 20.506411516694794,
      "learning_rate": 3.1873889880326425e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.71875,
      "logps/chosen": -294.0,
      "logps/rejected": -584.0,
      "loss": 0.4585,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.78125,
      "rewards/margins": 2.96875,
      "rewards/rejected": -4.75,
      "step": 6250
    },
    {
      "epoch": 0.4709954104281093,
      "grad_norm": 24.377586961154968,
      "learning_rate": 3.1810733853689153e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.53125,
      "logps/chosen": -326.0,
      "logps/rejected": -648.0,
      "loss": 0.4295,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.734375,
      "rewards/margins": 3.25,
      "rewards/rejected": -4.96875,
      "step": 6260
    },
    {
      "epoch": 0.4717477992626589,
      "grad_norm": 23.838411950548917,
      "learning_rate": 3.1747530842139157e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.640625,
      "logps/chosen": -296.0,
      "logps/rejected": -620.0,
      "loss": 0.4582,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.765625,
      "rewards/margins": 3.234375,
      "rewards/rejected": -5.0,
      "step": 6270
    },
    {
      "epoch": 0.4725001880972086,
      "grad_norm": 18.80558899991819,
      "learning_rate": 3.1684281281692305e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.609375,
      "logps/chosen": -292.0,
      "logps/rejected": -576.0,
      "loss": 0.4686,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.765625,
      "rewards/margins": 2.828125,
      "rewards/rejected": -4.59375,
      "step": 6280
    },
    {
      "epoch": 0.4732525769317583,
      "grad_norm": 30.21237178872267,
      "learning_rate": 3.162098560868558e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.75,
      "logps/chosen": -274.0,
      "logps/rejected": -568.0,
      "loss": 0.3818,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 3.078125,
      "rewards/rejected": -4.59375,
      "step": 6290
    },
    {
      "epoch": 0.474004965766308,
      "grad_norm": 26.30303860761958,
      "learning_rate": 3.155764425977408e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -3.03125,
      "logps/chosen": -322.0,
      "logps/rejected": -636.0,
      "loss": 0.3781,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.8984375,
      "rewards/margins": 3.484375,
      "rewards/rejected": -5.375,
      "step": 6300
    },
    {
      "epoch": 0.4747573546008577,
      "grad_norm": 22.50629712422882,
      "learning_rate": 3.1494257671928006e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.96875,
      "logps/chosen": -340.0,
      "logps/rejected": -636.0,
      "loss": 0.4545,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.296875,
      "rewards/margins": 3.0625,
      "rewards/rejected": -5.34375,
      "step": 6310
    },
    {
      "epoch": 0.4755097434354074,
      "grad_norm": 25.207894233151116,
      "learning_rate": 3.1430826282429653e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.953125,
      "logps/chosen": -320.0,
      "logps/rejected": -640.0,
      "loss": 0.393,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -5.375,
      "step": 6320
    },
    {
      "epoch": 0.4762621322699571,
      "grad_norm": 18.398709952676892,
      "learning_rate": 3.136735052887038e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.859375,
      "logps/chosen": -334.0,
      "logps/rejected": -604.0,
      "loss": 0.4349,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.0,
      "rewards/rejected": -4.9375,
      "step": 6330
    },
    {
      "epoch": 0.4770145211045068,
      "grad_norm": 17.179515295138163,
      "learning_rate": 3.1303830849147594e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.890625,
      "logps/chosen": -326.0,
      "logps/rejected": -588.0,
      "loss": 0.4459,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 2.78125,
      "rewards/rejected": -4.75,
      "step": 6340
    },
    {
      "epoch": 0.4777669099390565,
      "grad_norm": 24.21673927268538,
      "learning_rate": 3.124026768146176e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.8125,
      "logps/chosen": -282.0,
      "logps/rejected": -604.0,
      "loss": 0.4282,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 3.25,
      "rewards/rejected": -4.84375,
      "step": 6350
    },
    {
      "epoch": 0.4785192987736062,
      "grad_norm": 20.836302620179534,
      "learning_rate": 3.117666146431331e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.796875,
      "logps/chosen": -326.0,
      "logps/rejected": -612.0,
      "loss": 0.4317,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.015625,
      "rewards/margins": 2.921875,
      "rewards/rejected": -4.9375,
      "step": 6360
    },
    {
      "epoch": 0.4792716876081559,
      "grad_norm": 20.014498446295203,
      "learning_rate": 3.111301263649969e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.609375,
      "logps/chosen": -286.0,
      "logps/rejected": -592.0,
      "loss": 0.4378,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.703125,
      "rewards/margins": 3.078125,
      "rewards/rejected": -4.78125,
      "step": 6370
    },
    {
      "epoch": 0.4800240764427056,
      "grad_norm": 32.06448053826409,
      "learning_rate": 3.1049321637112303e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.78125,
      "logps/chosen": -278.0,
      "logps/rejected": -612.0,
      "loss": 0.4342,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 3.421875,
      "rewards/rejected": -4.96875,
      "step": 6380
    },
    {
      "epoch": 0.48077646527725526,
      "grad_norm": 20.276032961708808,
      "learning_rate": 3.0985588905533456e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.59375,
      "logps/chosen": -278.0,
      "logps/rejected": -608.0,
      "loss": 0.3882,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 3.4375,
      "rewards/rejected": -4.96875,
      "step": 6390
    },
    {
      "epoch": 0.48152885411180496,
      "grad_norm": 23.586105067154516,
      "learning_rate": 3.0921814881433373e-07,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.296875,
      "logps/chosen": -296.0,
      "logps/rejected": -628.0,
      "loss": 0.4436,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.75,
      "rewards/margins": 3.25,
      "rewards/rejected": -5.0,
      "step": 6400
    },
    {
      "epoch": 0.48228124294635466,
      "grad_norm": 19.053899523358723,
      "learning_rate": 3.0858000004767125e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.515625,
      "logps/chosen": -324.0,
      "logps/rejected": -608.0,
      "loss": 0.4212,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.984375,
      "rewards/margins": 2.9375,
      "rewards/rejected": -4.9375,
      "step": 6410
    },
    {
      "epoch": 0.48303363178090436,
      "grad_norm": 19.580078886362944,
      "learning_rate": 3.079414471577163e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.546875,
      "logps/chosen": -296.0,
      "logps/rejected": -620.0,
      "loss": 0.4174,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.734375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -5.15625,
      "step": 6420
    },
    {
      "epoch": 0.48378602061545406,
      "grad_norm": 20.807512848622203,
      "learning_rate": 3.073024945496258e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.734375,
      "logps/chosen": -288.0,
      "logps/rejected": -616.0,
      "loss": 0.4265,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.546875,
      "rewards/margins": 3.4375,
      "rewards/rejected": -4.96875,
      "step": 6430
    },
    {
      "epoch": 0.48453840945000376,
      "grad_norm": 23.80572474461615,
      "learning_rate": 3.0666314663131435e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.71875,
      "logps/chosen": -328.0,
      "logps/rejected": -624.0,
      "loss": 0.4129,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0,
      "rewards/margins": 3.1875,
      "rewards/rejected": -5.1875,
      "step": 6440
    },
    {
      "epoch": 0.48529079828455346,
      "grad_norm": 21.92209218188957,
      "learning_rate": 3.0602340781342343e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.90625,
      "logps/chosen": -332.0,
      "logps/rejected": -624.0,
      "loss": 0.4059,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.21875,
      "rewards/rejected": -5.15625,
      "step": 6450
    },
    {
      "epoch": 0.48604318711910316,
      "grad_norm": 23.348072491623288,
      "learning_rate": 3.0538328250929147e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.765625,
      "logps/chosen": -304.0,
      "logps/rejected": -684.0,
      "loss": 0.4517,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.78125,
      "step": 6460
    },
    {
      "epoch": 0.48679557595365286,
      "grad_norm": 22.332936473757986,
      "learning_rate": 3.047427751349228e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.828125,
      "logps/chosen": -332.0,
      "logps/rejected": -632.0,
      "loss": 0.3993,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.84375,
      "rewards/margins": 3.34375,
      "rewards/rejected": -5.1875,
      "step": 6470
    },
    {
      "epoch": 0.48754796478820256,
      "grad_norm": 26.64294267414569,
      "learning_rate": 3.041018901089579e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.71875,
      "logps/chosen": -278.0,
      "logps/rejected": -596.0,
      "loss": 0.4246,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.703125,
      "rewards/margins": 3.0,
      "rewards/rejected": -4.71875,
      "step": 6480
    },
    {
      "epoch": 0.48830035362275226,
      "grad_norm": 31.822303207082776,
      "learning_rate": 3.034606318526423e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.890625,
      "logps/chosen": -276.0,
      "logps/rejected": -600.0,
      "loss": 0.398,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 3.40625,
      "rewards/rejected": -4.96875,
      "step": 6490
    },
    {
      "epoch": 0.48905274245730196,
      "grad_norm": 19.119007214178072,
      "learning_rate": 3.028190047897964e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.859375,
      "logps/chosen": -308.0,
      "logps/rejected": -704.0,
      "loss": 0.4293,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.984375,
      "rewards/margins": 3.84375,
      "rewards/rejected": -5.84375,
      "step": 6500
    },
    {
      "epoch": 0.4898051312918516,
      "grad_norm": 29.30670328758456,
      "learning_rate": 3.021770133467848e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.671875,
      "logps/chosen": -334.0,
      "logps/rejected": -664.0,
      "loss": 0.4706,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.046875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -5.25,
      "step": 6510
    },
    {
      "epoch": 0.4905575201264013,
      "grad_norm": 25.21170089759504,
      "learning_rate": 3.015346619524859e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.59375,
      "logps/chosen": -332.0,
      "logps/rejected": -688.0,
      "loss": 0.386,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.046875,
      "rewards/margins": 3.65625,
      "rewards/rejected": -5.71875,
      "step": 6520
    },
    {
      "epoch": 0.491309908960951,
      "grad_norm": 27.672411371998734,
      "learning_rate": 3.008919550382613e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.765625,
      "logps/chosen": -310.0,
      "logps/rejected": -656.0,
      "loss": 0.449,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8125,
      "rewards/margins": 3.421875,
      "rewards/rejected": -5.25,
      "step": 6530
    },
    {
      "epoch": 0.4920622977955007,
      "grad_norm": 19.305867927696372,
      "learning_rate": 3.0024889703792513e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.625,
      "logps/chosen": -320.0,
      "logps/rejected": -616.0,
      "loss": 0.4387,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.09375,
      "rewards/rejected": -4.96875,
      "step": 6540
    },
    {
      "epoch": 0.4928146866300504,
      "grad_norm": 23.483324177560583,
      "learning_rate": 2.996054923877136e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.65625,
      "logps/chosen": -292.0,
      "logps/rejected": -572.0,
      "loss": 0.4394,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.84375,
      "rewards/margins": 2.796875,
      "rewards/rejected": -4.65625,
      "step": 6550
    },
    {
      "epoch": 0.4935670754646001,
      "grad_norm": 23.97974491466278,
      "learning_rate": 2.9896174552625427e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.625,
      "logps/chosen": -302.0,
      "logps/rejected": -680.0,
      "loss": 0.4028,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.53125,
      "step": 6560
    },
    {
      "epoch": 0.4943194642991498,
      "grad_norm": 18.771070508437525,
      "learning_rate": 2.983176608945356e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.71875,
      "logps/chosen": -310.0,
      "logps/rejected": -684.0,
      "loss": 0.44,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -5.65625,
      "step": 6570
    },
    {
      "epoch": 0.4950718531336995,
      "grad_norm": 29.249852899991488,
      "learning_rate": 2.9767324293587617e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5625,
      "logps/chosen": -312.0,
      "logps/rejected": -700.0,
      "loss": 0.4347,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.921875,
      "rewards/rejected": -5.71875,
      "step": 6580
    },
    {
      "epoch": 0.4958242419682492,
      "grad_norm": 30.066347719364796,
      "learning_rate": 2.9702849609589403e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.734375,
      "logps/chosen": -286.0,
      "logps/rejected": -612.0,
      "loss": 0.4483,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.7578125,
      "rewards/margins": 3.015625,
      "rewards/rejected": -4.78125,
      "step": 6590
    },
    {
      "epoch": 0.4965766308027989,
      "grad_norm": 18.471434010231004,
      "learning_rate": 2.96383424822476e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.515625,
      "logps/chosen": -290.0,
      "logps/rejected": -592.0,
      "loss": 0.4181,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 3.25,
      "rewards/rejected": -4.875,
      "step": 6600
    },
    {
      "epoch": 0.4973290196373486,
      "grad_norm": 26.390309514413445,
      "learning_rate": 2.957380335657473e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.703125,
      "logps/chosen": -294.0,
      "logps/rejected": -664.0,
      "loss": 0.4166,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.4375,
      "step": 6610
    },
    {
      "epoch": 0.4980814084718983,
      "grad_norm": 25.168998724905553,
      "learning_rate": 2.950923267780405e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.6875,
      "logps/chosen": -320.0,
      "logps/rejected": -612.0,
      "loss": 0.4321,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 3.0,
      "rewards/rejected": -5.0,
      "step": 6620
    },
    {
      "epoch": 0.49883379730644795,
      "grad_norm": 19.687093298789943,
      "learning_rate": 2.944463089138648e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.765625,
      "logps/chosen": -280.0,
      "logps/rejected": -600.0,
      "loss": 0.4469,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 3.328125,
      "rewards/rejected": -4.9375,
      "step": 6630
    },
    {
      "epoch": 0.49958618614099765,
      "grad_norm": 20.215335038688302,
      "learning_rate": 2.937999844298753e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.859375,
      "logps/chosen": -266.0,
      "logps/rejected": -584.0,
      "loss": 0.4254,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.578125,
      "rewards/margins": 3.296875,
      "rewards/rejected": -4.875,
      "step": 6640
    },
    {
      "epoch": 0.5003385749755473,
      "grad_norm": 24.57809379208284,
      "learning_rate": 2.931533577848428e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.8125,
      "logps/chosen": -328.0,
      "logps/rejected": -644.0,
      "loss": 0.4387,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.375,
      "rewards/rejected": -5.3125,
      "step": 6650
    },
    {
      "epoch": 0.5010909638100971,
      "grad_norm": 22.32285837084119,
      "learning_rate": 2.9250643343962216e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.59375,
      "logps/chosen": -324.0,
      "logps/rejected": -652.0,
      "loss": 0.4213,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.921875,
      "rewards/margins": 3.328125,
      "rewards/rejected": -5.25,
      "step": 6660
    },
    {
      "epoch": 0.5018433526446467,
      "grad_norm": 30.29218099149348,
      "learning_rate": 2.918592158571223e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.875,
      "logps/chosen": -288.0,
      "logps/rejected": -620.0,
      "loss": 0.4262,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 3.3125,
      "rewards/rejected": -5.03125,
      "step": 6670
    },
    {
      "epoch": 0.5025957414791965,
      "grad_norm": 29.893187030768157,
      "learning_rate": 2.912117095022749e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.65625,
      "logps/chosen": -322.0,
      "logps/rejected": -672.0,
      "loss": 0.4166,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.90625,
      "rewards/margins": 3.484375,
      "rewards/rejected": -5.375,
      "step": 6680
    },
    {
      "epoch": 0.5033481303137461,
      "grad_norm": 16.286032616235577,
      "learning_rate": 2.9056391884200374e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.875,
      "logps/chosen": -302.0,
      "logps/rejected": -628.0,
      "loss": 0.3971,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 3.375,
      "rewards/rejected": -5.21875,
      "step": 6690
    },
    {
      "epoch": 0.5041005191482958,
      "grad_norm": 21.16062326686524,
      "learning_rate": 2.8991584834519405e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.828125,
      "logps/chosen": -304.0,
      "logps/rejected": -600.0,
      "loss": 0.431,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.671875,
      "rewards/margins": 3.0,
      "rewards/rejected": -4.6875,
      "step": 6700
    },
    {
      "epoch": 0.5048529079828455,
      "grad_norm": 22.133861144080488,
      "learning_rate": 2.8926750248266165e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.796875,
      "logps/chosen": -324.0,
      "logps/rejected": -640.0,
      "loss": 0.4158,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.828125,
      "rewards/margins": 3.328125,
      "rewards/rejected": -5.15625,
      "step": 6710
    },
    {
      "epoch": 0.5056052968173952,
      "grad_norm": 38.98141632735112,
      "learning_rate": 2.886188857271217e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.8125,
      "logps/chosen": -314.0,
      "logps/rejected": -640.0,
      "loss": 0.4531,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.4375,
      "rewards/rejected": -5.3125,
      "step": 6720
    },
    {
      "epoch": 0.5063576856519449,
      "grad_norm": 23.586841601540044,
      "learning_rate": 2.879700025531583e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.6875,
      "logps/chosen": -288.0,
      "logps/rejected": -584.0,
      "loss": 0.4228,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.1875,
      "rewards/rejected": -4.9375,
      "step": 6730
    },
    {
      "epoch": 0.5071100744864946,
      "grad_norm": 20.701542295325293,
      "learning_rate": 2.873208574371937e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.65625,
      "logps/chosen": -316.0,
      "logps/rejected": -612.0,
      "loss": 0.4585,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.09375,
      "rewards/rejected": -4.96875,
      "step": 6740
    },
    {
      "epoch": 0.5078624633210443,
      "grad_norm": 20.51673847990532,
      "learning_rate": 2.8667145485745684e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.6875,
      "logps/chosen": -310.0,
      "logps/rejected": -644.0,
      "loss": 0.4401,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 3.546875,
      "rewards/rejected": -5.21875,
      "step": 6750
    },
    {
      "epoch": 0.508614852155594,
      "grad_norm": 27.013658145775498,
      "learning_rate": 2.860217992939532e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.53125,
      "logps/chosen": -272.0,
      "logps/rejected": -604.0,
      "loss": 0.4146,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.515625,
      "rewards/margins": 3.3125,
      "rewards/rejected": -4.84375,
      "step": 6760
    },
    {
      "epoch": 0.5093672409901437,
      "grad_norm": 26.733051448028302,
      "learning_rate": 2.853718952284331e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.78125,
      "logps/chosen": -324.0,
      "logps/rejected": -640.0,
      "loss": 0.436,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 3.40625,
      "rewards/rejected": -5.28125,
      "step": 6770
    },
    {
      "epoch": 0.5101196298246934,
      "grad_norm": 18.220792057676444,
      "learning_rate": 2.8472174714436137e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.796875,
      "logps/chosen": -296.0,
      "logps/rejected": -656.0,
      "loss": 0.3915,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.46875,
      "step": 6780
    },
    {
      "epoch": 0.5108720186592431,
      "grad_norm": 21.206212913555213,
      "learning_rate": 2.8407135952688634e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.8125,
      "logps/chosen": -296.0,
      "logps/rejected": -592.0,
      "loss": 0.3665,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.765625,
      "rewards/margins": 3.09375,
      "rewards/rejected": -4.875,
      "step": 6790
    },
    {
      "epoch": 0.5116244074937928,
      "grad_norm": 17.148141561468496,
      "learning_rate": 2.834207368628088e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.734375,
      "logps/chosen": -302.0,
      "logps/rejected": -644.0,
      "loss": 0.4133,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.375,
      "rewards/rejected": -5.25,
      "step": 6800
    },
    {
      "epoch": 0.5123767963283424,
      "grad_norm": 20.07875541473972,
      "learning_rate": 2.8276988364055076e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.8125,
      "logps/chosen": -294.0,
      "logps/rejected": -684.0,
      "loss": 0.3685,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.546875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -5.625,
      "step": 6810
    },
    {
      "epoch": 0.5131291851628922,
      "grad_norm": 19.298543916644547,
      "learning_rate": 2.821188043501251e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.078125,
      "logps/chosen": -314.0,
      "logps/rejected": -620.0,
      "loss": 0.4268,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.046875,
      "rewards/margins": 3.125,
      "rewards/rejected": -5.1875,
      "step": 6820
    },
    {
      "epoch": 0.5138815739974418,
      "grad_norm": 24.761013608176373,
      "learning_rate": 2.8146750348310406e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.84375,
      "logps/chosen": -292.0,
      "logps/rejected": -604.0,
      "loss": 0.4209,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 3.28125,
      "rewards/rejected": -4.84375,
      "step": 6830
    },
    {
      "epoch": 0.5146339628319916,
      "grad_norm": 19.517129852967233,
      "learning_rate": 2.8081598553258863e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.734375,
      "logps/chosen": -304.0,
      "logps/rejected": -632.0,
      "loss": 0.3871,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.65625,
      "rewards/margins": 3.375,
      "rewards/rejected": -5.03125,
      "step": 6840
    },
    {
      "epoch": 0.5153863516665412,
      "grad_norm": 20.153680072423505,
      "learning_rate": 2.801642549931773e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -310.0,
      "logps/rejected": -628.0,
      "loss": 0.4019,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.296875,
      "rewards/rejected": -5.03125,
      "step": 6850
    },
    {
      "epoch": 0.516138740501091,
      "grad_norm": 17.86329538331758,
      "learning_rate": 2.7951231636093496e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.828125,
      "logps/chosen": -314.0,
      "logps/rejected": -672.0,
      "loss": 0.3497,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.5625,
      "step": 6860
    },
    {
      "epoch": 0.5168911293356406,
      "grad_norm": 29.09531192893328,
      "learning_rate": 2.788601741333623e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.828125,
      "logps/chosen": -318.0,
      "logps/rejected": -696.0,
      "loss": 0.3725,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.8125,
      "step": 6870
    },
    {
      "epoch": 0.5176435181701904,
      "grad_norm": 20.346466012302507,
      "learning_rate": 2.782078328093646e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.71875,
      "logps/chosen": -282.0,
      "logps/rejected": -668.0,
      "loss": 0.3895,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.375,
      "step": 6880
    },
    {
      "epoch": 0.51839590700474,
      "grad_norm": 28.020407412479013,
      "learning_rate": 2.775552968892205e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.640625,
      "logps/chosen": -292.0,
      "logps/rejected": -608.0,
      "loss": 0.3974,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 3.25,
      "rewards/rejected": -5.0,
      "step": 6890
    },
    {
      "epoch": 0.5191482958392898,
      "grad_norm": 28.982237377017793,
      "learning_rate": 2.76902570874551e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -286.0,
      "logps/rejected": -676.0,
      "loss": 0.4177,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.59375,
      "step": 6900
    },
    {
      "epoch": 0.5199006846738394,
      "grad_norm": 23.13596935563281,
      "learning_rate": 2.7624965926828856e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.8125,
      "logps/chosen": -252.0,
      "logps/rejected": -608.0,
      "loss": 0.4309,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 3.53125,
      "rewards/rejected": -5.0,
      "step": 6910
    },
    {
      "epoch": 0.5206530735083892,
      "grad_norm": 30.61769043138959,
      "learning_rate": 2.7559656657464615e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.765625,
      "logps/chosen": -304.0,
      "logps/rejected": -588.0,
      "loss": 0.471,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 2.96875,
      "rewards/rejected": -4.71875,
      "step": 6920
    },
    {
      "epoch": 0.5214054623429388,
      "grad_norm": 27.662777927127532,
      "learning_rate": 2.7494329729908585e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5625,
      "logps/chosen": -288.0,
      "logps/rejected": -796.0,
      "loss": 0.4196,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 5.0,
      "rewards/rejected": -6.78125,
      "step": 6930
    },
    {
      "epoch": 0.5221578511774885,
      "grad_norm": 20.570309223370433,
      "learning_rate": 2.7428985594828785e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.796875,
      "logps/chosen": -290.0,
      "logps/rejected": -664.0,
      "loss": 0.4452,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.546875,
      "rewards/margins": 3.734375,
      "rewards/rejected": -5.28125,
      "step": 6940
    },
    {
      "epoch": 0.5229102400120382,
      "grad_norm": 19.218502516216013,
      "learning_rate": 2.736362470301195e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.828125,
      "logps/chosen": -280.0,
      "logps/rejected": -576.0,
      "loss": 0.3886,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.7734375,
      "rewards/margins": 3.015625,
      "rewards/rejected": -4.78125,
      "step": 6950
    },
    {
      "epoch": 0.5236626288465879,
      "grad_norm": 20.664634314523642,
      "learning_rate": 2.7298247505360414e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.84375,
      "logps/chosen": -288.0,
      "logps/rejected": -668.0,
      "loss": 0.397,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 3.890625,
      "rewards/rejected": -5.46875,
      "step": 6960
    },
    {
      "epoch": 0.5244150176811376,
      "grad_norm": 26.51965215713236,
      "learning_rate": 2.723285445288902e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.765625,
      "logps/chosen": -282.0,
      "logps/rejected": -604.0,
      "loss": 0.4017,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.59375,
      "rewards/margins": 3.359375,
      "rewards/rejected": -4.96875,
      "step": 6970
    },
    {
      "epoch": 0.5251674065156873,
      "grad_norm": 25.91140483526548,
      "learning_rate": 2.7167445996721954e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.921875,
      "logps/chosen": -316.0,
      "logps/rejected": -632.0,
      "loss": 0.4021,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.8984375,
      "rewards/margins": 3.203125,
      "rewards/rejected": -5.09375,
      "step": 6980
    },
    {
      "epoch": 0.525919795350237,
      "grad_norm": 22.28574384584974,
      "learning_rate": 2.710202258808967e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.765625,
      "logps/chosen": -320.0,
      "logps/rejected": -644.0,
      "loss": 0.3935,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.84375,
      "rewards/margins": 3.40625,
      "rewards/rejected": -5.25,
      "step": 6990
    },
    {
      "epoch": 0.5266721841847867,
      "grad_norm": 25.09782469141051,
      "learning_rate": 2.70365846783258e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.90625,
      "logps/chosen": -316.0,
      "logps/rejected": -676.0,
      "loss": 0.4355,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.609375,
      "rewards/rejected": -5.53125,
      "step": 7000
    },
    {
      "epoch": 0.5266721841847867,
      "eval_logits/chosen": -2.625,
      "eval_logits/rejected": -2.71875,
      "eval_logps/chosen": -316.0,
      "eval_logps/rejected": -620.0,
      "eval_loss": 0.26912862062454224,
      "eval_rewards/accuracies": 0.8821089863777161,
      "eval_rewards/chosen": -1.8359375,
      "eval_rewards/margins": 3.1875,
      "eval_rewards/rejected": -5.03125,
      "eval_runtime": 2668.7832,
      "eval_samples_per_second": 35.407,
      "eval_steps_per_second": 0.553,
      "step": 7000
    },
    {
      "epoch": 0.5274245730193364,
      "grad_norm": 22.977293734326217,
      "learning_rate": 2.6971132718864005e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.828125,
      "logps/chosen": -290.0,
      "logps/rejected": -624.0,
      "loss": 0.4023,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 3.453125,
      "rewards/rejected": -5.0625,
      "step": 7010
    },
    {
      "epoch": 0.5281769618538861,
      "grad_norm": 22.90349494473043,
      "learning_rate": 2.6905667161234844e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.90625,
      "logps/chosen": -284.0,
      "logps/rejected": -600.0,
      "loss": 0.4598,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.6875,
      "rewards/margins": 3.140625,
      "rewards/rejected": -4.8125,
      "step": 7020
    },
    {
      "epoch": 0.5289293506884358,
      "grad_norm": 19.57132334597457,
      "learning_rate": 2.6840188457062725e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.96875,
      "logps/chosen": -332.0,
      "logps/rejected": -628.0,
      "loss": 0.3827,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.765625,
      "rewards/margins": 3.390625,
      "rewards/rejected": -5.15625,
      "step": 7030
    },
    {
      "epoch": 0.5296817395229855,
      "grad_norm": 29.97069812269175,
      "learning_rate": 2.6774697058062713e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.828125,
      "logps/chosen": -312.0,
      "logps/rejected": -660.0,
      "loss": 0.4017,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 3.546875,
      "rewards/rejected": -5.46875,
      "step": 7040
    },
    {
      "epoch": 0.5304341283575352,
      "grad_norm": 19.36586303518725,
      "learning_rate": 2.6709193416037475e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.703125,
      "logps/chosen": -314.0,
      "logps/rejected": -680.0,
      "loss": 0.3965,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.984375,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.5625,
      "step": 7050
    },
    {
      "epoch": 0.5311865171920849,
      "grad_norm": 22.357258717286243,
      "learning_rate": 2.6643677982874133e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.96875,
      "logps/chosen": -298.0,
      "logps/rejected": -668.0,
      "loss": 0.4343,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.75,
      "step": 7060
    },
    {
      "epoch": 0.5319389060266345,
      "grad_norm": 20.832478569602312,
      "learning_rate": 2.657815121054115e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.875,
      "logps/chosen": -314.0,
      "logps/rejected": -656.0,
      "loss": 0.4018,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 3.4375,
      "rewards/rejected": -5.28125,
      "step": 7070
    },
    {
      "epoch": 0.5326912948611843,
      "grad_norm": 23.18753334251661,
      "learning_rate": 2.6512613551085214e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.75,
      "logps/chosen": -332.0,
      "logps/rejected": -628.0,
      "loss": 0.3822,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.90625,
      "rewards/margins": 3.21875,
      "rewards/rejected": -5.125,
      "step": 7080
    },
    {
      "epoch": 0.5334436836957339,
      "grad_norm": 18.890243773000634,
      "learning_rate": 2.6447065456628105e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.640625,
      "logps/chosen": -304.0,
      "logps/rejected": -704.0,
      "loss": 0.3921,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.78125,
      "rewards/margins": 4.03125,
      "rewards/rejected": -5.8125,
      "step": 7090
    },
    {
      "epoch": 0.5341960725302837,
      "grad_norm": 19.939439436623125,
      "learning_rate": 2.6381507379363627e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.90625,
      "logps/chosen": -294.0,
      "logps/rejected": -624.0,
      "loss": 0.4083,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.59375,
      "rewards/margins": 3.46875,
      "rewards/rejected": -5.0625,
      "step": 7100
    },
    {
      "epoch": 0.5349484613648333,
      "grad_norm": 25.401112078554707,
      "learning_rate": 2.6315939771554407e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.734375,
      "logps/chosen": -290.0,
      "logps/rejected": -628.0,
      "loss": 0.3903,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.125,
      "step": 7110
    },
    {
      "epoch": 0.5357008501993831,
      "grad_norm": 26.8888194064635,
      "learning_rate": 2.6250363085528867e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.578125,
      "logps/chosen": -286.0,
      "logps/rejected": -608.0,
      "loss": 0.4263,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.1875,
      "rewards/rejected": -4.9375,
      "step": 7120
    },
    {
      "epoch": 0.5364532390339327,
      "grad_norm": 17.71383333023575,
      "learning_rate": 2.618477777367801e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -302.0,
      "logps/rejected": -664.0,
      "loss": 0.4291,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 3.828125,
      "rewards/rejected": -5.5625,
      "step": 7130
    },
    {
      "epoch": 0.5372056278684825,
      "grad_norm": 23.704175431704233,
      "learning_rate": 2.6119184288452377e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.84375,
      "logps/chosen": -280.0,
      "logps/rejected": -648.0,
      "loss": 0.3916,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.28125,
      "step": 7140
    },
    {
      "epoch": 0.5379580167030321,
      "grad_norm": 20.842336883969544,
      "learning_rate": 2.6053583082358887e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.796875,
      "logps/chosen": -274.0,
      "logps/rejected": -648.0,
      "loss": 0.3864,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.5625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.34375,
      "step": 7150
    },
    {
      "epoch": 0.5387104055375819,
      "grad_norm": 32.13810909393088,
      "learning_rate": 2.598797460795772e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.90625,
      "logps/chosen": -306.0,
      "logps/rejected": -604.0,
      "loss": 0.4116,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.78125,
      "rewards/margins": 3.234375,
      "rewards/rejected": -5.0,
      "step": 7160
    },
    {
      "epoch": 0.5394627943721315,
      "grad_norm": 21.40514714095739,
      "learning_rate": 2.5922359317859195e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.6875,
      "logps/chosen": -282.0,
      "logps/rejected": -648.0,
      "loss": 0.4028,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.59375,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.34375,
      "step": 7170
    },
    {
      "epoch": 0.5402151832066812,
      "grad_norm": 29.467520119982865,
      "learning_rate": 2.585673766472065e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.65625,
      "logps/chosen": -314.0,
      "logps/rejected": -680.0,
      "loss": 0.3919,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.59375,
      "step": 7180
    },
    {
      "epoch": 0.5409675720412309,
      "grad_norm": 24.705768368043508,
      "learning_rate": 2.5791110101243337e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.65625,
      "logps/chosen": -316.0,
      "logps/rejected": -684.0,
      "loss": 0.4094,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.0625,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.75,
      "step": 7190
    },
    {
      "epoch": 0.5417199608757806,
      "grad_norm": 20.494807320385224,
      "learning_rate": 2.5725477080169243e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.703125,
      "logps/chosen": -320.0,
      "logps/rejected": -648.0,
      "loss": 0.4182,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -5.28125,
      "step": 7200
    },
    {
      "epoch": 0.5424723497103303,
      "grad_norm": 18.31822009354534,
      "learning_rate": 2.565983905427806e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.84375,
      "logps/chosen": -302.0,
      "logps/rejected": -636.0,
      "loss": 0.3996,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.3125,
      "rewards/rejected": -5.21875,
      "step": 7210
    },
    {
      "epoch": 0.54322473854488,
      "grad_norm": 25.618424102375485,
      "learning_rate": 2.559419647638395e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.71875,
      "logps/chosen": -300.0,
      "logps/rejected": -632.0,
      "loss": 0.4153,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.40625,
      "rewards/rejected": -5.28125,
      "step": 7220
    },
    {
      "epoch": 0.5439771273794297,
      "grad_norm": 18.82389803275027,
      "learning_rate": 2.5528549799332515e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.4375,
      "logps/chosen": -318.0,
      "logps/rejected": -684.0,
      "loss": 0.4077,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.09375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.78125,
      "step": 7230
    },
    {
      "epoch": 0.5447295162139794,
      "grad_norm": 22.814648421244293,
      "learning_rate": 2.5462899475997617e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.8125,
      "logps/chosen": -306.0,
      "logps/rejected": -680.0,
      "loss": 0.424,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.859375,
      "rewards/rejected": -5.75,
      "step": 7240
    },
    {
      "epoch": 0.5454819050485291,
      "grad_norm": 25.699747931066316,
      "learning_rate": 2.5397245959278284e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.6875,
      "logps/chosen": -312.0,
      "logps/rejected": -676.0,
      "loss": 0.3919,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 3.625,
      "rewards/rejected": -5.5,
      "step": 7250
    },
    {
      "epoch": 0.5462342938830788,
      "grad_norm": 25.42826461616508,
      "learning_rate": 2.533158970209558e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.609375,
      "logps/chosen": -342.0,
      "logps/rejected": -692.0,
      "loss": 0.4291,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.59375,
      "rewards/rejected": -5.625,
      "step": 7260
    },
    {
      "epoch": 0.5469866827176285,
      "grad_norm": 24.055970268302982,
      "learning_rate": 2.526593115738945e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.859375,
      "logps/chosen": -340.0,
      "logps/rejected": -644.0,
      "loss": 0.3838,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.125,
      "rewards/margins": 3.25,
      "rewards/rejected": -5.375,
      "step": 7270
    },
    {
      "epoch": 0.5477390715521782,
      "grad_norm": 21.628354047326,
      "learning_rate": 2.5200270778115634e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.546875,
      "logps/chosen": -294.0,
      "logps/rejected": -668.0,
      "loss": 0.3935,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 3.984375,
      "rewards/rejected": -5.59375,
      "step": 7280
    },
    {
      "epoch": 0.5484914603867279,
      "grad_norm": 24.595644071611332,
      "learning_rate": 2.513460901724253e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.671875,
      "logps/chosen": -320.0,
      "logps/rejected": -676.0,
      "loss": 0.4063,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.625,
      "step": 7290
    },
    {
      "epoch": 0.5492438492212776,
      "grad_norm": 25.463586584905848,
      "learning_rate": 2.5068946327748073e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.703125,
      "logps/chosen": -276.0,
      "logps/rejected": -688.0,
      "loss": 0.3736,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.578125,
      "rewards/margins": 3.96875,
      "rewards/rejected": -5.5625,
      "step": 7300
    },
    {
      "epoch": 0.5499962380558272,
      "grad_norm": 21.194954952002302,
      "learning_rate": 2.5003283162616585e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.609375,
      "logps/chosen": -308.0,
      "logps/rejected": -644.0,
      "loss": 0.3869,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.734375,
      "rewards/margins": 3.484375,
      "rewards/rejected": -5.21875,
      "step": 7310
    },
    {
      "epoch": 0.550748626890377,
      "grad_norm": 34.48173594340019,
      "learning_rate": 2.493761997483569e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.671875,
      "logps/chosen": -310.0,
      "logps/rejected": -672.0,
      "loss": 0.4576,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.953125,
      "rewards/margins": 3.53125,
      "rewards/rejected": -5.5,
      "step": 7320
    },
    {
      "epoch": 0.5515010157249266,
      "grad_norm": 27.84779636805644,
      "learning_rate": 2.4871957217393155e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.703125,
      "logps/chosen": -328.0,
      "logps/rejected": -672.0,
      "loss": 0.4163,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.90625,
      "rewards/margins": 3.546875,
      "rewards/rejected": -5.4375,
      "step": 7330
    },
    {
      "epoch": 0.5522534045594764,
      "grad_norm": 21.630255487106542,
      "learning_rate": 2.480629534327378e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.78125,
      "logps/chosen": -294.0,
      "logps/rejected": -656.0,
      "loss": 0.3871,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.6875,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.375,
      "step": 7340
    },
    {
      "epoch": 0.553005793394026,
      "grad_norm": 22.683603350482848,
      "learning_rate": 2.474063480545628e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.8125,
      "logps/chosen": -294.0,
      "logps/rejected": -636.0,
      "loss": 0.3945,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8125,
      "rewards/margins": 3.4375,
      "rewards/rejected": -5.25,
      "step": 7350
    },
    {
      "epoch": 0.5537581822285758,
      "grad_norm": 20.50275273363717,
      "learning_rate": 2.4674976056910136e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.671875,
      "logps/chosen": -310.0,
      "logps/rejected": -680.0,
      "loss": 0.3998,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.65625,
      "step": 7360
    },
    {
      "epoch": 0.5545105710631254,
      "grad_norm": 24.980022068412268,
      "learning_rate": 2.460931955059251e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.984375,
      "logps/chosen": -296.0,
      "logps/rejected": -688.0,
      "loss": 0.3959,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.84375,
      "rewards/rejected": -5.8125,
      "step": 7370
    },
    {
      "epoch": 0.5552629598976752,
      "grad_norm": 21.95418413812032,
      "learning_rate": 2.4543665739445054e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.921875,
      "logps/chosen": -292.0,
      "logps/rejected": -664.0,
      "loss": 0.3829,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.5,
      "step": 7380
    },
    {
      "epoch": 0.5560153487322248,
      "grad_norm": 27.416702373004696,
      "learning_rate": 2.447801507639087e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.875,
      "logps/chosen": -308.0,
      "logps/rejected": -656.0,
      "loss": 0.4003,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.90625,
      "rewards/margins": 3.4375,
      "rewards/rejected": -5.34375,
      "step": 7390
    },
    {
      "epoch": 0.5567677375667746,
      "grad_norm": 37.630557657693316,
      "learning_rate": 2.4412368014331326e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.796875,
      "logps/chosen": -282.0,
      "logps/rejected": -664.0,
      "loss": 0.4592,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 3.734375,
      "rewards/rejected": -5.5625,
      "step": 7400
    },
    {
      "epoch": 0.5575201264013242,
      "grad_norm": 20.87003212462832,
      "learning_rate": 2.434672500614294e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.6875,
      "logps/chosen": -296.0,
      "logps/rejected": -628.0,
      "loss": 0.4315,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.375,
      "rewards/rejected": -5.125,
      "step": 7410
    },
    {
      "epoch": 0.5582725152358738,
      "grad_norm": 26.38914402902215,
      "learning_rate": 2.428108650467427e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.8125,
      "logps/chosen": -342.0,
      "logps/rejected": -664.0,
      "loss": 0.4258,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.078125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -5.4375,
      "step": 7420
    },
    {
      "epoch": 0.5590249040704236,
      "grad_norm": 23.44526532904103,
      "learning_rate": 2.4215452962742795e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -310.0,
      "logps/rejected": -696.0,
      "loss": 0.372,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.875,
      "rewards/rejected": -5.8125,
      "step": 7430
    },
    {
      "epoch": 0.5597772929049732,
      "grad_norm": 25.571704008704444,
      "learning_rate": 2.4149824833131766e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -3.09375,
      "logps/chosen": -298.0,
      "logps/rejected": -668.0,
      "loss": 0.3739,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 3.859375,
      "rewards/rejected": -5.71875,
      "step": 7440
    },
    {
      "epoch": 0.560529681739523,
      "grad_norm": 25.35525678205713,
      "learning_rate": 2.408420256858709e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.859375,
      "logps/chosen": -314.0,
      "logps/rejected": -640.0,
      "loss": 0.394,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.046875,
      "rewards/margins": 3.21875,
      "rewards/rejected": -5.25,
      "step": 7450
    },
    {
      "epoch": 0.5612820705740726,
      "grad_norm": 22.835843654170773,
      "learning_rate": 2.4018586621814245e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.765625,
      "logps/chosen": -316.0,
      "logps/rejected": -672.0,
      "loss": 0.3726,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 3.578125,
      "rewards/rejected": -5.53125,
      "step": 7460
    },
    {
      "epoch": 0.5620344594086224,
      "grad_norm": 21.692389156917873,
      "learning_rate": 2.395297744547507e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.6875,
      "logps/chosen": -318.0,
      "logps/rejected": -628.0,
      "loss": 0.447,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.0,
      "rewards/margins": 3.125,
      "rewards/rejected": -5.125,
      "step": 7470
    },
    {
      "epoch": 0.562786848243172,
      "grad_norm": 21.99906476747082,
      "learning_rate": 2.3887375492184754e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.75,
      "logps/chosen": -316.0,
      "logps/rejected": -640.0,
      "loss": 0.3668,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.328125,
      "rewards/rejected": -5.25,
      "step": 7480
    },
    {
      "epoch": 0.5635392370777218,
      "grad_norm": 22.706129994141087,
      "learning_rate": 2.3821781214508624e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.84375,
      "logps/chosen": -320.0,
      "logps/rejected": -624.0,
      "loss": 0.4321,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.171875,
      "rewards/rejected": -4.96875,
      "step": 7490
    },
    {
      "epoch": 0.5642916259122714,
      "grad_norm": 20.552659706400647,
      "learning_rate": 2.3756195064959062e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -290.0,
      "logps/rejected": -632.0,
      "loss": 0.3753,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.609375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -5.21875,
      "step": 7500
    },
    {
      "epoch": 0.5650440147468212,
      "grad_norm": 19.357663809784178,
      "learning_rate": 2.369061749599238e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.671875,
      "logps/chosen": -354.0,
      "logps/rejected": -676.0,
      "loss": 0.3867,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.125,
      "rewards/margins": 3.390625,
      "rewards/rejected": -5.5,
      "step": 7510
    },
    {
      "epoch": 0.5657964035813708,
      "grad_norm": 34.41770622065713,
      "learning_rate": 2.36250489600057e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.8125,
      "logps/chosen": -290.0,
      "logps/rejected": -608.0,
      "loss": 0.3992,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 3.359375,
      "rewards/rejected": -4.96875,
      "step": 7520
    },
    {
      "epoch": 0.5665487924159206,
      "grad_norm": 25.04319794179983,
      "learning_rate": 2.3559489909333812e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.8125,
      "logps/chosen": -308.0,
      "logps/rejected": -656.0,
      "loss": 0.3663,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.453125,
      "rewards/rejected": -5.375,
      "step": 7530
    },
    {
      "epoch": 0.5673011812504702,
      "grad_norm": 28.05893284044512,
      "learning_rate": 2.3493940796246088e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -3.0,
      "logps/chosen": -288.0,
      "logps/rejected": -668.0,
      "loss": 0.4002,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.828125,
      "rewards/margins": 3.625,
      "rewards/rejected": -5.4375,
      "step": 7540
    },
    {
      "epoch": 0.5680535700850199,
      "grad_norm": 22.750449807879015,
      "learning_rate": 2.342840207294335e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -3.0,
      "logps/chosen": -304.0,
      "logps/rejected": -676.0,
      "loss": 0.3616,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.84375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.625,
      "step": 7550
    },
    {
      "epoch": 0.5688059589195696,
      "grad_norm": 20.77916482113924,
      "learning_rate": 2.33628741915547e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.90625,
      "logps/chosen": -286.0,
      "logps/rejected": -696.0,
      "loss": 0.3338,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.828125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -5.875,
      "step": 7560
    },
    {
      "epoch": 0.5695583477541193,
      "grad_norm": 24.377193433941216,
      "learning_rate": 2.32973576041345e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.75,
      "logps/chosen": -306.0,
      "logps/rejected": -732.0,
      "loss": 0.4057,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.765625,
      "rewards/margins": 4.3125,
      "rewards/rejected": -6.0625,
      "step": 7570
    },
    {
      "epoch": 0.570310736588669,
      "grad_norm": 35.62606238955454,
      "learning_rate": 2.3231852762659165e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.8125,
      "logps/chosen": -276.0,
      "logps/rejected": -648.0,
      "loss": 0.4183,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.578125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -5.375,
      "step": 7580
    },
    {
      "epoch": 0.5710631254232187,
      "grad_norm": 19.749072736167527,
      "learning_rate": 2.3166360119024094e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.71875,
      "logps/chosen": -330.0,
      "logps/rejected": -688.0,
      "loss": 0.3382,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.65625,
      "step": 7590
    },
    {
      "epoch": 0.5718155142577684,
      "grad_norm": 16.050513911138722,
      "learning_rate": 2.310088012504053e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.875,
      "logps/chosen": -292.0,
      "logps/rejected": -648.0,
      "loss": 0.4024,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.78125,
      "rewards/margins": 3.484375,
      "rewards/rejected": -5.25,
      "step": 7600
    },
    {
      "epoch": 0.5725679030923181,
      "grad_norm": 22.273310037634246,
      "learning_rate": 2.303541323243246e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.96875,
      "logps/chosen": -292.0,
      "logps/rejected": -632.0,
      "loss": 0.3861,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.25,
      "step": 7610
    },
    {
      "epoch": 0.5733202919268678,
      "grad_norm": 22.440112551848898,
      "learning_rate": 2.2969959892833474e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.78125,
      "logps/chosen": -294.0,
      "logps/rejected": -680.0,
      "loss": 0.3962,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.53125,
      "step": 7620
    },
    {
      "epoch": 0.5740726807614175,
      "grad_norm": 22.56707289583169,
      "learning_rate": 2.290452055778367e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.890625,
      "logps/chosen": -318.0,
      "logps/rejected": -664.0,
      "loss": 0.4131,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.59375,
      "step": 7630
    },
    {
      "epoch": 0.5748250695959672,
      "grad_norm": 22.864164171604493,
      "learning_rate": 2.2839095678726553e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.890625,
      "logps/chosen": -270.0,
      "logps/rejected": -644.0,
      "loss": 0.4052,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.578125,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.21875,
      "step": 7640
    },
    {
      "epoch": 0.5755774584305169,
      "grad_norm": 18.605730725206673,
      "learning_rate": 2.2773685707005863e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.671875,
      "logps/chosen": -352.0,
      "logps/rejected": -680.0,
      "loss": 0.3847,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.140625,
      "rewards/margins": 3.375,
      "rewards/rejected": -5.5,
      "step": 7650
    },
    {
      "epoch": 0.5763298472650666,
      "grad_norm": 24.69493710044081,
      "learning_rate": 2.270829109386253e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.59375,
      "logps/chosen": -312.0,
      "logps/rejected": -644.0,
      "loss": 0.4211,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8125,
      "rewards/margins": 3.46875,
      "rewards/rejected": -5.28125,
      "step": 7660
    },
    {
      "epoch": 0.5770822360996163,
      "grad_norm": 17.71415233422055,
      "learning_rate": 2.2642912290431518e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.65625,
      "logps/chosen": -290.0,
      "logps/rejected": -684.0,
      "loss": 0.4022,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 4.03125,
      "rewards/rejected": -5.71875,
      "step": 7670
    },
    {
      "epoch": 0.5778346249341659,
      "grad_norm": 13.921653158100282,
      "learning_rate": 2.257754974773873e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.734375,
      "logps/chosen": -316.0,
      "logps/rejected": -732.0,
      "loss": 0.4096,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.859375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -6.0,
      "step": 7680
    },
    {
      "epoch": 0.5785870137687157,
      "grad_norm": 24.04735794686543,
      "learning_rate": 2.2512203916697892e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.859375,
      "logps/chosen": -296.0,
      "logps/rejected": -644.0,
      "loss": 0.3801,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.21875,
      "step": 7690
    },
    {
      "epoch": 0.5793394026032653,
      "grad_norm": 20.301822241700357,
      "learning_rate": 2.2446875248107432e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.75,
      "logps/chosen": -318.0,
      "logps/rejected": -708.0,
      "loss": 0.4057,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.921875,
      "rewards/rejected": -5.9375,
      "step": 7700
    },
    {
      "epoch": 0.5800917914378151,
      "grad_norm": 21.742008933771693,
      "learning_rate": 2.2381564192647396e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -300.0,
      "logps/rejected": -644.0,
      "loss": 0.4066,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.765625,
      "rewards/margins": 3.484375,
      "rewards/rejected": -5.25,
      "step": 7710
    },
    {
      "epoch": 0.5808441802723647,
      "grad_norm": 19.925863573964538,
      "learning_rate": 2.231627120087631e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.625,
      "logps/chosen": -278.0,
      "logps/rejected": -712.0,
      "loss": 0.3684,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 4.34375,
      "rewards/rejected": -5.96875,
      "step": 7720
    },
    {
      "epoch": 0.5815965691069145,
      "grad_norm": 24.569930718816003,
      "learning_rate": 2.2250996723228104e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -3.03125,
      "logps/chosen": -326.0,
      "logps/rejected": -704.0,
      "loss": 0.3883,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.09375,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.84375,
      "step": 7730
    },
    {
      "epoch": 0.5823489579414641,
      "grad_norm": 24.218426984518082,
      "learning_rate": 2.2185741210008947e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.703125,
      "logps/chosen": -332.0,
      "logps/rejected": -688.0,
      "loss": 0.3895,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.609375,
      "rewards/rejected": -5.5,
      "step": 7740
    },
    {
      "epoch": 0.5831013467760139,
      "grad_norm": 20.661519727510246,
      "learning_rate": 2.2120505111394224e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.8125,
      "logps/chosen": -336.0,
      "logps/rejected": -676.0,
      "loss": 0.3879,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.453125,
      "rewards/rejected": -5.5,
      "step": 7750
    },
    {
      "epoch": 0.5838537356105635,
      "grad_norm": 25.941191594357846,
      "learning_rate": 2.205528887742535e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.640625,
      "logps/chosen": -340.0,
      "logps/rejected": -716.0,
      "loss": 0.3999,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.953125,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.75,
      "step": 7760
    },
    {
      "epoch": 0.5846061244451133,
      "grad_norm": 20.62723894345592,
      "learning_rate": 2.199009295800672e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5,
      "logps/chosen": -336.0,
      "logps/rejected": -712.0,
      "loss": 0.3612,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.140625,
      "rewards/margins": 3.875,
      "rewards/rejected": -6.03125,
      "step": 7770
    },
    {
      "epoch": 0.5853585132796629,
      "grad_norm": 30.926884351341485,
      "learning_rate": 2.192491780290259e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.859375,
      "logps/chosen": -352.0,
      "logps/rejected": -712.0,
      "loss": 0.377,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.171875,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.90625,
      "step": 7780
    },
    {
      "epoch": 0.5861109021142126,
      "grad_norm": 34.94246825760253,
      "learning_rate": 2.1859763861733947e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.78125,
      "logps/chosen": -290.0,
      "logps/rejected": -608.0,
      "loss": 0.4128,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.75,
      "rewards/margins": 3.359375,
      "rewards/rejected": -5.125,
      "step": 7790
    },
    {
      "epoch": 0.5868632909487623,
      "grad_norm": 26.599011675206718,
      "learning_rate": 2.179463158397545e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.78125,
      "logps/chosen": -296.0,
      "logps/rejected": -652.0,
      "loss": 0.4084,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.84375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -5.4375,
      "step": 7800
    },
    {
      "epoch": 0.587615679783312,
      "grad_norm": 26.607490289331373,
      "learning_rate": 2.1729521418952304e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.90625,
      "logps/chosen": -270.0,
      "logps/rejected": -664.0,
      "loss": 0.3753,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.515625,
      "rewards/margins": 3.96875,
      "rewards/rejected": -5.5,
      "step": 7810
    },
    {
      "epoch": 0.5883680686178617,
      "grad_norm": 23.263643953910073,
      "learning_rate": 2.166443381583718e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.78125,
      "logps/chosen": -318.0,
      "logps/rejected": -684.0,
      "loss": 0.3569,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.0625,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.5625,
      "step": 7820
    },
    {
      "epoch": 0.5891204574524114,
      "grad_norm": 26.658786067934898,
      "learning_rate": 2.1599369223647068e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.875,
      "logps/chosen": -316.0,
      "logps/rejected": -612.0,
      "loss": 0.4324,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.1875,
      "rewards/rejected": -5.125,
      "step": 7830
    },
    {
      "epoch": 0.5898728462869611,
      "grad_norm": 23.480417468927673,
      "learning_rate": 2.1534328091240238e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.890625,
      "logps/chosen": -308.0,
      "logps/rejected": -648.0,
      "loss": 0.4125,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.375,
      "rewards/rejected": -5.21875,
      "step": 7840
    },
    {
      "epoch": 0.5906252351215108,
      "grad_norm": 19.792817263244977,
      "learning_rate": 2.1469310867313118e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.703125,
      "logps/chosen": -338.0,
      "logps/rejected": -700.0,
      "loss": 0.3957,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.921875,
      "rewards/margins": 3.890625,
      "rewards/rejected": -5.8125,
      "step": 7850
    },
    {
      "epoch": 0.5913776239560605,
      "grad_norm": 19.247848585205535,
      "learning_rate": 2.1404318000397192e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.703125,
      "logps/chosen": -300.0,
      "logps/rejected": -628.0,
      "loss": 0.3862,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.46875,
      "rewards/rejected": -5.21875,
      "step": 7860
    },
    {
      "epoch": 0.5921300127906102,
      "grad_norm": 20.417820245007572,
      "learning_rate": 2.1339349938855929e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.875,
      "logps/chosen": -264.0,
      "logps/rejected": -608.0,
      "loss": 0.3782,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 3.484375,
      "rewards/rejected": -5.09375,
      "step": 7870
    },
    {
      "epoch": 0.5928824016251599,
      "grad_norm": 25.479098904987865,
      "learning_rate": 2.127440713088165e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.859375,
      "logps/chosen": -306.0,
      "logps/rejected": -672.0,
      "loss": 0.4133,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.734375,
      "rewards/rejected": -5.53125,
      "step": 7880
    },
    {
      "epoch": 0.5936347904597096,
      "grad_norm": 25.519424180802368,
      "learning_rate": 2.1209490024492477e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.875,
      "logps/chosen": -298.0,
      "logps/rejected": -640.0,
      "loss": 0.4076,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.78125,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.34375,
      "step": 7890
    },
    {
      "epoch": 0.5943871792942593,
      "grad_norm": 33.85814210164457,
      "learning_rate": 2.114459906752923e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.9375,
      "logps/chosen": -322.0,
      "logps/rejected": -656.0,
      "loss": 0.394,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.484375,
      "rewards/rejected": -5.4375,
      "step": 7900
    },
    {
      "epoch": 0.595139568128809,
      "grad_norm": 20.80632666284628,
      "learning_rate": 2.1079734707652315e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.703125,
      "logps/chosen": -310.0,
      "logps/rejected": -660.0,
      "loss": 0.4351,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.5,
      "step": 7910
    },
    {
      "epoch": 0.5958919569633586,
      "grad_norm": 22.12608229590259,
      "learning_rate": 2.101489739233867e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.671875,
      "logps/chosen": -286.0,
      "logps/rejected": -692.0,
      "loss": 0.3728,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.640625,
      "rewards/margins": 4.09375,
      "rewards/rejected": -5.75,
      "step": 7920
    },
    {
      "epoch": 0.5966443457979084,
      "grad_norm": 15.08923873042706,
      "learning_rate": 2.0950087568878643e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.859375,
      "logps/chosen": -296.0,
      "logps/rejected": -676.0,
      "loss": 0.4006,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.7734375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -5.625,
      "step": 7930
    },
    {
      "epoch": 0.597396734632458,
      "grad_norm": 23.998716399560436,
      "learning_rate": 2.088530568437295e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.84375,
      "logps/chosen": -302.0,
      "logps/rejected": -680.0,
      "loss": 0.4159,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.890625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.6875,
      "step": 7940
    },
    {
      "epoch": 0.5981491234670078,
      "grad_norm": 23.26481597718282,
      "learning_rate": 2.0820552185729553e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.78125,
      "logps/chosen": -314.0,
      "logps/rejected": -644.0,
      "loss": 0.3868,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.3125,
      "step": 7950
    },
    {
      "epoch": 0.5989015123015574,
      "grad_norm": 23.43169082540461,
      "learning_rate": 2.0755827519660585e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.703125,
      "logps/chosen": -298.0,
      "logps/rejected": -680.0,
      "loss": 0.3907,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.671875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -5.75,
      "step": 7960
    },
    {
      "epoch": 0.5996539011361072,
      "grad_norm": 26.805003207194037,
      "learning_rate": 2.069113213267928e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.90625,
      "logps/chosen": -310.0,
      "logps/rejected": -628.0,
      "loss": 0.357,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 3.25,
      "rewards/rejected": -5.125,
      "step": 7970
    },
    {
      "epoch": 0.6004062899706568,
      "grad_norm": 23.120108497111673,
      "learning_rate": 2.062646647109688e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.90625,
      "logps/chosen": -278.0,
      "logps/rejected": -636.0,
      "loss": 0.4013,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.3125,
      "step": 7980
    },
    {
      "epoch": 0.6011586788052066,
      "grad_norm": 27.94795959068527,
      "learning_rate": 2.0561830981019582e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.8125,
      "logps/chosen": -322.0,
      "logps/rejected": -680.0,
      "loss": 0.3937,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 3.65625,
      "rewards/rejected": -5.65625,
      "step": 7990
    },
    {
      "epoch": 0.6019110676397562,
      "grad_norm": 14.869048608425004,
      "learning_rate": 2.04972261083454e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.484375,
      "logps/chosen": -320.0,
      "logps/rejected": -692.0,
      "loss": 0.3471,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.5625,
      "step": 8000
    },
    {
      "epoch": 0.6019110676397562,
      "eval_logits/chosen": -2.625,
      "eval_logits/rejected": -2.703125,
      "eval_logps/chosen": -322.0,
      "eval_logps/rejected": -624.0,
      "eval_loss": 0.265441358089447,
      "eval_rewards/accuracies": 0.8835477232933044,
      "eval_rewards/chosen": -1.8984375,
      "eval_rewards/margins": 3.15625,
      "eval_rewards/rejected": -5.0625,
      "eval_runtime": 2669.0007,
      "eval_samples_per_second": 35.404,
      "eval_steps_per_second": 0.553,
      "step": 8000
    },
    {
      "epoch": 0.602663456474306,
      "grad_norm": 26.35099901086607,
      "learning_rate": 2.0432652298761176e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.78125,
      "logps/chosen": -304.0,
      "logps/rejected": -640.0,
      "loss": 0.4165,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 3.359375,
      "rewards/rejected": -5.15625,
      "step": 8010
    },
    {
      "epoch": 0.6034158453088556,
      "grad_norm": 16.744262129912247,
      "learning_rate": 2.0368109997739415e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.703125,
      "logps/chosen": -278.0,
      "logps/rejected": -660.0,
      "loss": 0.3481,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.609375,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.3125,
      "step": 8020
    },
    {
      "epoch": 0.6041682341434053,
      "grad_norm": 23.87878334304586,
      "learning_rate": 2.0303599650535283e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.40625,
      "logps/chosen": -322.0,
      "logps/rejected": -664.0,
      "loss": 0.4043,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.4375,
      "step": 8030
    },
    {
      "epoch": 0.604920622977955,
      "grad_norm": 18.911479491092393,
      "learning_rate": 2.0239121702183505e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.828125,
      "logps/chosen": -274.0,
      "logps/rejected": -648.0,
      "loss": 0.3864,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.375,
      "step": 8040
    },
    {
      "epoch": 0.6056730118125047,
      "grad_norm": 18.91456939330801,
      "learning_rate": 2.017467659749528e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.78125,
      "logps/chosen": -344.0,
      "logps/rejected": -656.0,
      "loss": 0.3916,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 3.578125,
      "rewards/rejected": -5.4375,
      "step": 8050
    },
    {
      "epoch": 0.6064254006470544,
      "grad_norm": 21.993161710057038,
      "learning_rate": 2.011026478105525e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.8125,
      "logps/chosen": -308.0,
      "logps/rejected": -656.0,
      "loss": 0.3896,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 3.53125,
      "rewards/rejected": -5.3125,
      "step": 8060
    },
    {
      "epoch": 0.6071777894816041,
      "grad_norm": 12.9047557036142,
      "learning_rate": 2.004588669721841e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.703125,
      "logps/chosen": -304.0,
      "logps/rejected": -668.0,
      "loss": 0.3818,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.4375,
      "step": 8070
    },
    {
      "epoch": 0.6079301783161538,
      "grad_norm": 20.762711425067078,
      "learning_rate": 1.9981542790107032e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.84375,
      "logps/chosen": -312.0,
      "logps/rejected": -672.0,
      "loss": 0.383,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.34375,
      "step": 8080
    },
    {
      "epoch": 0.6086825671507035,
      "grad_norm": 29.172213302679097,
      "learning_rate": 1.9917233503607625e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.984375,
      "logps/chosen": -290.0,
      "logps/rejected": -704.0,
      "loss": 0.344,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.796875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -5.9375,
      "step": 8090
    },
    {
      "epoch": 0.6094349559852532,
      "grad_norm": 22.58456100715218,
      "learning_rate": 1.9852959281367872e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.875,
      "logps/chosen": -328.0,
      "logps/rejected": -664.0,
      "loss": 0.3789,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.109375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -5.53125,
      "step": 8100
    },
    {
      "epoch": 0.6101873448198029,
      "grad_norm": 19.739883895813595,
      "learning_rate": 1.9788720566793527e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.953125,
      "logps/chosen": -304.0,
      "logps/rejected": -656.0,
      "loss": 0.3791,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.4375,
      "step": 8110
    },
    {
      "epoch": 0.6109397336543526,
      "grad_norm": 25.883627085963223,
      "learning_rate": 1.9724517803045417e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.734375,
      "logps/chosen": -304.0,
      "logps/rejected": -652.0,
      "loss": 0.4018,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.890625,
      "rewards/margins": 3.3125,
      "rewards/rejected": -5.21875,
      "step": 8120
    },
    {
      "epoch": 0.6116921224889023,
      "grad_norm": 26.432719129497084,
      "learning_rate": 1.966035143303636e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.6875,
      "logps/chosen": -340.0,
      "logps/rejected": -648.0,
      "loss": 0.3645,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.0,
      "rewards/margins": 3.296875,
      "rewards/rejected": -5.28125,
      "step": 8130
    },
    {
      "epoch": 0.612444511323452,
      "grad_norm": 20.27741347073601,
      "learning_rate": 1.959622189942808e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.71875,
      "logps/chosen": -294.0,
      "logps/rejected": -664.0,
      "loss": 0.3771,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.734375,
      "rewards/margins": 3.625,
      "rewards/rejected": -5.34375,
      "step": 8140
    },
    {
      "epoch": 0.6131969001580017,
      "grad_norm": 21.508856016546858,
      "learning_rate": 1.9532129644628204e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.78125,
      "logps/chosen": -304.0,
      "logps/rejected": -620.0,
      "loss": 0.3981,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 3.1875,
      "rewards/rejected": -5.0625,
      "step": 8150
    },
    {
      "epoch": 0.6139492889925513,
      "grad_norm": 31.196599550814586,
      "learning_rate": 1.946807511078718e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.875,
      "logps/chosen": -306.0,
      "logps/rejected": -640.0,
      "loss": 0.3906,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.6875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.25,
      "step": 8160
    },
    {
      "epoch": 0.6147016778271011,
      "grad_norm": 23.990819530352287,
      "learning_rate": 1.9404058739795217e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.90625,
      "logps/chosen": -316.0,
      "logps/rejected": -672.0,
      "loss": 0.3754,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.625,
      "step": 8170
    },
    {
      "epoch": 0.6154540666616507,
      "grad_norm": 18.646864210036544,
      "learning_rate": 1.9340080973279268e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -308.0,
      "logps/rejected": -696.0,
      "loss": 0.3655,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.84375,
      "rewards/margins": 3.890625,
      "rewards/rejected": -5.75,
      "step": 8180
    },
    {
      "epoch": 0.6162064554962005,
      "grad_norm": 38.98500061554626,
      "learning_rate": 1.9276142252599971e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.875,
      "logps/chosen": -304.0,
      "logps/rejected": -644.0,
      "loss": 0.36,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.578125,
      "rewards/rejected": -5.4375,
      "step": 8190
    },
    {
      "epoch": 0.6169588443307501,
      "grad_norm": 24.385294403015358,
      "learning_rate": 1.9212243018848572e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.75,
      "logps/chosen": -302.0,
      "logps/rejected": -792.0,
      "loss": 0.3552,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 4.875,
      "rewards/rejected": -6.6875,
      "step": 8200
    },
    {
      "epoch": 0.6177112331652999,
      "grad_norm": 24.383430645615675,
      "learning_rate": 1.9148383712843946e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.875,
      "logps/chosen": -324.0,
      "logps/rejected": -656.0,
      "loss": 0.3793,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.4375,
      "step": 8210
    },
    {
      "epoch": 0.6184636219998495,
      "grad_norm": 27.79916296347552,
      "learning_rate": 1.908456477512949e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.578125,
      "logps/chosen": -312.0,
      "logps/rejected": -680.0,
      "loss": 0.3841,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.59375,
      "step": 8220
    },
    {
      "epoch": 0.6192160108343993,
      "grad_norm": 27.365603637256232,
      "learning_rate": 1.9020786645970132e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.484375,
      "logps/chosen": -308.0,
      "logps/rejected": -668.0,
      "loss": 0.433,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.46875,
      "step": 8230
    },
    {
      "epoch": 0.6199683996689489,
      "grad_norm": 25.835973721024285,
      "learning_rate": 1.8957049765349275e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.609375,
      "logps/chosen": -296.0,
      "logps/rejected": -656.0,
      "loss": 0.3818,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.46875,
      "step": 8240
    },
    {
      "epoch": 0.6207207885034987,
      "grad_norm": 26.163007294745185,
      "learning_rate": 1.8893354572965764e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.796875,
      "logps/chosen": -302.0,
      "logps/rejected": -656.0,
      "loss": 0.3926,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.015625,
      "rewards/margins": 3.4375,
      "rewards/rejected": -5.4375,
      "step": 8250
    },
    {
      "epoch": 0.6214731773380483,
      "grad_norm": 24.551369578929496,
      "learning_rate": 1.882970150823083e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.6875,
      "logps/chosen": -320.0,
      "logps/rejected": -676.0,
      "loss": 0.3934,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.921875,
      "rewards/margins": 3.53125,
      "rewards/rejected": -5.46875,
      "step": 8260
    },
    {
      "epoch": 0.6222255661725979,
      "grad_norm": 18.78599036631989,
      "learning_rate": 1.8766091010265107e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.78125,
      "logps/chosen": -310.0,
      "logps/rejected": -676.0,
      "loss": 0.3531,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.8125,
      "rewards/margins": 3.828125,
      "rewards/rejected": -5.625,
      "step": 8270
    },
    {
      "epoch": 0.6229779550071477,
      "grad_norm": 20.277448714588903,
      "learning_rate": 1.870252351789557e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.78125,
      "logps/chosen": -298.0,
      "logps/rejected": -672.0,
      "loss": 0.3975,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.796875,
      "rewards/rejected": -5.59375,
      "step": 8280
    },
    {
      "epoch": 0.6237303438416973,
      "grad_norm": 22.205916456708767,
      "learning_rate": 1.8638999469652485e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.796875,
      "logps/chosen": -298.0,
      "logps/rejected": -652.0,
      "loss": 0.4101,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.40625,
      "step": 8290
    },
    {
      "epoch": 0.6244827326762471,
      "grad_norm": 22.510295629559664,
      "learning_rate": 1.8575519303766456e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.765625,
      "logps/chosen": -328.0,
      "logps/rejected": -688.0,
      "loss": 0.3771,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.828125,
      "rewards/rejected": -5.78125,
      "step": 8300
    },
    {
      "epoch": 0.6252351215107967,
      "grad_norm": 23.704735209212245,
      "learning_rate": 1.8512083458165322e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.734375,
      "logps/chosen": -324.0,
      "logps/rejected": -664.0,
      "loss": 0.4085,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.53125,
      "step": 8310
    },
    {
      "epoch": 0.6259875103453465,
      "grad_norm": 32.95376151946613,
      "learning_rate": 1.8448692370471197e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.59375,
      "logps/chosen": -282.0,
      "logps/rejected": -648.0,
      "loss": 0.3978,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 3.59375,
      "rewards/rejected": -5.21875,
      "step": 8320
    },
    {
      "epoch": 0.6267398991798961,
      "grad_norm": 26.89986787631278,
      "learning_rate": 1.838534647799741e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.390625,
      "logps/chosen": -328.0,
      "logps/rejected": -628.0,
      "loss": 0.407,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.171875,
      "rewards/rejected": -5.125,
      "step": 8330
    },
    {
      "epoch": 0.6274922880144459,
      "grad_norm": 16.033999465061115,
      "learning_rate": 1.83220462177455e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5625,
      "logps/chosen": -292.0,
      "logps/rejected": -668.0,
      "loss": 0.4032,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.78125,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.53125,
      "step": 8340
    },
    {
      "epoch": 0.6282446768489955,
      "grad_norm": 23.282075218915782,
      "learning_rate": 1.825879202640222e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -318.0,
      "logps/rejected": -680.0,
      "loss": 0.424,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.90625,
      "rewards/margins": 3.546875,
      "rewards/rejected": -5.46875,
      "step": 8350
    },
    {
      "epoch": 0.6289970656835453,
      "grad_norm": 20.60334656750855,
      "learning_rate": 1.81955843403365e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.6875,
      "logps/chosen": -280.0,
      "logps/rejected": -656.0,
      "loss": 0.3748,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.46875,
      "step": 8360
    },
    {
      "epoch": 0.6297494545180949,
      "grad_norm": 18.186618721446354,
      "learning_rate": 1.8132423595596464e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.703125,
      "logps/chosen": -312.0,
      "logps/rejected": -660.0,
      "loss": 0.3967,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -5.46875,
      "step": 8370
    },
    {
      "epoch": 0.6305018433526447,
      "grad_norm": 25.161552573359092,
      "learning_rate": 1.8069310227906365e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.703125,
      "logps/chosen": -314.0,
      "logps/rejected": -636.0,
      "loss": 0.3962,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.640625,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.1875,
      "step": 8380
    },
    {
      "epoch": 0.6312542321871943,
      "grad_norm": 23.190259982934872,
      "learning_rate": 1.8006244672663644e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.453125,
      "logps/chosen": -296.0,
      "logps/rejected": -636.0,
      "loss": 0.353,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.75,
      "rewards/margins": 3.390625,
      "rewards/rejected": -5.125,
      "step": 8390
    },
    {
      "epoch": 0.632006621021744,
      "grad_norm": 24.725972763747933,
      "learning_rate": 1.794322736493591e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.796875,
      "logps/chosen": -320.0,
      "logps/rejected": -672.0,
      "loss": 0.4143,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.84375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.53125,
      "step": 8400
    },
    {
      "epoch": 0.6327590098562937,
      "grad_norm": 13.275631976664028,
      "learning_rate": 1.7880258739457905e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.890625,
      "logps/chosen": -308.0,
      "logps/rejected": -672.0,
      "loss": 0.4018,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -5.5,
      "step": 8410
    },
    {
      "epoch": 0.6335113986908434,
      "grad_norm": 20.147263140141092,
      "learning_rate": 1.7817339230628553e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.84375,
      "logps/chosen": -332.0,
      "logps/rejected": -664.0,
      "loss": 0.4215,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 3.59375,
      "rewards/rejected": -5.53125,
      "step": 8420
    },
    {
      "epoch": 0.6342637875253931,
      "grad_norm": 24.028278426432678,
      "learning_rate": 1.7754469272507914e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.578125,
      "logps/chosen": -298.0,
      "logps/rejected": -740.0,
      "loss": 0.3672,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.546875,
      "rewards/margins": 4.59375,
      "rewards/rejected": -6.15625,
      "step": 8430
    },
    {
      "epoch": 0.6350161763599428,
      "grad_norm": 26.34342510862242,
      "learning_rate": 1.7691649298814232e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.796875,
      "logps/chosen": -308.0,
      "logps/rejected": -672.0,
      "loss": 0.3736,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.5625,
      "step": 8440
    },
    {
      "epoch": 0.6357685651944925,
      "grad_norm": 27.680397263453877,
      "learning_rate": 1.7628879742920928e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.578125,
      "logps/chosen": -314.0,
      "logps/rejected": -800.0,
      "loss": 0.4143,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 4.59375,
      "rewards/rejected": -6.5625,
      "step": 8450
    },
    {
      "epoch": 0.6365209540290422,
      "grad_norm": 24.44295778247494,
      "learning_rate": 1.7566161037853594e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.640625,
      "logps/chosen": -312.0,
      "logps/rejected": -732.0,
      "loss": 0.3818,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.796875,
      "rewards/margins": 4.3125,
      "rewards/rejected": -6.125,
      "step": 8460
    },
    {
      "epoch": 0.6372733428635919,
      "grad_norm": 27.239371316100744,
      "learning_rate": 1.7503493616287023e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.875,
      "logps/chosen": -302.0,
      "logps/rejected": -688.0,
      "loss": 0.4303,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.84375,
      "rewards/rejected": -5.71875,
      "step": 8470
    },
    {
      "epoch": 0.6380257316981416,
      "grad_norm": 15.847448533094955,
      "learning_rate": 1.7440877910542225e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -330.0,
      "logps/rejected": -712.0,
      "loss": 0.3899,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -5.90625,
      "step": 8480
    },
    {
      "epoch": 0.6387781205326913,
      "grad_norm": 30.65386567082319,
      "learning_rate": 1.7378314352583446e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.671875,
      "logps/chosen": -332.0,
      "logps/rejected": -720.0,
      "loss": 0.3673,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.71875,
      "step": 8490
    },
    {
      "epoch": 0.639530509367241,
      "grad_norm": 27.476236324383382,
      "learning_rate": 1.731580337401517e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.703125,
      "logps/chosen": -318.0,
      "logps/rejected": -700.0,
      "loss": 0.3908,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.890625,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.6875,
      "step": 8500
    },
    {
      "epoch": 0.6402828982017907,
      "grad_norm": 24.863608783572282,
      "learning_rate": 1.7253345406079161e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.84375,
      "logps/chosen": -340.0,
      "logps/rejected": -684.0,
      "loss": 0.4288,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.609375,
      "rewards/rejected": -5.65625,
      "step": 8510
    },
    {
      "epoch": 0.6410352870363404,
      "grad_norm": 25.478305735259354,
      "learning_rate": 1.719094087965148e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.546875,
      "logps/chosen": -326.0,
      "logps/rejected": -700.0,
      "loss": 0.3697,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.65625,
      "step": 8520
    },
    {
      "epoch": 0.64178767587089,
      "grad_norm": 29.840018114598386,
      "learning_rate": 1.7128590225239515e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.78125,
      "logps/chosen": -308.0,
      "logps/rejected": -664.0,
      "loss": 0.4375,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.890625,
      "rewards/margins": 3.65625,
      "rewards/rejected": -5.53125,
      "step": 8530
    },
    {
      "epoch": 0.6425400647054398,
      "grad_norm": 23.012992782968908,
      "learning_rate": 1.7066293872979005e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.453125,
      "logps/chosen": -356.0,
      "logps/rejected": -744.0,
      "loss": 0.3869,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.0625,
      "rewards/margins": 3.953125,
      "rewards/rejected": -6.03125,
      "step": 8540
    },
    {
      "epoch": 0.6432924535399894,
      "grad_norm": 23.136127550122218,
      "learning_rate": 1.7004052252631085e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.765625,
      "logps/chosen": -332.0,
      "logps/rejected": -684.0,
      "loss": 0.3978,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.0625,
      "rewards/margins": 3.546875,
      "rewards/rejected": -5.625,
      "step": 8550
    },
    {
      "epoch": 0.6440448423745392,
      "grad_norm": 24.36367333249792,
      "learning_rate": 1.6941865793579302e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.671875,
      "logps/chosen": -290.0,
      "logps/rejected": -788.0,
      "loss": 0.384,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 4.875,
      "rewards/rejected": -6.625,
      "step": 8560
    },
    {
      "epoch": 0.6447972312090888,
      "grad_norm": 19.744910772583864,
      "learning_rate": 1.687973492482666e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.71875,
      "logps/chosen": -328.0,
      "logps/rejected": -676.0,
      "loss": 0.3831,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.96875,
      "rewards/margins": 3.515625,
      "rewards/rejected": -5.46875,
      "step": 8570
    },
    {
      "epoch": 0.6455496200436386,
      "grad_norm": 19.83249357450793,
      "learning_rate": 1.6817660074992694e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.921875,
      "logps/chosen": -314.0,
      "logps/rejected": -636.0,
      "loss": 0.3529,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.953125,
      "rewards/margins": 3.46875,
      "rewards/rejected": -5.40625,
      "step": 8580
    },
    {
      "epoch": 0.6463020088781882,
      "grad_norm": 20.172476345553978,
      "learning_rate": 1.6755641672310456e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.59375,
      "logps/chosen": -284.0,
      "logps/rejected": -664.0,
      "loss": 0.3699,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.734375,
      "rewards/rejected": -5.59375,
      "step": 8590
    },
    {
      "epoch": 0.647054397712738,
      "grad_norm": 24.458561186607955,
      "learning_rate": 1.6693680144623594e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.828125,
      "logps/chosen": -332.0,
      "logps/rejected": -676.0,
      "loss": 0.3572,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.0625,
      "rewards/margins": 3.53125,
      "rewards/rejected": -5.59375,
      "step": 8600
    },
    {
      "epoch": 0.6478067865472876,
      "grad_norm": 35.82595492510001,
      "learning_rate": 1.6631775919383398e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.640625,
      "logps/chosen": -316.0,
      "logps/rejected": -752.0,
      "loss": 0.3914,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8125,
      "rewards/margins": 4.53125,
      "rewards/rejected": -6.34375,
      "step": 8610
    },
    {
      "epoch": 0.6485591753818374,
      "grad_norm": 21.211202215766885,
      "learning_rate": 1.656992942364585e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.6875,
      "logps/chosen": -332.0,
      "logps/rejected": -700.0,
      "loss": 0.3569,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.109375,
      "rewards/margins": 3.515625,
      "rewards/rejected": -5.625,
      "step": 8620
    },
    {
      "epoch": 0.649311564216387,
      "grad_norm": 21.63563987967729,
      "learning_rate": 1.6508141084068682e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.71875,
      "logps/chosen": -296.0,
      "logps/rejected": -664.0,
      "loss": 0.3649,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.828125,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.59375,
      "step": 8630
    },
    {
      "epoch": 0.6500639530509367,
      "grad_norm": 21.70232391298792,
      "learning_rate": 1.6446411326908415e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.6875,
      "logps/chosen": -328.0,
      "logps/rejected": -716.0,
      "loss": 0.3714,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.078125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -6.15625,
      "step": 8640
    },
    {
      "epoch": 0.6508163418854864,
      "grad_norm": 21.255007771107543,
      "learning_rate": 1.6384740578017419e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.828125,
      "logps/chosen": -324.0,
      "logps/rejected": -732.0,
      "loss": 0.3988,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.0625,
      "rewards/margins": 4.09375,
      "rewards/rejected": -6.15625,
      "step": 8650
    },
    {
      "epoch": 0.6515687307200361,
      "grad_norm": 24.671023990722226,
      "learning_rate": 1.6323129262841016e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.875,
      "logps/chosen": -290.0,
      "logps/rejected": -660.0,
      "loss": 0.3231,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8125,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.59375,
      "step": 8660
    },
    {
      "epoch": 0.6523211195545858,
      "grad_norm": 30.604726285235337,
      "learning_rate": 1.626157780641449e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.78125,
      "logps/chosen": -322.0,
      "logps/rejected": -668.0,
      "loss": 0.3884,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.625,
      "step": 8670
    },
    {
      "epoch": 0.6530735083891355,
      "grad_norm": 18.751932731472788,
      "learning_rate": 1.6200086633360203e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.640625,
      "logps/chosen": -302.0,
      "logps/rejected": -720.0,
      "loss": 0.3202,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.84375,
      "rewards/margins": 4.25,
      "rewards/rejected": -6.09375,
      "step": 8680
    },
    {
      "epoch": 0.6538258972236852,
      "grad_norm": 25.517110088738686,
      "learning_rate": 1.6138656167884615e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.71875,
      "logps/chosen": -306.0,
      "logps/rejected": -672.0,
      "loss": 0.4517,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.0,
      "rewards/margins": 3.734375,
      "rewards/rejected": -5.75,
      "step": 8690
    },
    {
      "epoch": 0.6545782860582349,
      "grad_norm": 29.4564031658717,
      "learning_rate": 1.6077286833775407e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.6875,
      "logps/chosen": -298.0,
      "logps/rejected": -696.0,
      "loss": 0.3512,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.890625,
      "rewards/rejected": -5.75,
      "step": 8700
    },
    {
      "epoch": 0.6553306748927846,
      "grad_norm": 25.107872331077854,
      "learning_rate": 1.6015979054398537e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.8125,
      "logps/chosen": -308.0,
      "logps/rejected": -648.0,
      "loss": 0.3925,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8984375,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.46875,
      "step": 8710
    },
    {
      "epoch": 0.6560830637273343,
      "grad_norm": 15.758395780116013,
      "learning_rate": 1.5954733252695297e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.65625,
      "logps/chosen": -276.0,
      "logps/rejected": -660.0,
      "loss": 0.3918,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 3.875,
      "rewards/rejected": -5.53125,
      "step": 8720
    },
    {
      "epoch": 0.656835452561884,
      "grad_norm": 16.950760410596533,
      "learning_rate": 1.5893549851179443e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -312.0,
      "logps/rejected": -716.0,
      "loss": 0.3542,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 4.0,
      "rewards/rejected": -6.0,
      "step": 8730
    },
    {
      "epoch": 0.6575878413964337,
      "grad_norm": 23.060851227654794,
      "learning_rate": 1.5832429271934216e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.625,
      "logps/chosen": -332.0,
      "logps/rejected": -668.0,
      "loss": 0.3771,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.09375,
      "rewards/margins": 3.46875,
      "rewards/rejected": -5.5625,
      "step": 8740
    },
    {
      "epoch": 0.6583402302309834,
      "grad_norm": 26.28762865856987,
      "learning_rate": 1.5771371936609512e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.875,
      "logps/chosen": -344.0,
      "logps/rejected": -692.0,
      "loss": 0.4456,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.25,
      "rewards/margins": 3.65625,
      "rewards/rejected": -5.90625,
      "step": 8750
    },
    {
      "epoch": 0.6590926190655331,
      "grad_norm": 27.445287358325672,
      "learning_rate": 1.5710378266418909e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -326.0,
      "logps/rejected": -748.0,
      "loss": 0.4074,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.125,
      "rewards/margins": 4.1875,
      "rewards/rejected": -6.3125,
      "step": 8760
    },
    {
      "epoch": 0.6598450079000827,
      "grad_norm": 21.208481222067736,
      "learning_rate": 1.5649448682136768e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.640625,
      "logps/chosen": -284.0,
      "logps/rejected": -640.0,
      "loss": 0.3724,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.703125,
      "rewards/margins": 3.625,
      "rewards/rejected": -5.3125,
      "step": 8770
    },
    {
      "epoch": 0.6605973967346325,
      "grad_norm": 26.74592928087496,
      "learning_rate": 1.558858360409536e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.640625,
      "logps/chosen": -264.0,
      "logps/rejected": -636.0,
      "loss": 0.3831,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.59375,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.15625,
      "step": 8780
    },
    {
      "epoch": 0.6613497855691821,
      "grad_norm": 23.673710540242688,
      "learning_rate": 1.5527783452181947e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.75,
      "logps/chosen": -332.0,
      "logps/rejected": -688.0,
      "loss": 0.3999,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.71875,
      "step": 8790
    },
    {
      "epoch": 0.6621021744037319,
      "grad_norm": 35.61870768814628,
      "learning_rate": 1.5467048645835895e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.703125,
      "logps/chosen": -314.0,
      "logps/rejected": -672.0,
      "loss": 0.3446,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.53125,
      "step": 8800
    },
    {
      "epoch": 0.6628545632382815,
      "grad_norm": 19.718248830361382,
      "learning_rate": 1.540637960404575e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.765625,
      "logps/chosen": -302.0,
      "logps/rejected": -648.0,
      "loss": 0.3852,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.828125,
      "rewards/margins": 3.484375,
      "rewards/rejected": -5.3125,
      "step": 8810
    },
    {
      "epoch": 0.6636069520728313,
      "grad_norm": 27.6061896163203,
      "learning_rate": 1.53457767453464e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.84375,
      "logps/chosen": -310.0,
      "logps/rejected": -648.0,
      "loss": 0.4134,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.515625,
      "rewards/rejected": -5.40625,
      "step": 8820
    },
    {
      "epoch": 0.6643593409073809,
      "grad_norm": 18.414129244618223,
      "learning_rate": 1.528524048781613e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.875,
      "logps/chosen": -268.0,
      "logps/rejected": -656.0,
      "loss": 0.3627,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.609375,
      "rewards/margins": 3.890625,
      "rewards/rejected": -5.5,
      "step": 8830
    },
    {
      "epoch": 0.6651117297419307,
      "grad_norm": 18.92167742573557,
      "learning_rate": 1.5224771249073787e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.6875,
      "logps/chosen": -312.0,
      "logps/rejected": -656.0,
      "loss": 0.3763,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.359375,
      "rewards/rejected": -5.28125,
      "step": 8840
    },
    {
      "epoch": 0.6658641185764803,
      "grad_norm": 23.810663171982615,
      "learning_rate": 1.5164369446275878e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -3.0,
      "logps/chosen": -322.0,
      "logps/rejected": -656.0,
      "loss": 0.3702,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 3.578125,
      "rewards/rejected": -5.53125,
      "step": 8850
    },
    {
      "epoch": 0.6666165074110301,
      "grad_norm": 21.310921614950153,
      "learning_rate": 1.5104035496113672e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.9375,
      "logps/chosen": -318.0,
      "logps/rejected": -664.0,
      "loss": 0.3558,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.53125,
      "step": 8860
    },
    {
      "epoch": 0.6673688962455797,
      "grad_norm": 21.70227454327363,
      "learning_rate": 1.5043769814810374e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.46875,
      "logps/chosen": -312.0,
      "logps/rejected": -724.0,
      "loss": 0.396,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -5.9375,
      "step": 8870
    },
    {
      "epoch": 0.6681212850801294,
      "grad_norm": 28.309630620225708,
      "learning_rate": 1.4983572818118213e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.390625,
      "logps/chosen": -304.0,
      "logps/rejected": -680.0,
      "loss": 0.403,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.78125,
      "rewards/margins": 3.625,
      "rewards/rejected": -5.40625,
      "step": 8880
    },
    {
      "epoch": 0.6688736739146791,
      "grad_norm": 26.62891766114843,
      "learning_rate": 1.4923444921315575e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.578125,
      "logps/chosen": -292.0,
      "logps/rejected": -696.0,
      "loss": 0.3865,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6875,
      "rewards/margins": 3.921875,
      "rewards/rejected": -5.625,
      "step": 8890
    },
    {
      "epoch": 0.6696260627492288,
      "grad_norm": 19.68921830279629,
      "learning_rate": 1.4863386539204167e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.859375,
      "logps/chosen": -322.0,
      "logps/rejected": -648.0,
      "loss": 0.3895,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.40625,
      "rewards/rejected": -5.28125,
      "step": 8900
    },
    {
      "epoch": 0.6703784515837785,
      "grad_norm": 23.45571492387625,
      "learning_rate": 1.480339808610614e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.71875,
      "logps/chosen": -282.0,
      "logps/rejected": -684.0,
      "loss": 0.3979,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 3.96875,
      "rewards/rejected": -5.59375,
      "step": 8910
    },
    {
      "epoch": 0.6711308404183282,
      "grad_norm": 25.574470972450403,
      "learning_rate": 1.47434799758612e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.671875,
      "logps/chosen": -320.0,
      "logps/rejected": -624.0,
      "loss": 0.4038,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.921875,
      "rewards/margins": 3.25,
      "rewards/rejected": -5.1875,
      "step": 8920
    },
    {
      "epoch": 0.6718832292528779,
      "grad_norm": 23.071986474003666,
      "learning_rate": 1.4683632621823822e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.5625,
      "logps/chosen": -284.0,
      "logps/rejected": -596.0,
      "loss": 0.4059,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 3.40625,
      "rewards/rejected": -4.9375,
      "step": 8930
    },
    {
      "epoch": 0.6726356180874276,
      "grad_norm": 29.32138276972959,
      "learning_rate": 1.4623856436860322e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.609375,
      "logps/chosen": -276.0,
      "logps/rejected": -688.0,
      "loss": 0.4236,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.59375,
      "rewards/margins": 3.953125,
      "rewards/rejected": -5.5625,
      "step": 8940
    },
    {
      "epoch": 0.6733880069219773,
      "grad_norm": 24.23920752840604,
      "learning_rate": 1.4564151833346072e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.75,
      "logps/chosen": -318.0,
      "logps/rejected": -676.0,
      "loss": 0.3968,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.53125,
      "step": 8950
    },
    {
      "epoch": 0.674140395756527,
      "grad_norm": 20.517844572848677,
      "learning_rate": 1.4504519223162602e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.796875,
      "logps/chosen": -324.0,
      "logps/rejected": -660.0,
      "loss": 0.3805,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.0,
      "rewards/margins": 3.40625,
      "rewards/rejected": -5.40625,
      "step": 8960
    },
    {
      "epoch": 0.6748927845910767,
      "grad_norm": 22.139176371462284,
      "learning_rate": 1.4444959017694827e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.953125,
      "logps/chosen": -316.0,
      "logps/rejected": -664.0,
      "loss": 0.3837,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.890625,
      "rewards/margins": 3.625,
      "rewards/rejected": -5.5,
      "step": 8970
    },
    {
      "epoch": 0.6756451734256264,
      "grad_norm": 17.302102101120177,
      "learning_rate": 1.4385471627828112e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.78125,
      "logps/chosen": -326.0,
      "logps/rejected": -712.0,
      "loss": 0.3473,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.84375,
      "step": 8980
    },
    {
      "epoch": 0.6763975622601761,
      "grad_norm": 24.78076548505025,
      "learning_rate": 1.4326057463945544e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.734375,
      "logps/chosen": -292.0,
      "logps/rejected": -692.0,
      "loss": 0.3571,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.625,
      "step": 8990
    },
    {
      "epoch": 0.6771499510947258,
      "grad_norm": 24.197053367779475,
      "learning_rate": 1.4266716935925025e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.8125,
      "logps/chosen": -308.0,
      "logps/rejected": -660.0,
      "loss": 0.3365,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.921875,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.6875,
      "step": 9000
    },
    {
      "epoch": 0.6771499510947258,
      "eval_logits/chosen": -2.65625,
      "eval_logits/rejected": -2.75,
      "eval_logps/chosen": -342.0,
      "eval_logps/rejected": -676.0,
      "eval_loss": 0.26604801416397095,
      "eval_rewards/accuracies": 0.8821089863777161,
      "eval_rewards/chosen": -2.09375,
      "eval_rewards/margins": 3.46875,
      "eval_rewards/rejected": -5.5625,
      "eval_runtime": 2669.1882,
      "eval_samples_per_second": 35.401,
      "eval_steps_per_second": 0.553,
      "step": 9000
    },
    {
      "epoch": 0.6779023399292754,
      "grad_norm": 20.07545214927782,
      "learning_rate": 1.4207450453136434e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.84375,
      "logps/chosen": -324.0,
      "logps/rejected": -704.0,
      "loss": 0.3941,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.953125,
      "rewards/margins": 3.921875,
      "rewards/rejected": -5.875,
      "step": 9010
    },
    {
      "epoch": 0.6786547287638252,
      "grad_norm": 25.55892343325029,
      "learning_rate": 1.4148258424438912e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.84375,
      "logps/chosen": -340.0,
      "logps/rejected": -704.0,
      "loss": 0.3759,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.078125,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.8125,
      "step": 9020
    },
    {
      "epoch": 0.6794071175983748,
      "grad_norm": 25.777241209749498,
      "learning_rate": 1.4089141258177894e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.90625,
      "logps/chosen": -276.0,
      "logps/rejected": -612.0,
      "loss": 0.36,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 3.515625,
      "rewards/rejected": -5.0625,
      "step": 9030
    },
    {
      "epoch": 0.6801595064329246,
      "grad_norm": 31.885483041346564,
      "learning_rate": 1.4030099362182429e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.84375,
      "logps/chosen": -316.0,
      "logps/rejected": -716.0,
      "loss": 0.3398,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.90625,
      "rewards/margins": 4.09375,
      "rewards/rejected": -6.0,
      "step": 9040
    },
    {
      "epoch": 0.6809118952674742,
      "grad_norm": 32.10553443600947,
      "learning_rate": 1.3971133143762255e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.671875,
      "logps/chosen": -290.0,
      "logps/rejected": -696.0,
      "loss": 0.3949,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 4.0,
      "rewards/rejected": -5.75,
      "step": 9050
    },
    {
      "epoch": 0.681664284102024,
      "grad_norm": 20.33996497776697,
      "learning_rate": 1.3912243009705043e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.75,
      "logps/chosen": -334.0,
      "logps/rejected": -652.0,
      "loss": 0.3983,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0625,
      "rewards/margins": 3.28125,
      "rewards/rejected": -5.34375,
      "step": 9060
    },
    {
      "epoch": 0.6824166729365736,
      "grad_norm": 18.539490218309464,
      "learning_rate": 1.3853429366273617e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -300.0,
      "logps/rejected": -684.0,
      "loss": 0.3995,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.5625,
      "step": 9070
    },
    {
      "epoch": 0.6831690617711234,
      "grad_norm": 23.74480759422917,
      "learning_rate": 1.379469261920308e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.484375,
      "logps/chosen": -318.0,
      "logps/rejected": -720.0,
      "loss": 0.35,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.84375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -6.0625,
      "step": 9080
    },
    {
      "epoch": 0.683921450605673,
      "grad_norm": 19.937603186964168,
      "learning_rate": 1.373603317369807e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.578125,
      "logps/chosen": -326.0,
      "logps/rejected": -740.0,
      "loss": 0.3777,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.015625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -6.1875,
      "step": 9090
    },
    {
      "epoch": 0.6846738394402228,
      "grad_norm": 22.19672664486699,
      "learning_rate": 1.3677451434429945e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.875,
      "logps/chosen": -274.0,
      "logps/rejected": -676.0,
      "loss": 0.3608,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.625,
      "rewards/margins": 4.125,
      "rewards/rejected": -5.75,
      "step": 9100
    },
    {
      "epoch": 0.6854262282747724,
      "grad_norm": 16.260838212130693,
      "learning_rate": 1.3618947805533993e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.609375,
      "logps/chosen": -310.0,
      "logps/rejected": -656.0,
      "loss": 0.3509,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.59375,
      "rewards/rejected": -5.46875,
      "step": 9110
    },
    {
      "epoch": 0.6861786171093222,
      "grad_norm": 16.487250161705905,
      "learning_rate": 1.3560522690606657e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -306.0,
      "logps/rejected": -688.0,
      "loss": 0.36,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.78125,
      "step": 9120
    },
    {
      "epoch": 0.6869310059438718,
      "grad_norm": 21.575790095048145,
      "learning_rate": 1.3502176492702732e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.609375,
      "logps/chosen": -320.0,
      "logps/rejected": -708.0,
      "loss": 0.3725,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.828125,
      "rewards/margins": 4.03125,
      "rewards/rejected": -5.875,
      "step": 9130
    },
    {
      "epoch": 0.6876833947784214,
      "grad_norm": 22.896301413837932,
      "learning_rate": 1.344390961433257e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.640625,
      "logps/chosen": -304.0,
      "logps/rejected": -688.0,
      "loss": 0.4015,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.953125,
      "rewards/rejected": -5.75,
      "step": 9140
    },
    {
      "epoch": 0.6884357836129712,
      "grad_norm": 18.08220095251193,
      "learning_rate": 1.338572245745937e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.71875,
      "logps/chosen": -262.0,
      "logps/rejected": -636.0,
      "loss": 0.3724,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.5625,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.3125,
      "step": 9150
    },
    {
      "epoch": 0.6891881724475208,
      "grad_norm": 25.729215206737326,
      "learning_rate": 1.3327615423496324e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.703125,
      "logps/chosen": -300.0,
      "logps/rejected": -672.0,
      "loss": 0.3639,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.671875,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.59375,
      "step": 9160
    },
    {
      "epoch": 0.6899405612820706,
      "grad_norm": 20.89756671444677,
      "learning_rate": 1.326958891330388e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -310.0,
      "logps/rejected": -708.0,
      "loss": 0.3987,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.8984375,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.71875,
      "step": 9170
    },
    {
      "epoch": 0.6906929501166202,
      "grad_norm": 30.55696467974752,
      "learning_rate": 1.3211643327187028e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.609375,
      "logps/chosen": -316.0,
      "logps/rejected": -744.0,
      "loss": 0.3676,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.0,
      "rewards/margins": 4.28125,
      "rewards/rejected": -6.28125,
      "step": 9180
    },
    {
      "epoch": 0.69144533895117,
      "grad_norm": 25.387262211686597,
      "learning_rate": 1.3153779064892417e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.84375,
      "logps/chosen": -328.0,
      "logps/rejected": -708.0,
      "loss": 0.3979,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.96875,
      "rewards/rejected": -5.90625,
      "step": 9190
    },
    {
      "epoch": 0.6921977277857196,
      "grad_norm": 32.03690152001648,
      "learning_rate": 1.309599652560574e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.859375,
      "logps/chosen": -288.0,
      "logps/rejected": -700.0,
      "loss": 0.358,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.828125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -5.90625,
      "step": 9200
    },
    {
      "epoch": 0.6929501166202694,
      "grad_norm": 20.857244103050956,
      "learning_rate": 1.3038296107948872e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.640625,
      "logps/chosen": -316.0,
      "logps/rejected": -700.0,
      "loss": 0.3754,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 3.828125,
      "rewards/rejected": -5.65625,
      "step": 9210
    },
    {
      "epoch": 0.693702505454819,
      "grad_norm": 28.185869341511378,
      "learning_rate": 1.2980678209977158e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.625,
      "logps/chosen": -314.0,
      "logps/rejected": -704.0,
      "loss": 0.3715,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.984375,
      "rewards/margins": 3.875,
      "rewards/rejected": -5.84375,
      "step": 9220
    },
    {
      "epoch": 0.6944548942893688,
      "grad_norm": 22.841464674403674,
      "learning_rate": 1.2923143229176696e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.71875,
      "logps/chosen": -338.0,
      "logps/rejected": -696.0,
      "loss": 0.4099,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.84375,
      "step": 9230
    },
    {
      "epoch": 0.6952072831239184,
      "grad_norm": 17.2392259664375,
      "learning_rate": 1.286569156246154e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.6875,
      "logps/chosen": -318.0,
      "logps/rejected": -708.0,
      "loss": 0.3296,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.0,
      "rewards/margins": 3.9375,
      "rewards/rejected": -5.9375,
      "step": 9240
    },
    {
      "epoch": 0.6959596719584681,
      "grad_norm": 27.92265476327664,
      "learning_rate": 1.2808323606171006e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.875,
      "logps/chosen": -332.0,
      "logps/rejected": -672.0,
      "loss": 0.3772,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.078125,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.75,
      "step": 9250
    },
    {
      "epoch": 0.6967120607930178,
      "grad_norm": 19.91869874137521,
      "learning_rate": 1.2751039756066907e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.484375,
      "logps/chosen": -312.0,
      "logps/rejected": -684.0,
      "loss": 0.3646,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.765625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.5625,
      "step": 9260
    },
    {
      "epoch": 0.6974644496275675,
      "grad_norm": 31.747110935652533,
      "learning_rate": 1.2693840407330838e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.828125,
      "logps/chosen": -292.0,
      "logps/rejected": -644.0,
      "loss": 0.3895,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.7734375,
      "rewards/margins": 3.578125,
      "rewards/rejected": -5.34375,
      "step": 9270
    },
    {
      "epoch": 0.6982168384621172,
      "grad_norm": 18.762766407800843,
      "learning_rate": 1.2636725954561479e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.59375,
      "logps/chosen": -314.0,
      "logps/rejected": -772.0,
      "loss": 0.386,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.75,
      "rewards/margins": 4.6875,
      "rewards/rejected": -6.46875,
      "step": 9280
    },
    {
      "epoch": 0.6989692272966669,
      "grad_norm": 25.14967055848488,
      "learning_rate": 1.257969679177177e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.59375,
      "logps/chosen": -296.0,
      "logps/rejected": -708.0,
      "loss": 0.4017,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.7578125,
      "rewards/margins": 4.0,
      "rewards/rejected": -5.75,
      "step": 9290
    },
    {
      "epoch": 0.6997216161312166,
      "grad_norm": 47.57908054351943,
      "learning_rate": 1.2522753312386347e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.609375,
      "logps/chosen": -306.0,
      "logps/rejected": -696.0,
      "loss": 0.3688,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 3.96875,
      "rewards/rejected": -5.8125,
      "step": 9300
    },
    {
      "epoch": 0.7004740049657663,
      "grad_norm": 21.553178832091938,
      "learning_rate": 1.2465895909238698e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.640625,
      "logps/chosen": -328.0,
      "logps/rejected": -660.0,
      "loss": 0.3834,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -5.4375,
      "step": 9310
    },
    {
      "epoch": 0.701226393800316,
      "grad_norm": 22.992236206007625,
      "learning_rate": 1.24091249745685e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.765625,
      "logps/chosen": -296.0,
      "logps/rejected": -660.0,
      "loss": 0.3745,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 3.71875,
      "rewards/rejected": -5.40625,
      "step": 9320
    },
    {
      "epoch": 0.7019787826348657,
      "grad_norm": 18.240661999516963,
      "learning_rate": 1.2352440900018943e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -282.0,
      "logps/rejected": -684.0,
      "loss": 0.3624,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.640625,
      "rewards/margins": 4.0,
      "rewards/rejected": -5.625,
      "step": 9330
    },
    {
      "epoch": 0.7027311714694154,
      "grad_norm": 33.103792804007796,
      "learning_rate": 1.2295844076633973e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.828125,
      "logps/chosen": -300.0,
      "logps/rejected": -676.0,
      "loss": 0.4099,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -5.6875,
      "step": 9340
    },
    {
      "epoch": 0.7034835603039651,
      "grad_norm": 32.73335817528822,
      "learning_rate": 1.2239334894855622e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.578125,
      "logps/chosen": -282.0,
      "logps/rejected": -668.0,
      "loss": 0.3947,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -5.5625,
      "step": 9350
    },
    {
      "epoch": 0.7042359491385148,
      "grad_norm": 18.689778081052264,
      "learning_rate": 1.2182913744521332e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.671875,
      "logps/chosen": -308.0,
      "logps/rejected": -652.0,
      "loss": 0.3914,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.4375,
      "step": 9360
    },
    {
      "epoch": 0.7049883379730645,
      "grad_norm": 29.218005126822653,
      "learning_rate": 1.2126581014861227e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.78125,
      "logps/chosen": -326.0,
      "logps/rejected": -676.0,
      "loss": 0.3884,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.890625,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.5625,
      "step": 9370
    },
    {
      "epoch": 0.7057407268076141,
      "grad_norm": 17.84852599135083,
      "learning_rate": 1.207033709449545e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.8125,
      "logps/chosen": -326.0,
      "logps/rejected": -684.0,
      "loss": 0.396,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.109375,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.875,
      "step": 9380
    },
    {
      "epoch": 0.7064931156421639,
      "grad_norm": 15.574705464411817,
      "learning_rate": 1.2014182371431486e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.671875,
      "logps/chosen": -298.0,
      "logps/rejected": -668.0,
      "loss": 0.3563,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.890625,
      "rewards/rejected": -5.625,
      "step": 9390
    },
    {
      "epoch": 0.7072455044767135,
      "grad_norm": 19.335379541164613,
      "learning_rate": 1.1958117233061465e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.515625,
      "logps/chosen": -304.0,
      "logps/rejected": -700.0,
      "loss": 0.3393,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 4.09375,
      "rewards/rejected": -5.875,
      "step": 9400
    },
    {
      "epoch": 0.7079978933112633,
      "grad_norm": 21.311785423171308,
      "learning_rate": 1.1902142066159529e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.734375,
      "logps/chosen": -302.0,
      "logps/rejected": -712.0,
      "loss": 0.3759,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -5.78125,
      "step": 9410
    },
    {
      "epoch": 0.7087502821458129,
      "grad_norm": 19.99025991153511,
      "learning_rate": 1.1846257256879116e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.6875,
      "logps/chosen": -310.0,
      "logps/rejected": -668.0,
      "loss": 0.3991,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.828125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.53125,
      "step": 9420
    },
    {
      "epoch": 0.7095026709803627,
      "grad_norm": 23.765325059157814,
      "learning_rate": 1.1790463190750313e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.78125,
      "logps/chosen": -328.0,
      "logps/rejected": -688.0,
      "loss": 0.4124,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.0625,
      "rewards/margins": 3.828125,
      "rewards/rejected": -5.875,
      "step": 9430
    },
    {
      "epoch": 0.7102550598149123,
      "grad_norm": 24.110368381403536,
      "learning_rate": 1.1734760252677228e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.53125,
      "logps/chosen": -304.0,
      "logps/rejected": -656.0,
      "loss": 0.3786,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.375,
      "step": 9440
    },
    {
      "epoch": 0.7110074486494621,
      "grad_norm": 27.93281206006685,
      "learning_rate": 1.1679148826935284e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.609375,
      "logps/chosen": -300.0,
      "logps/rejected": -660.0,
      "loss": 0.3932,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.609375,
      "rewards/rejected": -5.46875,
      "step": 9450
    },
    {
      "epoch": 0.7117598374840117,
      "grad_norm": 36.492765040243086,
      "learning_rate": 1.1623629297168599e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -288.0,
      "logps/rejected": -672.0,
      "loss": 0.3927,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.765625,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.5625,
      "step": 9460
    },
    {
      "epoch": 0.7125122263185615,
      "grad_norm": 30.397705137711025,
      "learning_rate": 1.1568202046387334e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.625,
      "logps/chosen": -332.0,
      "logps/rejected": -652.0,
      "loss": 0.3914,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.109375,
      "rewards/margins": 3.140625,
      "rewards/rejected": -5.25,
      "step": 9470
    },
    {
      "epoch": 0.7132646151531111,
      "grad_norm": 20.743568116497258,
      "learning_rate": 1.1512867456965036e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5625,
      "logps/chosen": -300.0,
      "logps/rejected": -656.0,
      "loss": 0.3642,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.40625,
      "step": 9480
    },
    {
      "epoch": 0.7140170039876608,
      "grad_norm": 23.20419102162868,
      "learning_rate": 1.1457625910636042e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.40625,
      "logps/chosen": -294.0,
      "logps/rejected": -664.0,
      "loss": 0.3729,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.765625,
      "rewards/margins": 3.734375,
      "rewards/rejected": -5.5,
      "step": 9490
    },
    {
      "epoch": 0.7147693928222105,
      "grad_norm": 27.569993080970157,
      "learning_rate": 1.1402477788492796e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -288.0,
      "logps/rejected": -716.0,
      "loss": 0.4272,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 4.125,
      "rewards/rejected": -5.84375,
      "step": 9500
    },
    {
      "epoch": 0.7155217816567602,
      "grad_norm": 14.902694259374737,
      "learning_rate": 1.134742347098323e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.78125,
      "logps/chosen": -256.0,
      "logps/rejected": -636.0,
      "loss": 0.3489,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.4609375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.375,
      "step": 9510
    },
    {
      "epoch": 0.7162741704913099,
      "grad_norm": 31.4096946628576,
      "learning_rate": 1.1292463337908185e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.625,
      "logps/chosen": -304.0,
      "logps/rejected": -648.0,
      "loss": 0.3873,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.828125,
      "rewards/margins": 3.578125,
      "rewards/rejected": -5.40625,
      "step": 9520
    },
    {
      "epoch": 0.7170265593258596,
      "grad_norm": 15.26256921268548,
      "learning_rate": 1.1237597768418714e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.53125,
      "logps/chosen": -308.0,
      "logps/rejected": -660.0,
      "loss": 0.3509,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.6875,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.46875,
      "step": 9530
    },
    {
      "epoch": 0.7177789481604093,
      "grad_norm": 28.86009201966912,
      "learning_rate": 1.1182827141013549e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.640625,
      "logps/chosen": -294.0,
      "logps/rejected": -616.0,
      "loss": 0.3956,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.765625,
      "rewards/margins": 3.390625,
      "rewards/rejected": -5.15625,
      "step": 9540
    },
    {
      "epoch": 0.718531336994959,
      "grad_norm": 22.430699441565146,
      "learning_rate": 1.112815183353642e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.6875,
      "logps/chosen": -314.0,
      "logps/rejected": -676.0,
      "loss": 0.3725,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.59375,
      "step": 9550
    },
    {
      "epoch": 0.7192837258295087,
      "grad_norm": 18.928003259375966,
      "learning_rate": 1.1073572223173489e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.46875,
      "logps/chosen": -302.0,
      "logps/rejected": -640.0,
      "loss": 0.335,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.7734375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -5.1875,
      "step": 9560
    },
    {
      "epoch": 0.7200361146640584,
      "grad_norm": 20.18737571436296,
      "learning_rate": 1.1019088686450731e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.796875,
      "logps/chosen": -314.0,
      "logps/rejected": -632.0,
      "loss": 0.3752,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.28125,
      "rewards/rejected": -5.15625,
      "step": 9570
    },
    {
      "epoch": 0.7207885034986081,
      "grad_norm": 21.010493676595203,
      "learning_rate": 1.0964701599231341e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.828125,
      "logps/chosen": -294.0,
      "logps/rejected": -704.0,
      "loss": 0.3167,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.734375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -5.96875,
      "step": 9580
    },
    {
      "epoch": 0.7215408923331578,
      "grad_norm": 24.27926172478564,
      "learning_rate": 1.091041133671316e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.828125,
      "logps/chosen": -340.0,
      "logps/rejected": -696.0,
      "loss": 0.3614,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.25,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.9375,
      "step": 9590
    },
    {
      "epoch": 0.7222932811677075,
      "grad_norm": 25.48476361675014,
      "learning_rate": 1.0856218273426049e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.6875,
      "logps/chosen": -312.0,
      "logps/rejected": -680.0,
      "loss": 0.3789,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.84375,
      "step": 9600
    },
    {
      "epoch": 0.7230456700022572,
      "grad_norm": 23.333107682599916,
      "learning_rate": 1.0802122783229323e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5625,
      "logps/chosen": -310.0,
      "logps/rejected": -692.0,
      "loss": 0.3272,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.875,
      "step": 9610
    },
    {
      "epoch": 0.7237980588368068,
      "grad_norm": 32.23670889896565,
      "learning_rate": 1.0748125239309197e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.46875,
      "logps/chosen": -336.0,
      "logps/rejected": -776.0,
      "loss": 0.3557,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.0625,
      "rewards/margins": 4.5625,
      "rewards/rejected": -6.625,
      "step": 9620
    },
    {
      "epoch": 0.7245504476713566,
      "grad_norm": 20.22555244004564,
      "learning_rate": 1.0694226014176167e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.453125,
      "logps/chosen": -312.0,
      "logps/rejected": -680.0,
      "loss": 0.3425,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.984375,
      "rewards/margins": 3.734375,
      "rewards/rejected": -5.71875,
      "step": 9630
    },
    {
      "epoch": 0.7253028365059062,
      "grad_norm": 23.89469819216425,
      "learning_rate": 1.0640425479662465e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.578125,
      "logps/chosen": -292.0,
      "logps/rejected": -720.0,
      "loss": 0.372,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -5.96875,
      "step": 9640
    },
    {
      "epoch": 0.726055225340456,
      "grad_norm": 26.893058300107235,
      "learning_rate": 1.0586724006919496e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.71875,
      "logps/chosen": -308.0,
      "logps/rejected": -692.0,
      "loss": 0.383,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -5.71875,
      "step": 9650
    },
    {
      "epoch": 0.7268076141750056,
      "grad_norm": 35.734284717892315,
      "learning_rate": 1.0533121966415257e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.953125,
      "logps/chosen": -302.0,
      "logps/rejected": -696.0,
      "loss": 0.35,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.84375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -5.875,
      "step": 9660
    },
    {
      "epoch": 0.7275600030095554,
      "grad_norm": 23.77326839528694,
      "learning_rate": 1.0479619727931827e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.765625,
      "logps/chosen": -310.0,
      "logps/rejected": -736.0,
      "loss": 0.3575,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 4.25,
      "rewards/rejected": -6.09375,
      "step": 9670
    },
    {
      "epoch": 0.728312391844105,
      "grad_norm": 17.92857758708908,
      "learning_rate": 1.0426217660562758e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.625,
      "logps/chosen": -316.0,
      "logps/rejected": -644.0,
      "loss": 0.3641,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -5.28125,
      "step": 9680
    },
    {
      "epoch": 0.7290647806786548,
      "grad_norm": 18.428978739959568,
      "learning_rate": 1.0372916132710555e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.828125,
      "logps/chosen": -340.0,
      "logps/rejected": -644.0,
      "loss": 0.3376,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.90625,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.40625,
      "step": 9690
    },
    {
      "epoch": 0.7298171695132044,
      "grad_norm": 27.848970039712423,
      "learning_rate": 1.031971551208416e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.828125,
      "logps/chosen": -278.0,
      "logps/rejected": -632.0,
      "loss": 0.3924,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.59375,
      "rewards/margins": 3.65625,
      "rewards/rejected": -5.25,
      "step": 9700
    },
    {
      "epoch": 0.7305695583477542,
      "grad_norm": 19.054747272289493,
      "learning_rate": 1.026661616569637e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.640625,
      "logps/chosen": -318.0,
      "logps/rejected": -672.0,
      "loss": 0.3537,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.0,
      "rewards/margins": 3.546875,
      "rewards/rejected": -5.5625,
      "step": 9710
    },
    {
      "epoch": 0.7313219471823038,
      "grad_norm": 19.98892418013842,
      "learning_rate": 1.0213618459861321e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.546875,
      "logps/chosen": -298.0,
      "logps/rejected": -668.0,
      "loss": 0.3797,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.90625,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.5625,
      "step": 9720
    },
    {
      "epoch": 0.7320743360168535,
      "grad_norm": 22.57232567206007,
      "learning_rate": 1.0160722760192e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.90625,
      "logps/chosen": -304.0,
      "logps/rejected": -664.0,
      "loss": 0.3898,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.859375,
      "rewards/rejected": -5.65625,
      "step": 9730
    },
    {
      "epoch": 0.7328267248514032,
      "grad_norm": 26.66962477821109,
      "learning_rate": 1.010792943159763e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.59375,
      "logps/chosen": -348.0,
      "logps/rejected": -672.0,
      "loss": 0.3961,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.078125,
      "rewards/margins": 3.28125,
      "rewards/rejected": -5.375,
      "step": 9740
    },
    {
      "epoch": 0.7335791136859529,
      "grad_norm": 31.046632624650545,
      "learning_rate": 1.0055238838281275e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -312.0,
      "logps/rejected": -696.0,
      "loss": 0.3551,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 3.65625,
      "rewards/rejected": -5.625,
      "step": 9750
    },
    {
      "epoch": 0.7343315025205026,
      "grad_norm": 28.59188662960998,
      "learning_rate": 1.000265134373722e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.71875,
      "logps/chosen": -318.0,
      "logps/rejected": -632.0,
      "loss": 0.4123,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.28125,
      "rewards/rejected": -5.21875,
      "step": 9760
    },
    {
      "epoch": 0.7350838913550523,
      "grad_norm": 21.729924776111968,
      "learning_rate": 9.950167310748516e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.640625,
      "logps/chosen": -324.0,
      "logps/rejected": -676.0,
      "loss": 0.3961,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.84375,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.5625,
      "step": 9770
    },
    {
      "epoch": 0.735836280189602,
      "grad_norm": 24.959701678354012,
      "learning_rate": 9.897787101384492e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.796875,
      "logps/chosen": -310.0,
      "logps/rejected": -688.0,
      "loss": 0.4328,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.6875,
      "step": 9780
    },
    {
      "epoch": 0.7365886690241517,
      "grad_norm": 20.494353691144823,
      "learning_rate": 9.845511076998195e-08,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.53125,
      "logps/chosen": -284.0,
      "logps/rejected": -624.0,
      "loss": 0.3709,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 3.546875,
      "rewards/rejected": -5.125,
      "step": 9790
    },
    {
      "epoch": 0.7373410578587014,
      "grad_norm": 20.404215391369316,
      "learning_rate": 9.79333959822397e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5,
      "logps/chosen": -288.0,
      "logps/rejected": -728.0,
      "loss": 0.408,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 4.28125,
      "rewards/rejected": -6.03125,
      "step": 9800
    },
    {
      "epoch": 0.738093446693251,
      "grad_norm": 32.062229734667795,
      "learning_rate": 9.741273024974919e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.453125,
      "logps/chosen": -318.0,
      "logps/rejected": -704.0,
      "loss": 0.4041,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.875,
      "rewards/rejected": -5.8125,
      "step": 9810
    },
    {
      "epoch": 0.7388458355278008,
      "grad_norm": 20.127563751678995,
      "learning_rate": 9.68931171644044e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -320.0,
      "logps/rejected": -680.0,
      "loss": 0.4113,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.015625,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.6875,
      "step": 9820
    },
    {
      "epoch": 0.7395982243623505,
      "grad_norm": 29.88958788947678,
      "learning_rate": 9.637456031083746e-08,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.453125,
      "logps/chosen": -310.0,
      "logps/rejected": -676.0,
      "loss": 0.3791,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.46875,
      "step": 9830
    },
    {
      "epoch": 0.7403506131969002,
      "grad_norm": 21.48632801183891,
      "learning_rate": 9.585706326639383e-08,
      "logits/chosen": -2.5,
      "logits/rejected": -2.6875,
      "logps/chosen": -312.0,
      "logps/rejected": -620.0,
      "loss": 0.3734,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.296875,
      "rewards/rejected": -5.09375,
      "step": 9840
    },
    {
      "epoch": 0.7411030020314499,
      "grad_norm": 26.123021875369368,
      "learning_rate": 9.534062960110803e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.515625,
      "logps/chosen": -300.0,
      "logps/rejected": -696.0,
      "loss": 0.3683,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.8125,
      "rewards/margins": 3.953125,
      "rewards/rejected": -5.78125,
      "step": 9850
    },
    {
      "epoch": 0.7418553908659995,
      "grad_norm": 19.66267245558727,
      "learning_rate": 9.482526287767836e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.59375,
      "logps/chosen": -312.0,
      "logps/rejected": -648.0,
      "loss": 0.3869,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 3.46875,
      "rewards/rejected": -5.375,
      "step": 9860
    },
    {
      "epoch": 0.7426077797005493,
      "grad_norm": 18.588702953092287,
      "learning_rate": 9.431096665144268e-08,
      "logits/chosen": -2.5,
      "logits/rejected": -2.5625,
      "logps/chosen": -336.0,
      "logps/rejected": -664.0,
      "loss": 0.3731,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.0,
      "rewards/margins": 3.609375,
      "rewards/rejected": -5.59375,
      "step": 9870
    },
    {
      "epoch": 0.7433601685350989,
      "grad_norm": 20.84162314891795,
      "learning_rate": 9.379774447035408e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.453125,
      "logps/chosen": -314.0,
      "logps/rejected": -728.0,
      "loss": 0.371,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.9375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -6.09375,
      "step": 9880
    },
    {
      "epoch": 0.7441125573696487,
      "grad_norm": 24.661677717406025,
      "learning_rate": 9.328559987495602e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.3125,
      "logps/chosen": -298.0,
      "logps/rejected": -696.0,
      "loss": 0.3984,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.625,
      "step": 9890
    },
    {
      "epoch": 0.7448649462041983,
      "grad_norm": 23.701689701341877,
      "learning_rate": 9.277453639835795e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.90625,
      "logps/chosen": -330.0,
      "logps/rejected": -656.0,
      "loss": 0.3691,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.96875,
      "rewards/margins": 3.375,
      "rewards/rejected": -5.34375,
      "step": 9900
    },
    {
      "epoch": 0.745617335038748,
      "grad_norm": 26.12266138758635,
      "learning_rate": 9.226455756621152e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.8125,
      "logps/chosen": -338.0,
      "logps/rejected": -632.0,
      "loss": 0.3737,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 3.296875,
      "rewards/rejected": -5.28125,
      "step": 9910
    },
    {
      "epoch": 0.7463697238732977,
      "grad_norm": 21.50181176138344,
      "learning_rate": 9.175566689668506e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.515625,
      "logps/chosen": -294.0,
      "logps/rejected": -656.0,
      "loss": 0.417,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.671875,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.4375,
      "step": 9920
    },
    {
      "epoch": 0.7471221127078475,
      "grad_norm": 23.462192247789723,
      "learning_rate": 9.124786790044076e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.828125,
      "logps/chosen": -278.0,
      "logps/rejected": -672.0,
      "loss": 0.3704,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 3.984375,
      "rewards/rejected": -5.59375,
      "step": 9930
    },
    {
      "epoch": 0.7478745015423971,
      "grad_norm": 18.074863270924357,
      "learning_rate": 9.074116408060931e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.6875,
      "logps/chosen": -320.0,
      "logps/rejected": -664.0,
      "loss": 0.358,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 3.59375,
      "rewards/rejected": -5.53125,
      "step": 9940
    },
    {
      "epoch": 0.7486268903769469,
      "grad_norm": 32.45944041086256,
      "learning_rate": 9.023555893276613e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.65625,
      "logps/chosen": -316.0,
      "logps/rejected": -688.0,
      "loss": 0.3769,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.3125,
      "step": 9950
    },
    {
      "epoch": 0.7493792792114965,
      "grad_norm": 24.820545725838866,
      "learning_rate": 8.973105594490766e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.484375,
      "logps/chosen": -308.0,
      "logps/rejected": -648.0,
      "loss": 0.368,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 3.53125,
      "rewards/rejected": -5.40625,
      "step": 9960
    },
    {
      "epoch": 0.7501316680460463,
      "grad_norm": 21.002597369920526,
      "learning_rate": 8.922765859742654e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.75,
      "logps/chosen": -298.0,
      "logps/rejected": -652.0,
      "loss": 0.3536,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.765625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.46875,
      "step": 9970
    },
    {
      "epoch": 0.7508840568805959,
      "grad_norm": 22.143616418296038,
      "learning_rate": 8.872537036308802e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.515625,
      "logps/chosen": -350.0,
      "logps/rejected": -728.0,
      "loss": 0.3197,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.078125,
      "rewards/margins": 3.921875,
      "rewards/rejected": -6.0,
      "step": 9980
    },
    {
      "epoch": 0.7516364457151455,
      "grad_norm": 27.324584897774958,
      "learning_rate": 8.822419470700624e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.734375,
      "logps/chosen": -324.0,
      "logps/rejected": -712.0,
      "loss": 0.3945,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.0625,
      "rewards/margins": 3.890625,
      "rewards/rejected": -5.9375,
      "step": 9990
    },
    {
      "epoch": 0.7523888345496953,
      "grad_norm": 22.086904255779835,
      "learning_rate": 8.772413508661972e-08,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.5625,
      "logps/chosen": -328.0,
      "logps/rejected": -700.0,
      "loss": 0.3505,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.0,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.6875,
      "step": 10000
    },
    {
      "epoch": 0.7523888345496953,
      "eval_logits/chosen": -2.59375,
      "eval_logits/rejected": -2.625,
      "eval_logps/chosen": -338.0,
      "eval_logps/rejected": -664.0,
      "eval_loss": 0.2634149193763733,
      "eval_rewards/accuracies": 0.8849018216133118,
      "eval_rewards/chosen": -2.0625,
      "eval_rewards/margins": 3.390625,
      "eval_rewards/rejected": -5.4375,
      "eval_runtime": 2669.0408,
      "eval_samples_per_second": 35.403,
      "eval_steps_per_second": 0.553,
      "step": 10000
    },
    {
      "epoch": 0.7531412233842449,
      "grad_norm": 23.321052051190726,
      "learning_rate": 8.722519495166799e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.703125,
      "logps/chosen": -306.0,
      "logps/rejected": -656.0,
      "loss": 0.3451,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.71875,
      "rewards/rejected": -5.625,
      "step": 10010
    },
    {
      "epoch": 0.7538936122187947,
      "grad_norm": 28.71745294432285,
      "learning_rate": 8.672737774416761e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.515625,
      "logps/chosen": -344.0,
      "logps/rejected": -752.0,
      "loss": 0.3657,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.171875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -6.28125,
      "step": 10020
    },
    {
      "epoch": 0.7546460010533443,
      "grad_norm": 26.937324491182206,
      "learning_rate": 8.623068689838836e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.65625,
      "logps/chosen": -340.0,
      "logps/rejected": -732.0,
      "loss": 0.3817,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.125,
      "rewards/margins": 3.9375,
      "rewards/rejected": -6.0625,
      "step": 10030
    },
    {
      "epoch": 0.7553983898878941,
      "grad_norm": 28.513356612058693,
      "learning_rate": 8.57351258408299e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.890625,
      "logps/chosen": -316.0,
      "logps/rejected": -736.0,
      "loss": 0.3626,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 4.21875,
      "rewards/rejected": -6.09375,
      "step": 10040
    },
    {
      "epoch": 0.7561507787224437,
      "grad_norm": 23.199335641734805,
      "learning_rate": 8.52406979901975e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.65625,
      "logps/chosen": -334.0,
      "logps/rejected": -728.0,
      "loss": 0.3639,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.078125,
      "rewards/margins": 3.984375,
      "rewards/rejected": -6.0625,
      "step": 10050
    },
    {
      "epoch": 0.7569031675569935,
      "grad_norm": 29.04369744685261,
      "learning_rate": 8.474740675737921e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.6875,
      "logps/chosen": -296.0,
      "logps/rejected": -724.0,
      "loss": 0.3513,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -5.96875,
      "step": 10060
    },
    {
      "epoch": 0.7576555563915431,
      "grad_norm": 22.845148052012092,
      "learning_rate": 8.425525554542167e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5,
      "logps/chosen": -344.0,
      "logps/rejected": -720.0,
      "loss": 0.4395,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.109375,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.875,
      "step": 10070
    },
    {
      "epoch": 0.7584079452260929,
      "grad_norm": 25.378074740666623,
      "learning_rate": 8.376424774950691e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -288.0,
      "logps/rejected": -692.0,
      "loss": 0.3712,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.65625,
      "step": 10080
    },
    {
      "epoch": 0.7591603340606425,
      "grad_norm": 25.042944103810285,
      "learning_rate": 8.327438675692921e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.75,
      "logps/chosen": -290.0,
      "logps/rejected": -672.0,
      "loss": 0.3811,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.5,
      "step": 10090
    },
    {
      "epoch": 0.7599127228951922,
      "grad_norm": 20.131320553157927,
      "learning_rate": 8.278567594707098e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.859375,
      "logps/chosen": -328.0,
      "logps/rejected": -688.0,
      "loss": 0.3577,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.890625,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.6875,
      "step": 10100
    },
    {
      "epoch": 0.7606651117297419,
      "grad_norm": 23.26082165255364,
      "learning_rate": 8.229811869138036e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.5625,
      "logps/chosen": -314.0,
      "logps/rejected": -772.0,
      "loss": 0.3981,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.859375,
      "rewards/margins": 4.5625,
      "rewards/rejected": -6.40625,
      "step": 10110
    },
    {
      "epoch": 0.7614175005642916,
      "grad_norm": 27.223546740877044,
      "learning_rate": 8.181171835334733e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.78125,
      "logps/chosen": -300.0,
      "logps/rejected": -672.0,
      "loss": 0.3793,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.84375,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.625,
      "step": 10120
    },
    {
      "epoch": 0.7621698893988413,
      "grad_norm": 25.164613157607466,
      "learning_rate": 8.132647828848052e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -282.0,
      "logps/rejected": -712.0,
      "loss": 0.3598,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -5.78125,
      "step": 10130
    },
    {
      "epoch": 0.762922278233391,
      "grad_norm": 23.782750962338206,
      "learning_rate": 8.084240184428465e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -308.0,
      "logps/rejected": -704.0,
      "loss": 0.3683,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.765625,
      "rewards/margins": 3.984375,
      "rewards/rejected": -5.75,
      "step": 10140
    },
    {
      "epoch": 0.7636746670679407,
      "grad_norm": 21.76363679279173,
      "learning_rate": 8.035949236023668e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.703125,
      "logps/chosen": -292.0,
      "logps/rejected": -696.0,
      "loss": 0.3839,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.75,
      "rewards/margins": 3.96875,
      "rewards/rejected": -5.71875,
      "step": 10150
    },
    {
      "epoch": 0.7644270559024904,
      "grad_norm": 23.89102012419261,
      "learning_rate": 7.987775316776311e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.859375,
      "logps/chosen": -290.0,
      "logps/rejected": -652.0,
      "loss": 0.3775,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.4375,
      "step": 10160
    },
    {
      "epoch": 0.7651794447370401,
      "grad_norm": 21.139117981553007,
      "learning_rate": 7.939718759021729e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.59375,
      "logps/chosen": -320.0,
      "logps/rejected": -612.0,
      "loss": 0.3792,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 3.09375,
      "rewards/rejected": -4.875,
      "step": 10170
    },
    {
      "epoch": 0.7659318335715898,
      "grad_norm": 27.851478196143994,
      "learning_rate": 7.891779894285597e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.765625,
      "logps/chosen": -314.0,
      "logps/rejected": -648.0,
      "loss": 0.3898,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 3.53125,
      "rewards/rejected": -5.34375,
      "step": 10180
    },
    {
      "epoch": 0.7666842224061395,
      "grad_norm": 23.344950740160495,
      "learning_rate": 7.843959053281663e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.734375,
      "logps/chosen": -322.0,
      "logps/rejected": -668.0,
      "loss": 0.3416,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.890625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.59375,
      "step": 10190
    },
    {
      "epoch": 0.7674366112406892,
      "grad_norm": 33.868265083262806,
      "learning_rate": 7.796256565909487e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.59375,
      "logps/chosen": -338.0,
      "logps/rejected": -740.0,
      "loss": 0.3583,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.0625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -6.09375,
      "step": 10200
    },
    {
      "epoch": 0.7681890000752389,
      "grad_norm": 21.389422631235963,
      "learning_rate": 7.748672761252123e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.703125,
      "logps/chosen": -316.0,
      "logps/rejected": -684.0,
      "loss": 0.3355,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.0625,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.84375,
      "step": 10210
    },
    {
      "epoch": 0.7689413889097886,
      "grad_norm": 20.678826897202068,
      "learning_rate": 7.701207967573911e-08,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.671875,
      "logps/chosen": -332.0,
      "logps/rejected": -704.0,
      "loss": 0.42,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.0,
      "rewards/margins": 3.921875,
      "rewards/rejected": -5.9375,
      "step": 10220
    },
    {
      "epoch": 0.7696937777443382,
      "grad_norm": 29.20633190470031,
      "learning_rate": 7.653862512318146e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.78125,
      "logps/chosen": -298.0,
      "logps/rejected": -676.0,
      "loss": 0.3909,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.59375,
      "step": 10230
    },
    {
      "epoch": 0.770446166578888,
      "grad_norm": 25.047434399395975,
      "learning_rate": 7.606636722104845e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.765625,
      "logps/chosen": -290.0,
      "logps/rejected": -684.0,
      "loss": 0.3494,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.828125,
      "rewards/margins": 3.875,
      "rewards/rejected": -5.71875,
      "step": 10240
    },
    {
      "epoch": 0.7711985554134376,
      "grad_norm": 24.388917809051573,
      "learning_rate": 7.559530922728527e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.5625,
      "logps/chosen": -326.0,
      "logps/rejected": -736.0,
      "loss": 0.35,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.0,
      "rewards/margins": 4.0625,
      "rewards/rejected": -6.0625,
      "step": 10250
    },
    {
      "epoch": 0.7719509442479874,
      "grad_norm": 19.109551824384315,
      "learning_rate": 7.512545439155904e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5,
      "logps/chosen": -318.0,
      "logps/rejected": -692.0,
      "loss": 0.3867,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.53125,
      "step": 10260
    },
    {
      "epoch": 0.772703333082537,
      "grad_norm": 32.00081811512626,
      "learning_rate": 7.46568059552369e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.65625,
      "logps/chosen": -332.0,
      "logps/rejected": -644.0,
      "loss": 0.3548,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.015625,
      "rewards/margins": 3.328125,
      "rewards/rejected": -5.34375,
      "step": 10270
    },
    {
      "epoch": 0.7734557219170868,
      "grad_norm": 21.358913895445404,
      "learning_rate": 7.418936715136334e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.75,
      "logps/chosen": -306.0,
      "logps/rejected": -764.0,
      "loss": 0.351,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.84375,
      "rewards/margins": 4.6875,
      "rewards/rejected": -6.53125,
      "step": 10280
    },
    {
      "epoch": 0.7742081107516364,
      "grad_norm": 18.73025391174228,
      "learning_rate": 7.372314120463798e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.546875,
      "logps/chosen": -308.0,
      "logps/rejected": -704.0,
      "loss": 0.3463,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.90625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -5.96875,
      "step": 10290
    },
    {
      "epoch": 0.7749604995861862,
      "grad_norm": 39.045785044120194,
      "learning_rate": 7.325813133139361e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.671875,
      "logps/chosen": -340.0,
      "logps/rejected": -700.0,
      "loss": 0.3691,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.125,
      "rewards/margins": 3.625,
      "rewards/rejected": -5.75,
      "step": 10300
    },
    {
      "epoch": 0.7757128884207358,
      "grad_norm": 30.014686107214896,
      "learning_rate": 7.279434073957349e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.640625,
      "logps/chosen": -350.0,
      "logps/rejected": -740.0,
      "loss": 0.3509,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.140625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -6.15625,
      "step": 10310
    },
    {
      "epoch": 0.7764652772552856,
      "grad_norm": 34.843359108943105,
      "learning_rate": 7.233177262870946e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.484375,
      "logps/chosen": -338.0,
      "logps/rejected": -716.0,
      "loss": 0.3702,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.1875,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.875,
      "step": 10320
    },
    {
      "epoch": 0.7772176660898352,
      "grad_norm": 30.237700881753756,
      "learning_rate": 7.187043018990016e-08,
      "logits/chosen": -2.5,
      "logits/rejected": -2.640625,
      "logps/chosen": -350.0,
      "logps/rejected": -688.0,
      "loss": 0.3585,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.140625,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.6875,
      "step": 10330
    },
    {
      "epoch": 0.7779700549243849,
      "grad_norm": 20.69640292211596,
      "learning_rate": 7.141031660578839e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.84375,
      "logps/chosen": -324.0,
      "logps/rejected": -688.0,
      "loss": 0.3519,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8984375,
      "rewards/margins": 3.84375,
      "rewards/rejected": -5.75,
      "step": 10340
    },
    {
      "epoch": 0.7787224437589346,
      "grad_norm": 18.437021282072745,
      "learning_rate": 7.095143505053982e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.5,
      "logps/chosen": -330.0,
      "logps/rejected": -704.0,
      "loss": 0.3342,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -5.8125,
      "step": 10350
    },
    {
      "epoch": 0.7794748325934843,
      "grad_norm": 28.310595031587766,
      "learning_rate": 7.049378868982065e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.890625,
      "logps/chosen": -282.0,
      "logps/rejected": -668.0,
      "loss": 0.3605,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 3.875,
      "rewards/rejected": -5.5625,
      "step": 10360
    },
    {
      "epoch": 0.780227221428034,
      "grad_norm": 23.09708654251049,
      "learning_rate": 7.003738068077564e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.4375,
      "logps/chosen": -292.0,
      "logps/rejected": -672.0,
      "loss": 0.3484,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -5.5625,
      "step": 10370
    },
    {
      "epoch": 0.7809796102625837,
      "grad_norm": 21.437441354153968,
      "learning_rate": 6.958221417200705e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.6875,
      "logps/chosen": -328.0,
      "logps/rejected": -696.0,
      "loss": 0.3625,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.90625,
      "rewards/margins": 3.828125,
      "rewards/rejected": -5.71875,
      "step": 10380
    },
    {
      "epoch": 0.7817319990971334,
      "grad_norm": 18.85689652401531,
      "learning_rate": 6.912829230355208e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.765625,
      "logps/chosen": -338.0,
      "logps/rejected": -728.0,
      "loss": 0.3437,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 3.921875,
      "rewards/rejected": -5.90625,
      "step": 10390
    },
    {
      "epoch": 0.7824843879316831,
      "grad_norm": 21.43537062683571,
      "learning_rate": 6.867561820686187e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.75,
      "logps/chosen": -324.0,
      "logps/rejected": -712.0,
      "loss": 0.3395,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 4.0,
      "rewards/rejected": -5.96875,
      "step": 10400
    },
    {
      "epoch": 0.7832367767662328,
      "grad_norm": 24.409208487666447,
      "learning_rate": 6.822419500477947e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.640625,
      "logps/chosen": -312.0,
      "logps/rejected": -688.0,
      "loss": 0.3902,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.75,
      "rewards/margins": 4.0,
      "rewards/rejected": -5.75,
      "step": 10410
    },
    {
      "epoch": 0.7839891656007825,
      "grad_norm": 32.88929527393551,
      "learning_rate": 6.777402581151825e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.515625,
      "logps/chosen": -334.0,
      "logps/rejected": -732.0,
      "loss": 0.4045,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.15625,
      "rewards/margins": 3.921875,
      "rewards/rejected": -6.0625,
      "step": 10420
    },
    {
      "epoch": 0.7847415544353322,
      "grad_norm": 28.12746841360069,
      "learning_rate": 6.732511373264107e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.609375,
      "logps/chosen": -304.0,
      "logps/rejected": -680.0,
      "loss": 0.4021,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.78125,
      "rewards/margins": 3.890625,
      "rewards/rejected": -5.65625,
      "step": 10430
    },
    {
      "epoch": 0.7854939432698819,
      "grad_norm": 28.708366733612376,
      "learning_rate": 6.687746186503796e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.625,
      "logps/chosen": -312.0,
      "logps/rejected": -732.0,
      "loss": 0.3944,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -6.0,
      "step": 10440
    },
    {
      "epoch": 0.7862463321044316,
      "grad_norm": 23.567157215239327,
      "learning_rate": 6.64310732969053e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.765625,
      "logps/chosen": -332.0,
      "logps/rejected": -652.0,
      "loss": 0.3682,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.09375,
      "rewards/margins": 3.390625,
      "rewards/rejected": -5.5,
      "step": 10450
    },
    {
      "epoch": 0.7869987209389813,
      "grad_norm": 33.46651476559856,
      "learning_rate": 6.598595110772467e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.765625,
      "logps/chosen": -310.0,
      "logps/rejected": -676.0,
      "loss": 0.3706,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.921875,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.6875,
      "step": 10460
    },
    {
      "epoch": 0.7877511097735309,
      "grad_norm": 21.419930547432394,
      "learning_rate": 6.554209836824081e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.640625,
      "logps/chosen": -318.0,
      "logps/rejected": -728.0,
      "loss": 0.3594,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.9375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -6.09375,
      "step": 10470
    },
    {
      "epoch": 0.7885034986080807,
      "grad_norm": 30.49401302899843,
      "learning_rate": 6.509951814044151e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.671875,
      "logps/chosen": -310.0,
      "logps/rejected": -652.0,
      "loss": 0.3564,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 3.515625,
      "rewards/rejected": -5.375,
      "step": 10480
    },
    {
      "epoch": 0.7892558874426303,
      "grad_norm": 24.68297991948101,
      "learning_rate": 6.465821347753555e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.65625,
      "logps/chosen": -318.0,
      "logps/rejected": -660.0,
      "loss": 0.3573,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.984375,
      "rewards/margins": 3.515625,
      "rewards/rejected": -5.5,
      "step": 10490
    },
    {
      "epoch": 0.7900082762771801,
      "grad_norm": 27.37619026207998,
      "learning_rate": 6.421818742393217e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.671875,
      "logps/chosen": -330.0,
      "logps/rejected": -712.0,
      "loss": 0.3656,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.078125,
      "rewards/margins": 3.71875,
      "rewards/rejected": -5.78125,
      "step": 10500
    },
    {
      "epoch": 0.7907606651117297,
      "grad_norm": 24.924303292945282,
      "learning_rate": 6.377944301522004e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.484375,
      "logps/chosen": -328.0,
      "logps/rejected": -688.0,
      "loss": 0.3513,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.71875,
      "step": 10510
    },
    {
      "epoch": 0.7915130539462795,
      "grad_norm": 19.835285639485917,
      "learning_rate": 6.3341983278146e-08,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.46875,
      "logps/chosen": -342.0,
      "logps/rejected": -692.0,
      "loss": 0.3865,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.265625,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.75,
      "step": 10520
    },
    {
      "epoch": 0.7922654427808291,
      "grad_norm": 21.93366130318321,
      "learning_rate": 6.290581123059441e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.609375,
      "logps/chosen": -316.0,
      "logps/rejected": -692.0,
      "loss": 0.3937,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.96875,
      "rewards/margins": 3.953125,
      "rewards/rejected": -5.90625,
      "step": 10530
    },
    {
      "epoch": 0.7930178316153789,
      "grad_norm": 26.9934681512597,
      "learning_rate": 6.247092988156652e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.953125,
      "logps/chosen": -350.0,
      "logps/rejected": -652.0,
      "loss": 0.3656,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.171875,
      "rewards/margins": 3.34375,
      "rewards/rejected": -5.53125,
      "step": 10540
    },
    {
      "epoch": 0.7937702204499285,
      "grad_norm": 22.750132361264725,
      "learning_rate": 6.203734223115922e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.625,
      "logps/chosen": -336.0,
      "logps/rejected": -768.0,
      "loss": 0.3278,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.078125,
      "rewards/margins": 4.25,
      "rewards/rejected": -6.3125,
      "step": 10550
    },
    {
      "epoch": 0.7945226092844783,
      "grad_norm": 23.252154525081618,
      "learning_rate": 6.160505127054475e-08,
      "logits/chosen": -2.5,
      "logits/rejected": -2.59375,
      "logps/chosen": -304.0,
      "logps/rejected": -724.0,
      "loss": 0.3825,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 4.0,
      "rewards/rejected": -5.875,
      "step": 10560
    },
    {
      "epoch": 0.7952749981190279,
      "grad_norm": 24.62747305104957,
      "learning_rate": 6.117405998194991e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.765625,
      "logps/chosen": -296.0,
      "logps/rejected": -692.0,
      "loss": 0.3575,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.8125,
      "rewards/margins": 3.921875,
      "rewards/rejected": -5.71875,
      "step": 10570
    },
    {
      "epoch": 0.7960273869535776,
      "grad_norm": 26.18387290687485,
      "learning_rate": 6.074437133863547e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.421875,
      "logps/chosen": -294.0,
      "logps/rejected": -696.0,
      "loss": 0.398,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.828125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -5.90625,
      "step": 10580
    },
    {
      "epoch": 0.7967797757881273,
      "grad_norm": 19.169606743700367,
      "learning_rate": 6.031598830487586e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.734375,
      "logps/chosen": -310.0,
      "logps/rejected": -656.0,
      "loss": 0.3546,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.890625,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.46875,
      "step": 10590
    },
    {
      "epoch": 0.797532164622677,
      "grad_norm": 19.62652753915341,
      "learning_rate": 5.98889138359383e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.796875,
      "logps/chosen": -316.0,
      "logps/rejected": -704.0,
      "loss": 0.3706,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.890625,
      "rewards/margins": 4.09375,
      "rewards/rejected": -5.96875,
      "step": 10600
    },
    {
      "epoch": 0.7982845534572267,
      "grad_norm": 20.747603973205266,
      "learning_rate": 5.946315087806294e-08,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.375,
      "logps/chosen": -320.0,
      "logps/rejected": -664.0,
      "loss": 0.3966,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.109375,
      "rewards/margins": 3.375,
      "rewards/rejected": -5.46875,
      "step": 10610
    },
    {
      "epoch": 0.7990369422917764,
      "grad_norm": 20.487632228619603,
      "learning_rate": 5.903870236844211e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.59375,
      "logps/chosen": -326.0,
      "logps/rejected": -684.0,
      "loss": 0.4241,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.078125,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.71875,
      "step": 10620
    },
    {
      "epoch": 0.7997893311263261,
      "grad_norm": 18.32148132408248,
      "learning_rate": 5.861557123520011e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.65625,
      "logps/chosen": -292.0,
      "logps/rejected": -708.0,
      "loss": 0.3676,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 4.03125,
      "rewards/rejected": -5.9375,
      "step": 10630
    },
    {
      "epoch": 0.8005417199608758,
      "grad_norm": 21.202133376625458,
      "learning_rate": 5.819376039737348e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.390625,
      "logps/chosen": -288.0,
      "logps/rejected": -720.0,
      "loss": 0.3228,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 4.46875,
      "rewards/rejected": -6.125,
      "step": 10640
    },
    {
      "epoch": 0.8012941087954255,
      "grad_norm": 20.40706235079479,
      "learning_rate": 5.7773272764889966e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.671875,
      "logps/chosen": -296.0,
      "logps/rejected": -664.0,
      "loss": 0.3648,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.84375,
      "rewards/margins": 3.625,
      "rewards/rejected": -5.46875,
      "step": 10650
    },
    {
      "epoch": 0.8020464976299752,
      "grad_norm": 13.630219496301066,
      "learning_rate": 5.735411123854952e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.625,
      "logps/chosen": -328.0,
      "logps/rejected": -692.0,
      "loss": 0.354,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.984375,
      "rewards/margins": 3.65625,
      "rewards/rejected": -5.65625,
      "step": 10660
    },
    {
      "epoch": 0.8027988864645249,
      "grad_norm": 19.615570229986616,
      "learning_rate": 5.693627871000337e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.671875,
      "logps/chosen": -294.0,
      "logps/rejected": -652.0,
      "loss": 0.3752,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.71875,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.53125,
      "step": 10670
    },
    {
      "epoch": 0.8035512752990746,
      "grad_norm": 21.390604667039668,
      "learning_rate": 5.651977806173452e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.671875,
      "logps/chosen": -336.0,
      "logps/rejected": -652.0,
      "loss": 0.3686,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.0625,
      "rewards/margins": 3.296875,
      "rewards/rejected": -5.375,
      "step": 10680
    },
    {
      "epoch": 0.8043036641336243,
      "grad_norm": 23.42238959442068,
      "learning_rate": 5.610461216703796e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.609375,
      "logps/chosen": -314.0,
      "logps/rejected": -648.0,
      "loss": 0.3589,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -5.28125,
      "step": 10690
    },
    {
      "epoch": 0.805056052968174,
      "grad_norm": 24.537134394318453,
      "learning_rate": 5.569078389000048e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.6875,
      "logps/chosen": -308.0,
      "logps/rejected": -688.0,
      "loss": 0.3747,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8125,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.5625,
      "step": 10700
    },
    {
      "epoch": 0.8058084418027236,
      "grad_norm": 18.309336153861217,
      "learning_rate": 5.5278296085481125e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.578125,
      "logps/chosen": -314.0,
      "logps/rejected": -700.0,
      "loss": 0.3308,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 3.984375,
      "rewards/rejected": -5.8125,
      "step": 10710
    },
    {
      "epoch": 0.8065608306372734,
      "grad_norm": 21.224313880784322,
      "learning_rate": 5.486715159909166e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.65625,
      "logps/chosen": -298.0,
      "logps/rejected": -668.0,
      "loss": 0.3454,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.53125,
      "step": 10720
    },
    {
      "epoch": 0.807313219471823,
      "grad_norm": 20.738352283077514,
      "learning_rate": 5.4457353267176545e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.671875,
      "logps/chosen": -306.0,
      "logps/rejected": -700.0,
      "loss": 0.3611,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.78125,
      "rewards/margins": 4.125,
      "rewards/rejected": -5.90625,
      "step": 10730
    },
    {
      "epoch": 0.8080656083063728,
      "grad_norm": 19.75013540032933,
      "learning_rate": 5.4048903916793676e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.71875,
      "logps/chosen": -312.0,
      "logps/rejected": -664.0,
      "loss": 0.404,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.890625,
      "rewards/margins": 3.53125,
      "rewards/rejected": -5.4375,
      "step": 10740
    },
    {
      "epoch": 0.8088179971409224,
      "grad_norm": 26.751238964631096,
      "learning_rate": 5.3641806365694765e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.578125,
      "logps/chosen": -310.0,
      "logps/rejected": -676.0,
      "loss": 0.3601,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.625,
      "step": 10750
    },
    {
      "epoch": 0.8095703859754722,
      "grad_norm": 21.9541119932258,
      "learning_rate": 5.323606342230591e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.609375,
      "logps/chosen": -298.0,
      "logps/rejected": -804.0,
      "loss": 0.367,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 5.125,
      "rewards/rejected": -6.9375,
      "step": 10760
    },
    {
      "epoch": 0.8103227748100218,
      "grad_norm": 21.72435377766189,
      "learning_rate": 5.283167788570836e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.6875,
      "logps/chosen": -288.0,
      "logps/rejected": -668.0,
      "loss": 0.3691,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.734375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -5.59375,
      "step": 10770
    },
    {
      "epoch": 0.8110751636445716,
      "grad_norm": 25.09323991156017,
      "learning_rate": 5.2428652545618954e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.640625,
      "logps/chosen": -302.0,
      "logps/rejected": -748.0,
      "loss": 0.3728,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.7578125,
      "rewards/margins": 4.4375,
      "rewards/rejected": -6.21875,
      "step": 10780
    },
    {
      "epoch": 0.8118275524791212,
      "grad_norm": 24.23680200099933,
      "learning_rate": 5.202699018237094e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.890625,
      "logps/chosen": -312.0,
      "logps/rejected": -664.0,
      "loss": 0.3937,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.953125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.65625,
      "step": 10790
    },
    {
      "epoch": 0.812579941313671,
      "grad_norm": 19.377169526684966,
      "learning_rate": 5.162669356689511e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.671875,
      "logps/chosen": -316.0,
      "logps/rejected": -668.0,
      "loss": 0.3589,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 3.65625,
      "rewards/rejected": -5.625,
      "step": 10800
    },
    {
      "epoch": 0.8133323301482206,
      "grad_norm": 20.881444260602585,
      "learning_rate": 5.122776546070015e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.6875,
      "logps/chosen": -324.0,
      "logps/rejected": -688.0,
      "loss": 0.3902,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.96875,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.6875,
      "step": 10810
    },
    {
      "epoch": 0.8140847189827704,
      "grad_norm": 24.94874147484874,
      "learning_rate": 5.083020861585413e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5625,
      "logps/chosen": -284.0,
      "logps/rejected": -696.0,
      "loss": 0.3573,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 4.03125,
      "rewards/rejected": -5.78125,
      "step": 10820
    },
    {
      "epoch": 0.81483710781732,
      "grad_norm": 30.118905690163082,
      "learning_rate": 5.0434025774965e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.75,
      "logps/chosen": -300.0,
      "logps/rejected": -652.0,
      "loss": 0.3169,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -5.53125,
      "step": 10830
    },
    {
      "epoch": 0.8155894966518696,
      "grad_norm": 16.37623923168267,
      "learning_rate": 5.003921967116201e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.796875,
      "logps/chosen": -298.0,
      "logps/rejected": -672.0,
      "loss": 0.3524,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.5625,
      "step": 10840
    },
    {
      "epoch": 0.8163418854864194,
      "grad_norm": 21.54635965713696,
      "learning_rate": 4.9645793028076975e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.71875,
      "logps/chosen": -294.0,
      "logps/rejected": -688.0,
      "loss": 0.4406,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.734375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -5.78125,
      "step": 10850
    },
    {
      "epoch": 0.817094274320969,
      "grad_norm": 29.804682346530715,
      "learning_rate": 4.925374855982495e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.609375,
      "logps/chosen": -316.0,
      "logps/rejected": -712.0,
      "loss": 0.3849,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.953125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -6.03125,
      "step": 10860
    },
    {
      "epoch": 0.8178466631555188,
      "grad_norm": 16.498986357668482,
      "learning_rate": 4.886308897098621e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.703125,
      "logps/chosen": -296.0,
      "logps/rejected": -776.0,
      "loss": 0.3329,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.734375,
      "rewards/margins": 4.875,
      "rewards/rejected": -6.59375,
      "step": 10870
    },
    {
      "epoch": 0.8185990519900684,
      "grad_norm": 22.914024930481446,
      "learning_rate": 4.847381695658692e-08,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.71875,
      "logps/chosen": -340.0,
      "logps/rejected": -692.0,
      "loss": 0.3668,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.125,
      "rewards/margins": 3.578125,
      "rewards/rejected": -5.6875,
      "step": 10880
    },
    {
      "epoch": 0.8193514408246182,
      "grad_norm": 29.49308186964694,
      "learning_rate": 4.80859352020809e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.5625,
      "logps/chosen": -360.0,
      "logps/rejected": -696.0,
      "loss": 0.3788,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.28125,
      "rewards/margins": 3.578125,
      "rewards/rejected": -5.84375,
      "step": 10890
    },
    {
      "epoch": 0.8201038296591678,
      "grad_norm": 30.15913103650645,
      "learning_rate": 4.769944638333123e-08,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.578125,
      "logps/chosen": -326.0,
      "logps/rejected": -688.0,
      "loss": 0.3481,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.953125,
      "rewards/margins": 3.828125,
      "rewards/rejected": -5.78125,
      "step": 10900
    },
    {
      "epoch": 0.8208562184937176,
      "grad_norm": 16.417673738696234,
      "learning_rate": 4.731435316659141e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.546875,
      "logps/chosen": -304.0,
      "logps/rejected": -700.0,
      "loss": 0.3689,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 3.984375,
      "rewards/rejected": -5.8125,
      "step": 10910
    },
    {
      "epoch": 0.8216086073282672,
      "grad_norm": 24.831997194521442,
      "learning_rate": 4.693065820848724e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.65625,
      "logps/chosen": -292.0,
      "logps/rejected": -696.0,
      "loss": 0.3904,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 3.890625,
      "rewards/rejected": -5.59375,
      "step": 10920
    },
    {
      "epoch": 0.822360996162817,
      "grad_norm": 18.666367631841194,
      "learning_rate": 4.654836415599836e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.84375,
      "logps/chosen": -304.0,
      "logps/rejected": -680.0,
      "loss": 0.3318,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 3.859375,
      "rewards/rejected": -5.71875,
      "step": 10930
    },
    {
      "epoch": 0.8231133849973666,
      "grad_norm": 26.75853615635492,
      "learning_rate": 4.616747364644008e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.609375,
      "logps/chosen": -310.0,
      "logps/rejected": -708.0,
      "loss": 0.3889,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.921875,
      "rewards/rejected": -5.78125,
      "step": 10940
    },
    {
      "epoch": 0.8238657738319163,
      "grad_norm": 27.43162255044618,
      "learning_rate": 4.578798930744523e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.609375,
      "logps/chosen": -298.0,
      "logps/rejected": -672.0,
      "loss": 0.3813,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.8125,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.5,
      "step": 10950
    },
    {
      "epoch": 0.824618162666466,
      "grad_norm": 22.89988155757462,
      "learning_rate": 4.5409913756945835e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.8125,
      "logps/chosen": -282.0,
      "logps/rejected": -640.0,
      "loss": 0.3343,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.640625,
      "rewards/margins": 3.71875,
      "rewards/rejected": -5.375,
      "step": 10960
    },
    {
      "epoch": 0.8253705515010157,
      "grad_norm": 38.55577455347173,
      "learning_rate": 4.5033249603155135e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.75,
      "logps/chosen": -334.0,
      "logps/rejected": -644.0,
      "loss": 0.3854,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 3.375,
      "rewards/rejected": -5.375,
      "step": 10970
    },
    {
      "epoch": 0.8261229403355654,
      "grad_norm": 21.855005255904928,
      "learning_rate": 4.465799944454984e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.828125,
      "logps/chosen": -326.0,
      "logps/rejected": -672.0,
      "loss": 0.3844,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.578125,
      "rewards/rejected": -5.4375,
      "step": 10980
    },
    {
      "epoch": 0.8268753291701151,
      "grad_norm": 17.051701500934733,
      "learning_rate": 4.428416586985184e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.65625,
      "logps/chosen": -318.0,
      "logps/rejected": -720.0,
      "loss": 0.3748,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 4.125,
      "rewards/rejected": -6.0,
      "step": 10990
    },
    {
      "epoch": 0.8276277180046648,
      "grad_norm": 25.81647010639844,
      "learning_rate": 4.3911751458010434e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.859375,
      "logps/chosen": -296.0,
      "logps/rejected": -652.0,
      "loss": 0.3162,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.46875,
      "step": 11000
    },
    {
      "epoch": 0.8276277180046648,
      "eval_logits/chosen": -2.5625,
      "eval_logits/rejected": -2.625,
      "eval_logps/chosen": -334.0,
      "eval_logps/rejected": -660.0,
      "eval_loss": 0.26222386956214905,
      "eval_rewards/accuracies": 0.8855788707733154,
      "eval_rewards/chosen": -2.03125,
      "eval_rewards/margins": 3.40625,
      "eval_rewards/rejected": -5.4375,
      "eval_runtime": 2669.0233,
      "eval_samples_per_second": 35.404,
      "eval_steps_per_second": 0.553,
      "step": 11000
    },
    {
      "epoch": 0.8283801068392145,
      "grad_norm": 26.11862636141394,
      "learning_rate": 4.354075877818475e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.765625,
      "logps/chosen": -338.0,
      "logps/rejected": -704.0,
      "loss": 0.3992,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.953125,
      "rewards/rejected": -6.0,
      "step": 11010
    },
    {
      "epoch": 0.8291324956737642,
      "grad_norm": 23.81210170639515,
      "learning_rate": 4.3171190389725746e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.640625,
      "logps/chosen": -272.0,
      "logps/rejected": -684.0,
      "loss": 0.3104,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.5625,
      "rewards/margins": 4.21875,
      "rewards/rejected": -5.78125,
      "step": 11020
    },
    {
      "epoch": 0.8298848845083139,
      "grad_norm": 32.063973886836564,
      "learning_rate": 4.280304884215885e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.703125,
      "logps/chosen": -336.0,
      "logps/rejected": -724.0,
      "loss": 0.3556,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.03125,
      "rewards/margins": 4.03125,
      "rewards/rejected": -6.03125,
      "step": 11030
    },
    {
      "epoch": 0.8306372733428636,
      "grad_norm": 30.82580846061098,
      "learning_rate": 4.2436336675166076e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.40625,
      "logps/chosen": -314.0,
      "logps/rejected": -728.0,
      "loss": 0.3879,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.921875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -6.09375,
      "step": 11040
    },
    {
      "epoch": 0.8313896621774133,
      "grad_norm": 27.754534285632026,
      "learning_rate": 4.207105641856859e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.515625,
      "logps/chosen": -306.0,
      "logps/rejected": -676.0,
      "loss": 0.3671,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.59375,
      "step": 11050
    },
    {
      "epoch": 0.832142051011963,
      "grad_norm": 26.316601215473234,
      "learning_rate": 4.17072105923095e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.5625,
      "logps/chosen": -314.0,
      "logps/rejected": -632.0,
      "loss": 0.3656,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 3.375,
      "rewards/rejected": -5.34375,
      "step": 11060
    },
    {
      "epoch": 0.8328944398465127,
      "grad_norm": 34.138830502008446,
      "learning_rate": 4.134480170643606e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.4375,
      "logps/chosen": -336.0,
      "logps/rejected": -732.0,
      "loss": 0.3901,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.203125,
      "rewards/margins": 3.9375,
      "rewards/rejected": -6.125,
      "step": 11070
    },
    {
      "epoch": 0.8336468286810623,
      "grad_norm": 20.399148273973292,
      "learning_rate": 4.0983832261082624e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.65625,
      "logps/chosen": -342.0,
      "logps/rejected": -708.0,
      "loss": 0.3535,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.09375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.875,
      "step": 11080
    },
    {
      "epoch": 0.8343992175156121,
      "grad_norm": 24.144686187943357,
      "learning_rate": 4.062430474645353e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.828125,
      "logps/chosen": -292.0,
      "logps/rejected": -700.0,
      "loss": 0.3199,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -5.96875,
      "step": 11090
    },
    {
      "epoch": 0.8351516063501617,
      "grad_norm": 23.185539666710994,
      "learning_rate": 4.0266221642805355e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.625,
      "logps/chosen": -314.0,
      "logps/rejected": -676.0,
      "loss": 0.3635,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.625,
      "step": 11100
    },
    {
      "epoch": 0.8359039951847115,
      "grad_norm": 26.290159036988697,
      "learning_rate": 3.990958542043052e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.65625,
      "logps/chosen": -310.0,
      "logps/rejected": -648.0,
      "loss": 0.3914,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.390625,
      "rewards/rejected": -5.34375,
      "step": 11110
    },
    {
      "epoch": 0.8366563840192611,
      "grad_norm": 29.26136586641466,
      "learning_rate": 3.9554398539639766e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.75,
      "logps/chosen": -312.0,
      "logps/rejected": -652.0,
      "loss": 0.3438,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -5.5625,
      "step": 11120
    },
    {
      "epoch": 0.8374087728538109,
      "grad_norm": 24.16804004690269,
      "learning_rate": 3.92006634507453e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.515625,
      "logps/chosen": -320.0,
      "logps/rejected": -692.0,
      "loss": 0.3671,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.890625,
      "rewards/rejected": -5.78125,
      "step": 11130
    },
    {
      "epoch": 0.8381611616883605,
      "grad_norm": 18.209102622303558,
      "learning_rate": 3.884838259404402e-08,
      "logits/chosen": -2.5,
      "logits/rejected": -2.625,
      "logps/chosen": -286.0,
      "logps/rejected": -692.0,
      "loss": 0.3833,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -5.65625,
      "step": 11140
    },
    {
      "epoch": 0.8389135505229103,
      "grad_norm": 23.742728131858307,
      "learning_rate": 3.8497558399800454e-08,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.40625,
      "logps/chosen": -298.0,
      "logps/rejected": -716.0,
      "loss": 0.3708,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 4.1875,
      "rewards/rejected": -5.90625,
      "step": 11150
    },
    {
      "epoch": 0.8396659393574599,
      "grad_norm": 22.367786974321834,
      "learning_rate": 3.814819328823027e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.65625,
      "logps/chosen": -338.0,
      "logps/rejected": -680.0,
      "loss": 0.3816,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.078125,
      "rewards/margins": 3.515625,
      "rewards/rejected": -5.59375,
      "step": 11160
    },
    {
      "epoch": 0.8404183281920097,
      "grad_norm": 32.248532030181295,
      "learning_rate": 3.780028966948326e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.828125,
      "logps/chosen": -332.0,
      "logps/rejected": -668.0,
      "loss": 0.3958,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.078125,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.5625,
      "step": 11170
    },
    {
      "epoch": 0.8411707170265593,
      "grad_norm": 24.301667499786845,
      "learning_rate": 3.7453849943626964e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.75,
      "logps/chosen": -322.0,
      "logps/rejected": -628.0,
      "loss": 0.3733,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.171875,
      "rewards/rejected": -5.125,
      "step": 11180
    },
    {
      "epoch": 0.841923105861109,
      "grad_norm": 27.88261287439373,
      "learning_rate": 3.710887650063005e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.625,
      "logps/chosen": -326.0,
      "logps/rejected": -628.0,
      "loss": 0.3785,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.0,
      "rewards/margins": 3.34375,
      "rewards/rejected": -5.34375,
      "step": 11190
    },
    {
      "epoch": 0.8426754946956587,
      "grad_norm": 28.579360022031228,
      "learning_rate": 3.676537172034563e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.328125,
      "logps/chosen": -320.0,
      "logps/rejected": -720.0,
      "loss": 0.3904,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.7734375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -5.84375,
      "step": 11200
    },
    {
      "epoch": 0.8434278835302084,
      "grad_norm": 22.633462211030594,
      "learning_rate": 3.642333797249536e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.671875,
      "logps/chosen": -320.0,
      "logps/rejected": -684.0,
      "loss": 0.3588,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.6875,
      "step": 11210
    },
    {
      "epoch": 0.8441802723647581,
      "grad_norm": 24.00081131550051,
      "learning_rate": 3.608277761665243e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5625,
      "logps/chosen": -304.0,
      "logps/rejected": -680.0,
      "loss": 0.3865,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.0,
      "rewards/margins": 3.515625,
      "rewards/rejected": -5.53125,
      "step": 11220
    },
    {
      "epoch": 0.8449326611993078,
      "grad_norm": 28.173585145512753,
      "learning_rate": 3.574369300222568e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.8125,
      "logps/chosen": -294.0,
      "logps/rejected": -660.0,
      "loss": 0.3514,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.46875,
      "step": 11230
    },
    {
      "epoch": 0.8456850500338575,
      "grad_norm": 29.752681220094825,
      "learning_rate": 3.540608646844348e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.453125,
      "logps/chosen": -294.0,
      "logps/rejected": -696.0,
      "loss": 0.34,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.5625,
      "step": 11240
    },
    {
      "epoch": 0.8464374388684072,
      "grad_norm": 28.317002768274715,
      "learning_rate": 3.506996034433723e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.71875,
      "logps/chosen": -308.0,
      "logps/rejected": -672.0,
      "loss": 0.3472,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 3.859375,
      "rewards/rejected": -5.65625,
      "step": 11250
    },
    {
      "epoch": 0.8471898277029569,
      "grad_norm": 19.572333789417254,
      "learning_rate": 3.473531694872556e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -310.0,
      "logps/rejected": -776.0,
      "loss": 0.3723,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.921875,
      "rewards/margins": 4.34375,
      "rewards/rejected": -6.25,
      "step": 11260
    },
    {
      "epoch": 0.8479422165375066,
      "grad_norm": 27.501093152347696,
      "learning_rate": 3.440215859019838e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.78125,
      "logps/chosen": -288.0,
      "logps/rejected": -668.0,
      "loss": 0.3177,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.75,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.65625,
      "step": 11270
    },
    {
      "epoch": 0.8486946053720563,
      "grad_norm": 17.527127895890818,
      "learning_rate": 3.4070487567100516e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.625,
      "logps/chosen": -322.0,
      "logps/rejected": -668.0,
      "loss": 0.3183,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.625,
      "step": 11280
    },
    {
      "epoch": 0.849446994206606,
      "grad_norm": 32.31618816359289,
      "learning_rate": 3.374030616751661e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.53125,
      "logps/chosen": -332.0,
      "logps/rejected": -700.0,
      "loss": 0.339,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.96875,
      "rewards/margins": 3.828125,
      "rewards/rejected": -5.8125,
      "step": 11290
    },
    {
      "epoch": 0.8501993830411557,
      "grad_norm": 26.993541253166335,
      "learning_rate": 3.3411616669254627e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.53125,
      "logps/chosen": -296.0,
      "logps/rejected": -636.0,
      "loss": 0.4093,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.421875,
      "rewards/rejected": -5.21875,
      "step": 11300
    },
    {
      "epoch": 0.8509517718757054,
      "grad_norm": 22.750639220973525,
      "learning_rate": 3.308442133983036e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.765625,
      "logps/chosen": -330.0,
      "logps/rejected": -672.0,
      "loss": 0.3642,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.84375,
      "rewards/rejected": -5.65625,
      "step": 11310
    },
    {
      "epoch": 0.851704160710255,
      "grad_norm": 24.11784257284009,
      "learning_rate": 3.275872243645214e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.796875,
      "logps/chosen": -334.0,
      "logps/rejected": -656.0,
      "loss": 0.3902,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 3.625,
      "rewards/rejected": -5.5625,
      "step": 11320
    },
    {
      "epoch": 0.8524565495448048,
      "grad_norm": 44.27927634101186,
      "learning_rate": 3.243452220600473e-08,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.546875,
      "logps/chosen": -322.0,
      "logps/rejected": -700.0,
      "loss": 0.3737,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.984375,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.6875,
      "step": 11330
    },
    {
      "epoch": 0.8532089383793544,
      "grad_norm": 17.23453824408464,
      "learning_rate": 3.2111822885034054e-08,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.3125,
      "logps/chosen": -308.0,
      "logps/rejected": -684.0,
      "loss": 0.3712,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -5.71875,
      "step": 11340
    },
    {
      "epoch": 0.8539613272139042,
      "grad_norm": 26.260983555939113,
      "learning_rate": 3.1790626699731955e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.578125,
      "logps/chosen": -310.0,
      "logps/rejected": -684.0,
      "loss": 0.3966,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.59375,
      "step": 11350
    },
    {
      "epoch": 0.8547137160484538,
      "grad_norm": 30.933767618953258,
      "learning_rate": 3.1470935865920505e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.53125,
      "logps/chosen": -310.0,
      "logps/rejected": -680.0,
      "loss": 0.37,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 3.796875,
      "rewards/rejected": -5.65625,
      "step": 11360
    },
    {
      "epoch": 0.8554661048830036,
      "grad_norm": 25.866609563007543,
      "learning_rate": 3.1152752589036904e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.59375,
      "logps/chosen": -302.0,
      "logps/rejected": -692.0,
      "loss": 0.3814,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8125,
      "rewards/margins": 3.921875,
      "rewards/rejected": -5.71875,
      "step": 11370
    },
    {
      "epoch": 0.8562184937175532,
      "grad_norm": 22.715026136737638,
      "learning_rate": 3.083607906411831e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.53125,
      "logps/chosen": -314.0,
      "logps/rejected": -656.0,
      "loss": 0.3285,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.109375,
      "rewards/margins": 3.296875,
      "rewards/rejected": -5.40625,
      "step": 11380
    },
    {
      "epoch": 0.856970882552103,
      "grad_norm": 23.764319584690128,
      "learning_rate": 3.052091747578644e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.8125,
      "logps/chosen": -330.0,
      "logps/rejected": -684.0,
      "loss": 0.3639,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.59375,
      "rewards/rejected": -5.5,
      "step": 11390
    },
    {
      "epoch": 0.8577232713866526,
      "grad_norm": 17.61120519483096,
      "learning_rate": 3.020726999823298e-08,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.6875,
      "logps/chosen": -296.0,
      "logps/rejected": -676.0,
      "loss": 0.3723,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.828125,
      "rewards/margins": 3.734375,
      "rewards/rejected": -5.5625,
      "step": 11400
    },
    {
      "epoch": 0.8584756602212024,
      "grad_norm": 23.623348539444322,
      "learning_rate": 2.989513879520394e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.5625,
      "logps/chosen": -310.0,
      "logps/rejected": -708.0,
      "loss": 0.3745,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.734375,
      "rewards/margins": 4.125,
      "rewards/rejected": -5.875,
      "step": 11410
    },
    {
      "epoch": 0.859228049055752,
      "grad_norm": 23.68921591868859,
      "learning_rate": 2.9584526019985373e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.40625,
      "logps/chosen": -292.0,
      "logps/rejected": -676.0,
      "loss": 0.3777,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.78125,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.53125,
      "step": 11420
    },
    {
      "epoch": 0.8599804378903018,
      "grad_norm": 21.249567217183706,
      "learning_rate": 2.927543381538805e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.515625,
      "logps/chosen": -302.0,
      "logps/rejected": -692.0,
      "loss": 0.3499,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.84375,
      "rewards/rejected": -5.71875,
      "step": 11430
    },
    {
      "epoch": 0.8607328267248514,
      "grad_norm": 17.90688121464688,
      "learning_rate": 2.8967864313732826e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.671875,
      "logps/chosen": -314.0,
      "logps/rejected": -692.0,
      "loss": 0.3277,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.921875,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.84375,
      "step": 11440
    },
    {
      "epoch": 0.861485215559401,
      "grad_norm": 18.912952548832116,
      "learning_rate": 2.866181963683617e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.625,
      "logps/chosen": -328.0,
      "logps/rejected": -688.0,
      "loss": 0.3678,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.015625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.8125,
      "step": 11450
    },
    {
      "epoch": 0.8622376043939508,
      "grad_norm": 20.981151709868982,
      "learning_rate": 2.8357301895994946e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.796875,
      "logps/chosen": -306.0,
      "logps/rejected": -680.0,
      "loss": 0.3279,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.921875,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.6875,
      "step": 11460
    },
    {
      "epoch": 0.8629899932285005,
      "grad_norm": 21.502579440389844,
      "learning_rate": 2.8054313191972574e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.609375,
      "logps/chosen": -320.0,
      "logps/rejected": -688.0,
      "loss": 0.341,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -5.53125,
      "step": 11470
    },
    {
      "epoch": 0.8637423820630502,
      "grad_norm": 37.11272075198176,
      "learning_rate": 2.775285561498397e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.703125,
      "logps/chosen": -324.0,
      "logps/rejected": -644.0,
      "loss": 0.3846,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.921875,
      "rewards/margins": 3.390625,
      "rewards/rejected": -5.3125,
      "step": 11480
    },
    {
      "epoch": 0.8644947708975999,
      "grad_norm": 27.082920090376742,
      "learning_rate": 2.7452931244681314e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.71875,
      "logps/chosen": -330.0,
      "logps/rejected": -696.0,
      "loss": 0.385,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.9375,
      "rewards/rejected": -5.875,
      "step": 11490
    },
    {
      "epoch": 0.8652471597321496,
      "grad_norm": 32.41038196006568,
      "learning_rate": 2.7154542150139876e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.828125,
      "logps/chosen": -320.0,
      "logps/rejected": -608.0,
      "loss": 0.4196,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.09375,
      "rewards/margins": 2.921875,
      "rewards/rejected": -5.0,
      "step": 11500
    },
    {
      "epoch": 0.8659995485666993,
      "grad_norm": 29.384049568130955,
      "learning_rate": 2.6857690389843478e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.734375,
      "logps/chosen": -308.0,
      "logps/rejected": -696.0,
      "loss": 0.3487,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 3.828125,
      "rewards/rejected": -5.6875,
      "step": 11510
    },
    {
      "epoch": 0.866751937401249,
      "grad_norm": 19.17555551357093,
      "learning_rate": 2.656237801167033e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.75,
      "logps/chosen": -310.0,
      "logps/rejected": -696.0,
      "loss": 0.3555,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.890625,
      "rewards/rejected": -5.75,
      "step": 11520
    },
    {
      "epoch": 0.8675043262357987,
      "grad_norm": 22.60303696772255,
      "learning_rate": 2.62686070528792e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.65625,
      "logps/chosen": -308.0,
      "logps/rejected": -680.0,
      "loss": 0.3353,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.609375,
      "rewards/rejected": -5.46875,
      "step": 11530
    },
    {
      "epoch": 0.8682567150703484,
      "grad_norm": 26.4183296148565,
      "learning_rate": 2.5976379540094907e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.6875,
      "logps/chosen": -318.0,
      "logps/rejected": -696.0,
      "loss": 0.3464,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.6875,
      "step": 11540
    },
    {
      "epoch": 0.869009103904898,
      "grad_norm": 21.58881081993634,
      "learning_rate": 2.5685697489294665e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.578125,
      "logps/chosen": -320.0,
      "logps/rejected": -656.0,
      "loss": 0.3372,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.78125,
      "rewards/margins": 3.484375,
      "rewards/rejected": -5.28125,
      "step": 11550
    },
    {
      "epoch": 0.8697614927394477,
      "grad_norm": 21.28988877884388,
      "learning_rate": 2.539656290579409e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.609375,
      "logps/chosen": -318.0,
      "logps/rejected": -704.0,
      "loss": 0.34,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.8984375,
      "rewards/margins": 3.9375,
      "rewards/rejected": -5.84375,
      "step": 11560
    },
    {
      "epoch": 0.8705138815739975,
      "grad_norm": 36.67418049771319,
      "learning_rate": 2.510897778423324e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.578125,
      "logps/chosen": -328.0,
      "logps/rejected": -668.0,
      "loss": 0.4176,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.109375,
      "rewards/margins": 3.46875,
      "rewards/rejected": -5.5625,
      "step": 11570
    },
    {
      "epoch": 0.8712662704085471,
      "grad_norm": 22.697785843791724,
      "learning_rate": 2.482294410856317e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.546875,
      "logps/chosen": -306.0,
      "logps/rejected": -700.0,
      "loss": 0.3534,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 4.0,
      "rewards/rejected": -5.84375,
      "step": 11580
    },
    {
      "epoch": 0.8720186592430968,
      "grad_norm": 24.05847856694563,
      "learning_rate": 2.4538463852031897e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.796875,
      "logps/chosen": -324.0,
      "logps/rejected": -660.0,
      "loss": 0.3915,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.59375,
      "step": 11590
    },
    {
      "epoch": 0.8727710480776465,
      "grad_norm": 25.92877757385005,
      "learning_rate": 2.425553897717092e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.65625,
      "logps/chosen": -320.0,
      "logps/rejected": -736.0,
      "loss": 0.3575,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8984375,
      "rewards/margins": 4.40625,
      "rewards/rejected": -6.28125,
      "step": 11600
    },
    {
      "epoch": 0.8735234369121962,
      "grad_norm": 31.477884612020762,
      "learning_rate": 2.3974171435781787e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.640625,
      "logps/chosen": -298.0,
      "logps/rejected": -660.0,
      "loss": 0.3821,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.734375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -5.53125,
      "step": 11610
    },
    {
      "epoch": 0.8742758257467459,
      "grad_norm": 20.959384936393583,
      "learning_rate": 2.3694363168922454e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.640625,
      "logps/chosen": -296.0,
      "logps/rejected": -700.0,
      "loss": 0.361,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.796875,
      "rewards/margins": 3.921875,
      "rewards/rejected": -5.71875,
      "step": 11620
    },
    {
      "epoch": 0.8750282145812956,
      "grad_norm": 25.83564481612912,
      "learning_rate": 2.3416116106894062e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.828125,
      "logps/chosen": -298.0,
      "logps/rejected": -660.0,
      "loss": 0.3521,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.65625,
      "step": 11630
    },
    {
      "epoch": 0.8757806034158453,
      "grad_norm": 19.15006208965266,
      "learning_rate": 2.3139432169227507e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.625,
      "logps/chosen": -302.0,
      "logps/rejected": -680.0,
      "loss": 0.3681,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.96875,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.71875,
      "step": 11640
    },
    {
      "epoch": 0.876532992250395,
      "grad_norm": 18.828835416718988,
      "learning_rate": 2.2864313264670058e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.578125,
      "logps/chosen": -316.0,
      "logps/rejected": -708.0,
      "loss": 0.3779,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 4.0,
      "rewards/rejected": -5.84375,
      "step": 11650
    },
    {
      "epoch": 0.8772853810849447,
      "grad_norm": 18.92647794288229,
      "learning_rate": 2.2590761291172655e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.453125,
      "logps/chosen": -310.0,
      "logps/rejected": -756.0,
      "loss": 0.3618,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 4.375,
      "rewards/rejected": -6.25,
      "step": 11660
    },
    {
      "epoch": 0.8780377699194944,
      "grad_norm": 22.590107065690248,
      "learning_rate": 2.2318778135876292e-08,
      "logits/chosen": -2.5,
      "logits/rejected": -2.484375,
      "logps/chosen": -316.0,
      "logps/rejected": -672.0,
      "loss": 0.3386,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.5,
      "step": 11670
    },
    {
      "epoch": 0.8787901587540441,
      "grad_norm": 24.55655835468251,
      "learning_rate": 2.2048365675099378e-08,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.609375,
      "logps/chosen": -318.0,
      "logps/rejected": -724.0,
      "loss": 0.4076,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.71875,
      "step": 11680
    },
    {
      "epoch": 0.8795425475885937,
      "grad_norm": 22.13522004548634,
      "learning_rate": 2.1779525774324514e-08,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.75,
      "logps/chosen": -316.0,
      "logps/rejected": -676.0,
      "loss": 0.3264,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.6875,
      "step": 11690
    },
    {
      "epoch": 0.8802949364231435,
      "grad_norm": 24.47851232334795,
      "learning_rate": 2.1512260288185786e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.46875,
      "logps/chosen": -336.0,
      "logps/rejected": -664.0,
      "loss": 0.347,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.015625,
      "rewards/margins": 3.421875,
      "rewards/rejected": -5.4375,
      "step": 11700
    },
    {
      "epoch": 0.8810473252576931,
      "grad_norm": 25.70833832795148,
      "learning_rate": 2.124657106045602e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -296.0,
      "logps/rejected": -660.0,
      "loss": 0.3418,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.625,
      "step": 11710
    },
    {
      "epoch": 0.8817997140922429,
      "grad_norm": 24.88020127418182,
      "learning_rate": 2.0982459924033857e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.609375,
      "logps/chosen": -322.0,
      "logps/rejected": -704.0,
      "loss": 0.3575,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.75,
      "step": 11720
    },
    {
      "epoch": 0.8825521029267925,
      "grad_norm": 18.402599502186053,
      "learning_rate": 2.071992870093131e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.734375,
      "logps/chosen": -320.0,
      "logps/rejected": -684.0,
      "loss": 0.3333,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.8984375,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.53125,
      "step": 11730
    },
    {
      "epoch": 0.8833044917613423,
      "grad_norm": 24.316231850314416,
      "learning_rate": 2.0458979202261057e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.625,
      "logps/chosen": -308.0,
      "logps/rejected": -692.0,
      "loss": 0.3519,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.6875,
      "step": 11740
    },
    {
      "epoch": 0.8840568805958919,
      "grad_norm": 30.72292743514893,
      "learning_rate": 2.0199613228224e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.546875,
      "logps/chosen": -322.0,
      "logps/rejected": -692.0,
      "loss": 0.3759,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.875,
      "rewards/rejected": -5.8125,
      "step": 11750
    },
    {
      "epoch": 0.8848092694304417,
      "grad_norm": 22.362605620076454,
      "learning_rate": 1.9941832568096978e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.640625,
      "logps/chosen": -306.0,
      "logps/rejected": -688.0,
      "loss": 0.3507,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.75,
      "step": 11760
    },
    {
      "epoch": 0.8855616582649913,
      "grad_norm": 31.235576869915416,
      "learning_rate": 1.9685639000220105e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.5625,
      "logps/chosen": -324.0,
      "logps/rejected": -728.0,
      "loss": 0.3798,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.015625,
      "rewards/margins": 3.890625,
      "rewards/rejected": -5.90625,
      "step": 11770
    },
    {
      "epoch": 0.8863140470995411,
      "grad_norm": 27.992110513898,
      "learning_rate": 1.9431034291984755e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.75,
      "logps/chosen": -326.0,
      "logps/rejected": -704.0,
      "loss": 0.3477,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.015625,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.9375,
      "step": 11780
    },
    {
      "epoch": 0.8870664359340907,
      "grad_norm": 30.551046019062618,
      "learning_rate": 1.9178020199821426e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.546875,
      "logps/chosen": -320.0,
      "logps/rejected": -716.0,
      "loss": 0.3859,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.890625,
      "rewards/margins": 3.875,
      "rewards/rejected": -5.78125,
      "step": 11790
    },
    {
      "epoch": 0.8878188247686404,
      "grad_norm": 35.18271020927999,
      "learning_rate": 1.892659846918737e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.609375,
      "logps/chosen": -312.0,
      "logps/rejected": -672.0,
      "loss": 0.3629,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 3.625,
      "rewards/rejected": -5.46875,
      "step": 11800
    },
    {
      "epoch": 0.8885712136031901,
      "grad_norm": 25.136976672802092,
      "learning_rate": 1.8676770834554655e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.640625,
      "logps/chosen": -324.0,
      "logps/rejected": -688.0,
      "loss": 0.3436,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.953125,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.625,
      "step": 11810
    },
    {
      "epoch": 0.8893236024377398,
      "grad_norm": 24.622200575693252,
      "learning_rate": 1.842853901939842e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.640625,
      "logps/chosen": -306.0,
      "logps/rejected": -704.0,
      "loss": 0.3469,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 4.0,
      "rewards/rejected": -5.9375,
      "step": 11820
    },
    {
      "epoch": 0.8900759912722895,
      "grad_norm": 43.734488392495535,
      "learning_rate": 1.8181904736184528e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.625,
      "logps/chosen": -342.0,
      "logps/rejected": -712.0,
      "loss": 0.3652,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.125,
      "rewards/margins": 3.875,
      "rewards/rejected": -6.0,
      "step": 11830
    },
    {
      "epoch": 0.8908283801068392,
      "grad_norm": 25.757955821483094,
      "learning_rate": 1.793686968635824e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.71875,
      "logps/chosen": -338.0,
      "logps/rejected": -664.0,
      "loss": 0.3336,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.015625,
      "rewards/margins": 3.625,
      "rewards/rejected": -5.65625,
      "step": 11840
    },
    {
      "epoch": 0.8915807689413889,
      "grad_norm": 22.732158324620375,
      "learning_rate": 1.7693435560332148e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.796875,
      "logps/chosen": -276.0,
      "logps/rejected": -640.0,
      "loss": 0.3351,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.703125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.40625,
      "step": 11850
    },
    {
      "epoch": 0.8923331577759386,
      "grad_norm": 21.073914915689173,
      "learning_rate": 1.7451604037474616e-08,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.6875,
      "logps/chosen": -312.0,
      "logps/rejected": -684.0,
      "loss": 0.3563,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.90625,
      "rewards/margins": 3.828125,
      "rewards/rejected": -5.75,
      "step": 11860
    },
    {
      "epoch": 0.8930855466104883,
      "grad_norm": 30.275258347170645,
      "learning_rate": 1.72113767860983e-08,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.53125,
      "logps/chosen": -300.0,
      "logps/rejected": -680.0,
      "loss": 0.3749,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.7734375,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.53125,
      "step": 11870
    },
    {
      "epoch": 0.893837935445038,
      "grad_norm": 23.349407315733636,
      "learning_rate": 1.6972755463448423e-08,
      "logits/chosen": -2.75,
      "logits/rejected": -2.859375,
      "logps/chosen": -340.0,
      "logps/rejected": -648.0,
      "loss": 0.3643,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.40625,
      "rewards/rejected": -5.4375,
      "step": 11880
    },
    {
      "epoch": 0.8945903242795877,
      "grad_norm": 32.32007005862833,
      "learning_rate": 1.6735741715691448e-08,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.75,
      "logps/chosen": -324.0,
      "logps/rejected": -668.0,
      "loss": 0.3913,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.5625,
      "step": 11890
    },
    {
      "epoch": 0.8953427131141374,
      "grad_norm": 26.547596275317467,
      "learning_rate": 1.650033717790389e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -304.0,
      "logps/rejected": -664.0,
      "loss": 0.3656,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.5625,
      "step": 11900
    },
    {
      "epoch": 0.8960951019486871,
      "grad_norm": 18.594351017887647,
      "learning_rate": 1.626654347406073e-08,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.578125,
      "logps/chosen": -312.0,
      "logps/rejected": -716.0,
      "loss": 0.331,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.84375,
      "rewards/rejected": -5.78125,
      "step": 11910
    },
    {
      "epoch": 0.8968474907832368,
      "grad_norm": 29.121052436616992,
      "learning_rate": 1.6034362217024396e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.59375,
      "logps/chosen": -310.0,
      "logps/rejected": -672.0,
      "loss": 0.4023,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.6875,
      "step": 11920
    },
    {
      "epoch": 0.8975998796177864,
      "grad_norm": 25.888187521159338,
      "learning_rate": 1.580379500853357e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.796875,
      "logps/chosen": -320.0,
      "logps/rejected": -696.0,
      "loss": 0.3501,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8984375,
      "rewards/margins": 3.921875,
      "rewards/rejected": -5.8125,
      "step": 11930
    },
    {
      "epoch": 0.8983522684523362,
      "grad_norm": 24.340115937794785,
      "learning_rate": 1.5574843439192213e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.609375,
      "logps/chosen": -286.0,
      "logps/rejected": -628.0,
      "loss": 0.3479,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.34375,
      "step": 11940
    },
    {
      "epoch": 0.8991046572868858,
      "grad_norm": 18.228951873647365,
      "learning_rate": 1.534750908845858e-08,
      "logits/chosen": -2.5,
      "logits/rejected": -2.46875,
      "logps/chosen": -304.0,
      "logps/rejected": -692.0,
      "loss": 0.3598,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 3.921875,
      "rewards/rejected": -5.75,
      "step": 11950
    },
    {
      "epoch": 0.8998570461214356,
      "grad_norm": 21.314943714183745,
      "learning_rate": 1.512179352463419e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.71875,
      "logps/chosen": -296.0,
      "logps/rejected": -648.0,
      "loss": 0.3195,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.3125,
      "step": 11960
    },
    {
      "epoch": 0.9006094349559852,
      "grad_norm": 29.57792054431613,
      "learning_rate": 1.4897698304853213e-08,
      "logits/chosen": -2.5,
      "logits/rejected": -2.625,
      "logps/chosen": -342.0,
      "logps/rejected": -736.0,
      "loss": 0.3656,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.09375,
      "rewards/margins": 4.0,
      "rewards/rejected": -6.09375,
      "step": 11970
    },
    {
      "epoch": 0.901361823790535,
      "grad_norm": 20.163310508043946,
      "learning_rate": 1.4675224975071565e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.65625,
      "logps/chosen": -286.0,
      "logps/rejected": -692.0,
      "loss": 0.3735,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 3.984375,
      "rewards/rejected": -5.78125,
      "step": 11980
    },
    {
      "epoch": 0.9021142126250846,
      "grad_norm": 33.68974936855441,
      "learning_rate": 1.445437507005623e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.515625,
      "logps/chosen": -308.0,
      "logps/rejected": -688.0,
      "loss": 0.4011,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.625,
      "step": 11990
    },
    {
      "epoch": 0.9028666014596344,
      "grad_norm": 23.772750482804952,
      "learning_rate": 1.4235150113374977e-08,
      "logits/chosen": -2.5,
      "logits/rejected": -2.859375,
      "logps/chosen": -300.0,
      "logps/rejected": -680.0,
      "loss": 0.3601,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.96875,
      "rewards/margins": 3.96875,
      "rewards/rejected": -5.9375,
      "step": 12000
    },
    {
      "epoch": 0.9028666014596344,
      "eval_logits/chosen": -2.5625,
      "eval_logits/rejected": -2.609375,
      "eval_logps/chosen": -340.0,
      "eval_logps/rejected": -668.0,
      "eval_loss": 0.2619945704936981,
      "eval_rewards/accuracies": 0.8853249549865723,
      "eval_rewards/chosen": -2.078125,
      "eval_rewards/margins": 3.4375,
      "eval_rewards/rejected": -5.53125,
      "eval_runtime": 2667.4894,
      "eval_samples_per_second": 35.424,
      "eval_steps_per_second": 0.554,
      "step": 12000
    },
    {
      "epoch": 0.903618990294184,
      "grad_norm": 22.699512749463675,
      "learning_rate": 1.4017551617385298e-08,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.609375,
      "logps/chosen": -322.0,
      "logps/rejected": -708.0,
      "loss": 0.341,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.953125,
      "rewards/margins": 3.9375,
      "rewards/rejected": -5.875,
      "step": 12010
    },
    {
      "epoch": 0.9043713791287338,
      "grad_norm": 27.269218205069592,
      "learning_rate": 1.3801581083224544e-08,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.734375,
      "logps/chosen": -336.0,
      "logps/rejected": -668.0,
      "loss": 0.3587,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.015625,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.5625,
      "step": 12020
    },
    {
      "epoch": 0.9051237679632834,
      "grad_norm": 23.37277573245976,
      "learning_rate": 1.3587240000799166e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.59375,
      "logps/chosen": -324.0,
      "logps/rejected": -700.0,
      "loss": 0.3484,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.046875,
      "rewards/margins": 3.796875,
      "rewards/rejected": -5.84375,
      "step": 12030
    },
    {
      "epoch": 0.9058761567978331,
      "grad_norm": 22.925405335233194,
      "learning_rate": 1.3374529848774685e-08,
      "logits/chosen": -2.5,
      "logits/rejected": -2.609375,
      "logps/chosen": -320.0,
      "logps/rejected": -688.0,
      "loss": 0.3653,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.984375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.65625,
      "step": 12040
    },
    {
      "epoch": 0.9066285456323828,
      "grad_norm": 27.033712999079256,
      "learning_rate": 1.3163452094565348e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.671875,
      "logps/chosen": -366.0,
      "logps/rejected": -680.0,
      "loss": 0.3362,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.265625,
      "rewards/margins": 3.265625,
      "rewards/rejected": -5.53125,
      "step": 12050
    },
    {
      "epoch": 0.9073809344669325,
      "grad_norm": 22.15632420223244,
      "learning_rate": 1.2954008194324046e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.6875,
      "logps/chosen": -300.0,
      "logps/rejected": -744.0,
      "loss": 0.3447,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.828125,
      "rewards/margins": 4.59375,
      "rewards/rejected": -6.40625,
      "step": 12060
    },
    {
      "epoch": 0.9081333233014822,
      "grad_norm": 19.48245565865233,
      "learning_rate": 1.2746199592932272e-08,
      "logits/chosen": -2.625,
      "logits/rejected": -2.65625,
      "logps/chosen": -330.0,
      "logps/rejected": -728.0,
      "loss": 0.3693,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.09375,
      "rewards/margins": 3.96875,
      "rewards/rejected": -6.0625,
      "step": 12070
    },
    {
      "epoch": 0.9088857121360319,
      "grad_norm": 23.40689896583894,
      "learning_rate": 1.254002772399021e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.65625,
      "logps/chosen": -280.0,
      "logps/rejected": -644.0,
      "loss": 0.3889,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.6875,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.34375,
      "step": 12080
    },
    {
      "epoch": 0.9096381009705816,
      "grad_norm": 32.29778192273769,
      "learning_rate": 1.2335494009806712e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.6875,
      "logps/chosen": -346.0,
      "logps/rejected": -660.0,
      "loss": 0.3993,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.109375,
      "rewards/margins": 3.46875,
      "rewards/rejected": -5.5625,
      "step": 12090
    },
    {
      "epoch": 0.9103904898051313,
      "grad_norm": 16.583900666666253,
      "learning_rate": 1.2132599861389591e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.6875,
      "logps/chosen": -326.0,
      "logps/rejected": -688.0,
      "loss": 0.3562,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.59375,
      "step": 12100
    },
    {
      "epoch": 0.911142878639681,
      "grad_norm": 16.71509389908307,
      "learning_rate": 1.1931346678435872e-08,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.546875,
      "logps/chosen": -302.0,
      "logps/rejected": -716.0,
      "loss": 0.331,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 4.125,
      "rewards/rejected": -5.90625,
      "step": 12110
    },
    {
      "epoch": 0.9118952674742307,
      "grad_norm": 23.235361729945627,
      "learning_rate": 1.1731735849322077e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.53125,
      "logps/chosen": -318.0,
      "logps/rejected": -648.0,
      "loss": 0.3398,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.84375,
      "rewards/margins": 3.484375,
      "rewards/rejected": -5.34375,
      "step": 12120
    },
    {
      "epoch": 0.9126476563087804,
      "grad_norm": 36.87451872691728,
      "learning_rate": 1.1533768751094798e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.59375,
      "logps/chosen": -340.0,
      "logps/rejected": -688.0,
      "loss": 0.3686,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.609375,
      "rewards/rejected": -5.65625,
      "step": 12130
    },
    {
      "epoch": 0.9134000451433301,
      "grad_norm": 22.799141452042598,
      "learning_rate": 1.1337446749461021e-08,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.484375,
      "logps/chosen": -324.0,
      "logps/rejected": -712.0,
      "loss": 0.3315,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.828125,
      "rewards/rejected": -5.75,
      "step": 12140
    },
    {
      "epoch": 0.9141524339778798,
      "grad_norm": 25.715988782480224,
      "learning_rate": 1.1142771198778683e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.71875,
      "logps/chosen": -316.0,
      "logps/rejected": -680.0,
      "loss": 0.3935,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.609375,
      "rewards/rejected": -5.625,
      "step": 12150
    },
    {
      "epoch": 0.9149048228124295,
      "grad_norm": 21.055397612690406,
      "learning_rate": 1.0949743442047632e-08,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.453125,
      "logps/chosen": -314.0,
      "logps/rejected": -728.0,
      "loss": 0.3485,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8125,
      "rewards/margins": 4.1875,
      "rewards/rejected": -6.0,
      "step": 12160
    },
    {
      "epoch": 0.9156572116469791,
      "grad_norm": 20.37758452320395,
      "learning_rate": 1.0758364810899977e-08,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.59375,
      "logps/chosen": -310.0,
      "logps/rejected": -672.0,
      "loss": 0.4025,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.515625,
      "rewards/rejected": -5.5625,
      "step": 12170
    },
    {
      "epoch": 0.9164096004815289,
      "grad_norm": 19.96897872808924,
      "learning_rate": 1.056863662559121e-08,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.3125,
      "logps/chosen": -334.0,
      "logps/rejected": -756.0,
      "loss": 0.3751,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.03125,
      "rewards/margins": 4.28125,
      "rewards/rejected": -6.3125,
      "step": 12180
    },
    {
      "epoch": 0.9171619893160785,
      "grad_norm": 31.309333816955274,
      "learning_rate": 1.0380560194990784e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.609375,
      "logps/chosen": -330.0,
      "logps/rejected": -680.0,
      "loss": 0.3601,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.9296875,
      "rewards/margins": 3.84375,
      "rewards/rejected": -5.78125,
      "step": 12190
    },
    {
      "epoch": 0.9179143781506283,
      "grad_norm": 24.794690101818603,
      "learning_rate": 1.0194136816573411e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.59375,
      "logps/chosen": -348.0,
      "logps/rejected": -676.0,
      "loss": 0.3732,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.125,
      "rewards/margins": 3.5,
      "rewards/rejected": -5.625,
      "step": 12200
    },
    {
      "epoch": 0.9186667669851779,
      "grad_norm": 42.263970471899945,
      "learning_rate": 1.0009367776409982e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.65625,
      "logps/chosen": -320.0,
      "logps/rejected": -716.0,
      "loss": 0.3648,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 3.921875,
      "rewards/rejected": -5.90625,
      "step": 12210
    },
    {
      "epoch": 0.9194191558197277,
      "grad_norm": 27.181947025826418,
      "learning_rate": 9.826254349158513e-09,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.59375,
      "logps/chosen": -314.0,
      "logps/rejected": -688.0,
      "loss": 0.3612,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.953125,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.75,
      "step": 12220
    },
    {
      "epoch": 0.9201715446542773,
      "grad_norm": 48.68693717221615,
      "learning_rate": 9.644797798055743e-09,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.640625,
      "logps/chosen": -308.0,
      "logps/rejected": -680.0,
      "loss": 0.3794,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.625,
      "step": 12230
    },
    {
      "epoch": 0.9209239334888271,
      "grad_norm": 19.614890362295867,
      "learning_rate": 9.464999374907994e-09,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.6875,
      "logps/chosen": -262.0,
      "logps/rejected": -724.0,
      "loss": 0.363,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.640625,
      "rewards/margins": 4.46875,
      "rewards/rejected": -6.125,
      "step": 12240
    },
    {
      "epoch": 0.9216763223233767,
      "grad_norm": 30.75318245042768,
      "learning_rate": 9.286860320082801e-09,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.625,
      "logps/chosen": -320.0,
      "logps/rejected": -688.0,
      "loss": 0.3567,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.71875,
      "step": 12250
    },
    {
      "epoch": 0.9224287111579265,
      "grad_norm": 23.52749098094884,
      "learning_rate": 9.110381862500349e-09,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.59375,
      "logps/chosen": -320.0,
      "logps/rejected": -720.0,
      "loss": 0.325,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.921875,
      "rewards/margins": 3.84375,
      "rewards/rejected": -5.75,
      "step": 12260
    },
    {
      "epoch": 0.9231810999924761,
      "grad_norm": 22.214400400112588,
      "learning_rate": 8.935565219624852e-09,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.515625,
      "logps/chosen": -308.0,
      "logps/rejected": -740.0,
      "loss": 0.3485,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 4.375,
      "rewards/rejected": -6.1875,
      "step": 12270
    },
    {
      "epoch": 0.9239334888270259,
      "grad_norm": 34.741564967209854,
      "learning_rate": 8.762411597456249e-09,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.53125,
      "logps/chosen": -326.0,
      "logps/rejected": -724.0,
      "loss": 0.3987,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.046875,
      "rewards/margins": 3.921875,
      "rewards/rejected": -5.96875,
      "step": 12280
    },
    {
      "epoch": 0.9246858776615755,
      "grad_norm": 26.65182761351099,
      "learning_rate": 8.590922190521904e-09,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.5,
      "logps/chosen": -322.0,
      "logps/rejected": -712.0,
      "loss": 0.3833,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -6.03125,
      "step": 12290
    },
    {
      "epoch": 0.9254382664961252,
      "grad_norm": 21.62392941432359,
      "learning_rate": 8.421098181868285e-09,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.625,
      "logps/chosen": -300.0,
      "logps/rejected": -640.0,
      "loss": 0.3485,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.734375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -5.34375,
      "step": 12300
    },
    {
      "epoch": 0.9261906553306749,
      "grad_norm": 21.375329188639185,
      "learning_rate": 8.25294074305291e-09,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.671875,
      "logps/chosen": -298.0,
      "logps/rejected": -716.0,
      "loss": 0.3318,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.90625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -6.09375,
      "step": 12310
    },
    {
      "epoch": 0.9269430441652246,
      "grad_norm": 17.38831272813536,
      "learning_rate": 8.086451034136187e-09,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.484375,
      "logps/chosen": -300.0,
      "logps/rejected": -688.0,
      "loss": 0.347,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 3.859375,
      "rewards/rejected": -5.65625,
      "step": 12320
    },
    {
      "epoch": 0.9276954329997743,
      "grad_norm": 22.86063141971569,
      "learning_rate": 7.921630203673341e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.859375,
      "logps/chosen": -298.0,
      "logps/rejected": -664.0,
      "loss": 0.3456,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.703125,
      "rewards/margins": 3.890625,
      "rewards/rejected": -5.59375,
      "step": 12330
    },
    {
      "epoch": 0.928447821834324,
      "grad_norm": 29.99806138814956,
      "learning_rate": 7.758479388706718e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.59375,
      "logps/chosen": -320.0,
      "logps/rejected": -728.0,
      "loss": 0.3834,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.8984375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -5.9375,
      "step": 12340
    },
    {
      "epoch": 0.9292002106688737,
      "grad_norm": 24.499085567161174,
      "learning_rate": 7.596999714757718e-09,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.75,
      "logps/chosen": -312.0,
      "logps/rejected": -696.0,
      "loss": 0.3644,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.78125,
      "rewards/margins": 4.03125,
      "rewards/rejected": -5.8125,
      "step": 12350
    },
    {
      "epoch": 0.9299525995034233,
      "grad_norm": 21.32506934978771,
      "learning_rate": 7.4371922958191e-09,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.296875,
      "logps/chosen": -318.0,
      "logps/rejected": -700.0,
      "loss": 0.3184,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.9609375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.84375,
      "step": 12360
    },
    {
      "epoch": 0.9307049883379731,
      "grad_norm": 19.477373090790746,
      "learning_rate": 7.279058234347352e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.65625,
      "logps/chosen": -288.0,
      "logps/rejected": -712.0,
      "loss": 0.3397,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.5625,
      "rewards/margins": 4.28125,
      "rewards/rejected": -5.84375,
      "step": 12370
    },
    {
      "epoch": 0.9314573771725227,
      "grad_norm": 20.34766442106942,
      "learning_rate": 7.122598621255027e-09,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.65625,
      "logps/chosen": -340.0,
      "logps/rejected": -668.0,
      "loss": 0.3613,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.125,
      "rewards/margins": 3.453125,
      "rewards/rejected": -5.5625,
      "step": 12380
    },
    {
      "epoch": 0.9322097660070725,
      "grad_norm": 27.398099352826783,
      "learning_rate": 6.967814535903283e-09,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.484375,
      "logps/chosen": -328.0,
      "logps/rejected": -680.0,
      "loss": 0.3739,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.890625,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.5625,
      "step": 12390
    },
    {
      "epoch": 0.9329621548416221,
      "grad_norm": 29.642005180568205,
      "learning_rate": 6.81470704609427e-09,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.625,
      "logps/chosen": -314.0,
      "logps/rejected": -740.0,
      "loss": 0.3507,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.84375,
      "rewards/margins": 4.40625,
      "rewards/rejected": -6.25,
      "step": 12400
    },
    {
      "epoch": 0.9337145436761718,
      "grad_norm": 22.557545904679905,
      "learning_rate": 6.6632772080639775e-09,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.53125,
      "logps/chosen": -328.0,
      "logps/rejected": -688.0,
      "loss": 0.3387,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.984375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -5.78125,
      "step": 12410
    },
    {
      "epoch": 0.9344669325107215,
      "grad_norm": 32.13314469590184,
      "learning_rate": 6.513526066474873e-09,
      "logits/chosen": -2.5,
      "logits/rejected": -2.4375,
      "logps/chosen": -320.0,
      "logps/rejected": -684.0,
      "loss": 0.3423,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.65625,
      "step": 12420
    },
    {
      "epoch": 0.9352193213452712,
      "grad_norm": 27.410781399352633,
      "learning_rate": 6.365454654408547e-09,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.625,
      "logps/chosen": -326.0,
      "logps/rejected": -700.0,
      "loss": 0.3488,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.078125,
      "rewards/margins": 3.875,
      "rewards/rejected": -5.9375,
      "step": 12430
    },
    {
      "epoch": 0.935971710179821,
      "grad_norm": 29.14305435806815,
      "learning_rate": 6.219063993358864e-09,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.4375,
      "logps/chosen": -348.0,
      "logps/rejected": -728.0,
      "loss": 0.381,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.046875,
      "rewards/margins": 3.890625,
      "rewards/rejected": -5.9375,
      "step": 12440
    },
    {
      "epoch": 0.9367240990143706,
      "grad_norm": 24.279970567801747,
      "learning_rate": 6.0743550932247086e-09,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.625,
      "logps/chosen": -326.0,
      "logps/rejected": -676.0,
      "loss": 0.348,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.0625,
      "rewards/margins": 3.65625,
      "rewards/rejected": -5.71875,
      "step": 12450
    },
    {
      "epoch": 0.9374764878489203,
      "grad_norm": 23.72460599020982,
      "learning_rate": 5.931328952302972e-09,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.609375,
      "logps/chosen": -342.0,
      "logps/rejected": -692.0,
      "loss": 0.3726,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.125,
      "rewards/margins": 3.546875,
      "rewards/rejected": -5.65625,
      "step": 12460
    },
    {
      "epoch": 0.93822887668347,
      "grad_norm": 29.306263181689005,
      "learning_rate": 5.7899865572819116e-09,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.703125,
      "logps/chosen": -310.0,
      "logps/rejected": -696.0,
      "loss": 0.3581,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.84375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.75,
      "step": 12470
    },
    {
      "epoch": 0.9389812655180197,
      "grad_norm": 25.4349601464968,
      "learning_rate": 5.650328883234134e-09,
      "logits/chosen": -2.5,
      "logits/rejected": -2.625,
      "logps/chosen": -294.0,
      "logps/rejected": -672.0,
      "loss": 0.3751,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.71875,
      "step": 12480
    },
    {
      "epoch": 0.9397336543525694,
      "grad_norm": 18.302411443999794,
      "learning_rate": 5.512356893609987e-09,
      "logits/chosen": -2.5,
      "logits/rejected": -2.5,
      "logps/chosen": -328.0,
      "logps/rejected": -728.0,
      "loss": 0.332,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.984375,
      "rewards/rejected": -5.9375,
      "step": 12490
    },
    {
      "epoch": 0.9404860431871191,
      "grad_norm": 23.065452999378785,
      "learning_rate": 5.376071540230787e-09,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.734375,
      "logps/chosen": -322.0,
      "logps/rejected": -684.0,
      "loss": 0.3623,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.0625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.78125,
      "step": 12500
    },
    {
      "epoch": 0.9412384320216688,
      "grad_norm": 19.97772019486917,
      "learning_rate": 5.24147376328235e-09,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.5,
      "logps/chosen": -284.0,
      "logps/rejected": -664.0,
      "loss": 0.3589,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.625,
      "rewards/margins": 3.8125,
      "rewards/rejected": -5.4375,
      "step": 12510
    },
    {
      "epoch": 0.9419908208562185,
      "grad_norm": 27.112416506437828,
      "learning_rate": 5.108564491308504e-09,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.484375,
      "logps/chosen": -306.0,
      "logps/rejected": -672.0,
      "loss": 0.3545,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8125,
      "rewards/margins": 3.71875,
      "rewards/rejected": -5.53125,
      "step": 12520
    },
    {
      "epoch": 0.9427432096907682,
      "grad_norm": 26.866455788581803,
      "learning_rate": 4.9773446412046675e-09,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.390625,
      "logps/chosen": -332.0,
      "logps/rejected": -756.0,
      "loss": 0.339,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.984375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -6.09375,
      "step": 12530
    },
    {
      "epoch": 0.9434955985253178,
      "grad_norm": 22.05553934879081,
      "learning_rate": 4.8478151182114735e-09,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.59375,
      "logps/chosen": -326.0,
      "logps/rejected": -692.0,
      "loss": 0.392,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.953125,
      "rewards/rejected": -5.75,
      "step": 12540
    },
    {
      "epoch": 0.9442479873598676,
      "grad_norm": 21.332890847048002,
      "learning_rate": 4.719976815908605e-09,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.6875,
      "logps/chosen": -336.0,
      "logps/rejected": -756.0,
      "loss": 0.404,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.09375,
      "rewards/margins": 4.3125,
      "rewards/rejected": -6.375,
      "step": 12550
    },
    {
      "epoch": 0.9450003761944172,
      "grad_norm": 31.00291427137929,
      "learning_rate": 4.59383061620855e-09,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.640625,
      "logps/chosen": -326.0,
      "logps/rejected": -720.0,
      "loss": 0.3277,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 4.0,
      "rewards/rejected": -5.90625,
      "step": 12560
    },
    {
      "epoch": 0.945752765028967,
      "grad_norm": 21.240382543671757,
      "learning_rate": 4.469377389350659e-09,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.75,
      "logps/chosen": -312.0,
      "logps/rejected": -708.0,
      "loss": 0.3212,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.734375,
      "rewards/margins": 4.125,
      "rewards/rejected": -5.875,
      "step": 12570
    },
    {
      "epoch": 0.9465051538635166,
      "grad_norm": 18.81336449614157,
      "learning_rate": 4.3466179938949635e-09,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.734375,
      "logps/chosen": -282.0,
      "logps/rejected": -656.0,
      "loss": 0.3648,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6953125,
      "rewards/margins": 3.84375,
      "rewards/rejected": -5.53125,
      "step": 12580
    },
    {
      "epoch": 0.9472575426980664,
      "grad_norm": 20.20319751686483,
      "learning_rate": 4.225553276716309e-09,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.53125,
      "logps/chosen": -314.0,
      "logps/rejected": -716.0,
      "loss": 0.3811,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8359375,
      "rewards/margins": 4.125,
      "rewards/rejected": -5.96875,
      "step": 12590
    },
    {
      "epoch": 0.948009931532616,
      "grad_norm": 24.808275715473727,
      "learning_rate": 4.106184072998647e-09,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.78125,
      "logps/chosen": -324.0,
      "logps/rejected": -684.0,
      "loss": 0.3488,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.046875,
      "rewards/margins": 3.796875,
      "rewards/rejected": -5.84375,
      "step": 12600
    },
    {
      "epoch": 0.9487623203671658,
      "grad_norm": 27.02312692444449,
      "learning_rate": 3.988511206229062e-09,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.734375,
      "logps/chosen": -316.0,
      "logps/rejected": -672.0,
      "loss": 0.369,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 3.59375,
      "rewards/rejected": -5.59375,
      "step": 12610
    },
    {
      "epoch": 0.9495147092017154,
      "grad_norm": 22.289796112872803,
      "learning_rate": 3.872535488192247e-09,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.515625,
      "logps/chosen": -326.0,
      "logps/rejected": -748.0,
      "loss": 0.345,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 4.34375,
      "rewards/rejected": -6.21875,
      "step": 12620
    },
    {
      "epoch": 0.9502670980362652,
      "grad_norm": 37.610708660615074,
      "learning_rate": 3.7582577189648194e-09,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.46875,
      "logps/chosen": -322.0,
      "logps/rejected": -652.0,
      "loss": 0.3596,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.09375,
      "rewards/margins": 3.390625,
      "rewards/rejected": -5.46875,
      "step": 12630
    },
    {
      "epoch": 0.9510194868708148,
      "grad_norm": 28.76079618475074,
      "learning_rate": 3.64567868690982e-09,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.484375,
      "logps/chosen": -350.0,
      "logps/rejected": -716.0,
      "loss": 0.381,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.15625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.9375,
      "step": 12640
    },
    {
      "epoch": 0.9517718757053645,
      "grad_norm": 29.262725023793912,
      "learning_rate": 3.53479916867136e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.8125,
      "logps/chosen": -298.0,
      "logps/rejected": -676.0,
      "loss": 0.3535,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.875,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.65625,
      "step": 12650
    },
    {
      "epoch": 0.9525242645399142,
      "grad_norm": 33.380625511289985,
      "learning_rate": 3.4256199291691214e-09,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.5625,
      "logps/chosen": -330.0,
      "logps/rejected": -728.0,
      "loss": 0.3739,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.015625,
      "rewards/margins": 4.0,
      "rewards/rejected": -6.03125,
      "step": 12660
    },
    {
      "epoch": 0.9532766533744639,
      "grad_norm": 16.909806433318877,
      "learning_rate": 3.318141721593143e-09,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.75,
      "logps/chosen": -300.0,
      "logps/rejected": -664.0,
      "loss": 0.3605,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.734375,
      "rewards/rejected": -5.59375,
      "step": 12670
    },
    {
      "epoch": 0.9540290422090136,
      "grad_norm": 24.29394336033683,
      "learning_rate": 3.212365287398655e-09,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.421875,
      "logps/chosen": -332.0,
      "logps/rejected": -696.0,
      "loss": 0.409,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.109375,
      "rewards/margins": 3.53125,
      "rewards/rejected": -5.65625,
      "step": 12680
    },
    {
      "epoch": 0.9547814310435633,
      "grad_norm": 22.056351030023716,
      "learning_rate": 3.1082913563009737e-09,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.46875,
      "logps/chosen": -308.0,
      "logps/rejected": -752.0,
      "loss": 0.3431,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 4.28125,
      "rewards/rejected": -6.0625,
      "step": 12690
    },
    {
      "epoch": 0.955533819878113,
      "grad_norm": 23.12266334233145,
      "learning_rate": 3.00592064627031e-09,
      "logits/chosen": -2.625,
      "logits/rejected": -2.640625,
      "logps/chosen": -306.0,
      "logps/rejected": -712.0,
      "loss": 0.4033,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -5.96875,
      "step": 12700
    },
    {
      "epoch": 0.9562862087126627,
      "grad_norm": 27.70057774508697,
      "learning_rate": 2.905253863527107e-09,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.734375,
      "logps/chosen": -310.0,
      "logps/rejected": -688.0,
      "loss": 0.3347,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.859375,
      "rewards/rejected": -5.78125,
      "step": 12710
    },
    {
      "epoch": 0.9570385975472124,
      "grad_norm": 27.494477275098667,
      "learning_rate": 2.8062917025368504e-09,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.796875,
      "logps/chosen": -310.0,
      "logps/rejected": -672.0,
      "loss": 0.3724,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.890625,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.59375,
      "step": 12720
    },
    {
      "epoch": 0.9577909863817621,
      "grad_norm": 22.18634161223623,
      "learning_rate": 2.7090348460055146e-09,
      "logits/chosen": -2.5,
      "logits/rejected": -2.59375,
      "logps/chosen": -324.0,
      "logps/rejected": -712.0,
      "loss": 0.4104,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.9140625,
      "rewards/margins": 3.890625,
      "rewards/rejected": -5.8125,
      "step": 12730
    },
    {
      "epoch": 0.9585433752163118,
      "grad_norm": 18.065848999212665,
      "learning_rate": 2.6134839648747075e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.84375,
      "logps/chosen": -318.0,
      "logps/rejected": -628.0,
      "loss": 0.3629,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.203125,
      "rewards/rejected": -5.125,
      "step": 12740
    },
    {
      "epoch": 0.9592957640508615,
      "grad_norm": 27.442825141747267,
      "learning_rate": 2.519639718317146e-09,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.359375,
      "logps/chosen": -330.0,
      "logps/rejected": -760.0,
      "loss": 0.3459,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -6.0625,
      "step": 12750
    },
    {
      "epoch": 0.9600481528854112,
      "grad_norm": 24.154633514674497,
      "learning_rate": 2.4275027537320745e-09,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.375,
      "logps/chosen": -320.0,
      "logps/rejected": -692.0,
      "loss": 0.376,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 3.828125,
      "rewards/rejected": -5.8125,
      "step": 12760
    },
    {
      "epoch": 0.9608005417199609,
      "grad_norm": 27.104381894734143,
      "learning_rate": 2.3370737067406886e-09,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.640625,
      "logps/chosen": -298.0,
      "logps/rejected": -656.0,
      "loss": 0.3362,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.765625,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.4375,
      "step": 12770
    },
    {
      "epoch": 0.9615529305545105,
      "grad_norm": 19.024779138307068,
      "learning_rate": 2.2483532011819408e-09,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.4375,
      "logps/chosen": -288.0,
      "logps/rejected": -716.0,
      "loss": 0.347,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 4.3125,
      "rewards/rejected": -5.90625,
      "step": 12780
    },
    {
      "epoch": 0.9623053193890603,
      "grad_norm": 29.90465276929567,
      "learning_rate": 2.161341849108156e-09,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.625,
      "logps/chosen": -316.0,
      "logps/rejected": -672.0,
      "loss": 0.3327,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.96875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -5.6875,
      "step": 12790
    },
    {
      "epoch": 0.9630577082236099,
      "grad_norm": 21.59600335226048,
      "learning_rate": 2.076040250780675e-09,
      "logits/chosen": -2.5,
      "logits/rejected": -2.734375,
      "logps/chosen": -306.0,
      "logps/rejected": -628.0,
      "loss": 0.3238,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.78125,
      "rewards/margins": 3.453125,
      "rewards/rejected": -5.21875,
      "step": 12800
    },
    {
      "epoch": 0.9638100970581597,
      "grad_norm": 20.21741001673008,
      "learning_rate": 1.9924489946659963e-09,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.609375,
      "logps/chosen": -316.0,
      "logps/rejected": -696.0,
      "loss": 0.3356,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.875,
      "step": 12810
    },
    {
      "epoch": 0.9645624858927093,
      "grad_norm": 22.617364591243316,
      "learning_rate": 1.910568657431416e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.578125,
      "logps/chosen": -316.0,
      "logps/rejected": -704.0,
      "loss": 0.408,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -6.0625,
      "step": 12820
    },
    {
      "epoch": 0.9653148747272591,
      "grad_norm": 26.2975014648195,
      "learning_rate": 1.830399803941285e-09,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.546875,
      "logps/chosen": -304.0,
      "logps/rejected": -668.0,
      "loss": 0.3495,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.65625,
      "step": 12830
    },
    {
      "epoch": 0.9660672635618087,
      "grad_norm": 20.06397599973853,
      "learning_rate": 1.7519429872529523e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.546875,
      "logps/chosen": -286.0,
      "logps/rejected": -660.0,
      "loss": 0.3495,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.53125,
      "step": 12840
    },
    {
      "epoch": 0.9668196523963585,
      "grad_norm": 15.583529612419216,
      "learning_rate": 1.6751987486130493e-09,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.453125,
      "logps/chosen": -308.0,
      "logps/rejected": -736.0,
      "loss": 0.3316,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.921875,
      "rewards/margins": 4.21875,
      "rewards/rejected": -6.125,
      "step": 12850
    },
    {
      "epoch": 0.9675720412309081,
      "grad_norm": 26.373138712162344,
      "learning_rate": 1.6001676174537127e-09,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.578125,
      "logps/chosen": -290.0,
      "logps/rejected": -668.0,
      "loss": 0.3526,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.8515625,
      "rewards/margins": 3.625,
      "rewards/rejected": -5.46875,
      "step": 12860
    },
    {
      "epoch": 0.9683244300654579,
      "grad_norm": 25.88111126648185,
      "learning_rate": 1.526850111388922e-09,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.53125,
      "logps/chosen": -316.0,
      "logps/rejected": -680.0,
      "loss": 0.3939,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -5.59375,
      "step": 12870
    },
    {
      "epoch": 0.9690768189000075,
      "grad_norm": 16.754438529489313,
      "learning_rate": 1.4552467362109744e-09,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.53125,
      "logps/chosen": -322.0,
      "logps/rejected": -704.0,
      "loss": 0.3516,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.90625,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.8125,
      "step": 12880
    },
    {
      "epoch": 0.9698292077345572,
      "grad_norm": 30.774400327692994,
      "learning_rate": 1.3853579858869313e-09,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.625,
      "logps/chosen": -326.0,
      "logps/rejected": -672.0,
      "loss": 0.3732,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9765625,
      "rewards/margins": 3.578125,
      "rewards/rejected": -5.5625,
      "step": 12890
    },
    {
      "epoch": 0.9705815965691069,
      "grad_norm": 26.985113572223693,
      "learning_rate": 1.3171843425553163e-09,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.796875,
      "logps/chosen": -308.0,
      "logps/rejected": -712.0,
      "loss": 0.3662,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.984375,
      "rewards/margins": 4.125,
      "rewards/rejected": -6.09375,
      "step": 12900
    },
    {
      "epoch": 0.9713339854036566,
      "grad_norm": 25.866977400309313,
      "learning_rate": 1.250726276522618e-09,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.578125,
      "logps/chosen": -316.0,
      "logps/rejected": -680.0,
      "loss": 0.3703,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8046875,
      "rewards/margins": 3.96875,
      "rewards/rejected": -5.78125,
      "step": 12910
    },
    {
      "epoch": 0.9720863742382063,
      "grad_norm": 33.78232747614065,
      "learning_rate": 1.1859842462602077e-09,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -340.0,
      "logps/rejected": -680.0,
      "loss": 0.368,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.125,
      "rewards/margins": 3.46875,
      "rewards/rejected": -5.59375,
      "step": 12920
    },
    {
      "epoch": 0.972838763072756,
      "grad_norm": 26.061195415442867,
      "learning_rate": 1.1229586984011496e-09,
      "logits/chosen": -2.3125,
      "logits/rejected": -2.625,
      "logps/chosen": -356.0,
      "logps/rejected": -668.0,
      "loss": 0.3524,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.109375,
      "rewards/margins": 3.46875,
      "rewards/rejected": -5.5625,
      "step": 12930
    },
    {
      "epoch": 0.9735911519073057,
      "grad_norm": 25.569284383658225,
      "learning_rate": 1.0616500677369799e-09,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.390625,
      "logps/chosen": -332.0,
      "logps/rejected": -732.0,
      "loss": 0.3279,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.046875,
      "rewards/margins": 3.9375,
      "rewards/rejected": -6.0,
      "step": 12940
    },
    {
      "epoch": 0.9743435407418554,
      "grad_norm": 21.911010935086335,
      "learning_rate": 1.0020587772149314e-09,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.71875,
      "logps/chosen": -302.0,
      "logps/rejected": -656.0,
      "loss": 0.3789,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 3.515625,
      "rewards/rejected": -5.40625,
      "step": 12950
    },
    {
      "epoch": 0.9750959295764051,
      "grad_norm": 26.17127053324939,
      "learning_rate": 9.441852379347969e-10,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.53125,
      "logps/chosen": -322.0,
      "logps/rejected": -720.0,
      "loss": 0.3041,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.890625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -5.9375,
      "step": 12960
    },
    {
      "epoch": 0.9758483184109548,
      "grad_norm": 25.579419369835957,
      "learning_rate": 8.880298491462934e-10,
      "logits/chosen": -2.5,
      "logits/rejected": -2.4375,
      "logps/chosen": -336.0,
      "logps/rejected": -708.0,
      "loss": 0.3501,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.09375,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.84375,
      "step": 12970
    },
    {
      "epoch": 0.9766007072455045,
      "grad_norm": 27.954700354151097,
      "learning_rate": 8.335929982460909e-10,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.71875,
      "logps/chosen": -306.0,
      "logps/rejected": -648.0,
      "loss": 0.3364,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.71875,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.40625,
      "step": 12980
    },
    {
      "epoch": 0.9773530960800542,
      "grad_norm": 28.91623288605852,
      "learning_rate": 7.808750607753711e-10,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.671875,
      "logps/chosen": -320.0,
      "logps/rejected": -704.0,
      "loss": 0.4004,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 4.125,
      "rewards/rejected": -6.0,
      "step": 12990
    },
    {
      "epoch": 0.9781054849146039,
      "grad_norm": 25.985779908176635,
      "learning_rate": 7.29876400417051e-10,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.46875,
      "logps/chosen": -308.0,
      "logps/rejected": -656.0,
      "loss": 0.3617,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8203125,
      "rewards/margins": 3.671875,
      "rewards/rejected": -5.5,
      "step": 13000
    },
    {
      "epoch": 0.9781054849146039,
      "eval_logits/chosen": -2.546875,
      "eval_logits/rejected": -2.609375,
      "eval_logps/chosen": -338.0,
      "eval_logps/rejected": -668.0,
      "eval_loss": 0.26205962896347046,
      "eval_rewards/accuracies": 0.8850710988044739,
      "eval_rewards/chosen": -2.0625,
      "eval_rewards/margins": 3.4375,
      "eval_rewards/rejected": -5.5,
      "eval_runtime": 2667.8907,
      "eval_samples_per_second": 35.419,
      "eval_steps_per_second": 0.554,
      "step": 13000
    },
    {
      "epoch": 0.9788578737491536,
      "grad_norm": 23.15205755719879,
      "learning_rate": 6.805973689933408e-10,
      "logits/chosen": -2.625,
      "logits/rejected": -2.640625,
      "logps/chosen": -318.0,
      "logps/rejected": -664.0,
      "loss": 0.3739,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.078125,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.625,
      "step": 13010
    },
    {
      "epoch": 0.9796102625837032,
      "grad_norm": 29.35501418135914,
      "learning_rate": 6.330383064633849e-10,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.859375,
      "logps/chosen": -306.0,
      "logps/rejected": -668.0,
      "loss": 0.3883,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.828125,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.625,
      "step": 13020
    },
    {
      "epoch": 0.980362651418253,
      "grad_norm": 24.421894066728203,
      "learning_rate": 5.871995409207908e-10,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.6875,
      "logps/chosen": -306.0,
      "logps/rejected": -688.0,
      "loss": 0.3984,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.859375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -5.75,
      "step": 13030
    },
    {
      "epoch": 0.9811150402528026,
      "grad_norm": 23.37314833382285,
      "learning_rate": 5.430813885914653e-10,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.59375,
      "logps/chosen": -310.0,
      "logps/rejected": -736.0,
      "loss": 0.365,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 4.34375,
      "rewards/rejected": -6.21875,
      "step": 13040
    },
    {
      "epoch": 0.9818674290873524,
      "grad_norm": 21.682377333153728,
      "learning_rate": 5.006841538313933e-10,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.703125,
      "logps/chosen": -324.0,
      "logps/rejected": -708.0,
      "loss": 0.3727,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.0,
      "rewards/margins": 4.0625,
      "rewards/rejected": -6.0625,
      "step": 13050
    },
    {
      "epoch": 0.982619817921902,
      "grad_norm": 17.975585160333914,
      "learning_rate": 4.600081291245006e-10,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.65625,
      "logps/chosen": -310.0,
      "logps/rejected": -672.0,
      "loss": 0.3608,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 3.625,
      "rewards/rejected": -5.5,
      "step": 13060
    },
    {
      "epoch": 0.9833722067564518,
      "grad_norm": 31.261600440666417,
      "learning_rate": 4.2105359508071147e-10,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.46875,
      "logps/chosen": -360.0,
      "logps/rejected": -716.0,
      "loss": 0.3702,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.25,
      "rewards/margins": 3.65625,
      "rewards/rejected": -5.90625,
      "step": 13070
    },
    {
      "epoch": 0.9841245955910014,
      "grad_norm": 24.985967491339412,
      "learning_rate": 3.8382082043400544e-10,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.5,
      "logps/chosen": -334.0,
      "logps/rejected": -708.0,
      "loss": 0.3941,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.96875,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.71875,
      "step": 13080
    },
    {
      "epoch": 0.9848769844255512,
      "grad_norm": 28.656305414499588,
      "learning_rate": 3.4831006204044666e-10,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.671875,
      "logps/chosen": -296.0,
      "logps/rejected": -736.0,
      "loss": 0.3549,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.90625,
      "rewards/margins": 4.25,
      "rewards/rejected": -6.15625,
      "step": 13090
    },
    {
      "epoch": 0.9856293732601008,
      "grad_norm": 23.893111132336312,
      "learning_rate": 3.14521564876602e-10,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.5,
      "logps/chosen": -328.0,
      "logps/rejected": -684.0,
      "loss": 0.3442,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.6875,
      "rewards/rejected": -5.625,
      "step": 13100
    },
    {
      "epoch": 0.9863817620946506,
      "grad_norm": 24.04886027003762,
      "learning_rate": 2.8245556203768116e-10,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.703125,
      "logps/chosen": -316.0,
      "logps/rejected": -672.0,
      "loss": 0.3755,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 3.609375,
      "rewards/rejected": -5.59375,
      "step": 13110
    },
    {
      "epoch": 0.9871341509292002,
      "grad_norm": 23.926751106628704,
      "learning_rate": 2.5211227473603826e-10,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.578125,
      "logps/chosen": -352.0,
      "logps/rejected": -716.0,
      "loss": 0.3964,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.109375,
      "rewards/margins": 3.765625,
      "rewards/rejected": -5.875,
      "step": 13120
    },
    {
      "epoch": 0.98788653976375,
      "grad_norm": 22.283834427770305,
      "learning_rate": 2.2349191229956156e-10,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.28125,
      "logps/chosen": -304.0,
      "logps/rejected": -652.0,
      "loss": 0.3543,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.890625,
      "rewards/margins": 3.546875,
      "rewards/rejected": -5.4375,
      "step": 13130
    },
    {
      "epoch": 0.9886389285982996,
      "grad_norm": 20.58929724122769,
      "learning_rate": 1.9659467217031377e-10,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.71875,
      "logps/chosen": -318.0,
      "logps/rejected": -684.0,
      "loss": 0.3939,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.90625,
      "rewards/margins": 3.828125,
      "rewards/rejected": -5.75,
      "step": 13140
    },
    {
      "epoch": 0.9893913174328492,
      "grad_norm": 23.732782242402198,
      "learning_rate": 1.7142073990308868e-10,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.734375,
      "logps/chosen": -304.0,
      "logps/rejected": -708.0,
      "loss": 0.3324,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.8125,
      "rewards/margins": 4.03125,
      "rewards/rejected": -5.875,
      "step": 13150
    },
    {
      "epoch": 0.990143706267399,
      "grad_norm": 31.550280803428222,
      "learning_rate": 1.4797028916424536e-10,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.65625,
      "logps/chosen": -316.0,
      "logps/rejected": -692.0,
      "loss": 0.3963,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9453125,
      "rewards/margins": 3.640625,
      "rewards/rejected": -5.59375,
      "step": 13160
    },
    {
      "epoch": 0.9908960951019486,
      "grad_norm": 28.403429647088956,
      "learning_rate": 1.2624348173034814e-10,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.5,
      "logps/chosen": -312.0,
      "logps/rejected": -680.0,
      "loss": 0.3784,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.8671875,
      "rewards/margins": 3.734375,
      "rewards/rejected": -5.59375,
      "step": 13170
    },
    {
      "epoch": 0.9916484839364984,
      "grad_norm": 23.92440814208176,
      "learning_rate": 1.0624046748716753e-10,
      "logits/chosen": -2.625,
      "logits/rejected": -2.734375,
      "logps/chosen": -308.0,
      "logps/rejected": -736.0,
      "loss": 0.3086,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.796875,
      "rewards/margins": 4.3125,
      "rewards/rejected": -6.09375,
      "step": 13180
    },
    {
      "epoch": 0.992400872771048,
      "grad_norm": 22.24409953820488,
      "learning_rate": 8.796138442868084e-11,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.625,
      "logps/chosen": -318.0,
      "logps/rejected": -692.0,
      "loss": 0.4023,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.828125,
      "rewards/margins": 4.0,
      "rewards/rejected": -5.8125,
      "step": 13190
    },
    {
      "epoch": 0.9931532616055978,
      "grad_norm": 17.07096980826093,
      "learning_rate": 7.140635865593437e-11,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.703125,
      "logps/chosen": -312.0,
      "logps/rejected": -688.0,
      "loss": 0.3342,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.046875,
      "rewards/margins": 3.828125,
      "rewards/rejected": -5.875,
      "step": 13200
    },
    {
      "epoch": 0.9939056504401474,
      "grad_norm": 23.374178740752242,
      "learning_rate": 5.65755043764049e-11,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.671875,
      "logps/chosen": -334.0,
      "logps/rejected": -696.0,
      "loss": 0.3561,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.96875,
      "rewards/margins": 3.78125,
      "rewards/rejected": -5.75,
      "step": 13210
    },
    {
      "epoch": 0.9946580392746972,
      "grad_norm": 19.59696738105851,
      "learning_rate": 4.346892390302836e-11,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.859375,
      "logps/chosen": -342.0,
      "logps/rejected": -684.0,
      "loss": 0.3709,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.09375,
      "rewards/margins": 3.484375,
      "rewards/rejected": -5.59375,
      "step": 13220
    },
    {
      "epoch": 0.9954104281092468,
      "grad_norm": 17.18044852481147,
      "learning_rate": 3.208670765364463e-11,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.65625,
      "logps/chosen": -318.0,
      "logps/rejected": -660.0,
      "loss": 0.3606,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.9375,
      "rewards/margins": 3.46875,
      "rewards/rejected": -5.40625,
      "step": 13230
    },
    {
      "epoch": 0.9961628169437966,
      "grad_norm": 25.774746349311112,
      "learning_rate": 2.2428934150192646e-11,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.421875,
      "logps/chosen": -318.0,
      "logps/rejected": -676.0,
      "loss": 0.3515,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -5.5625,
      "step": 13240
    },
    {
      "epoch": 0.9969152057783462,
      "grad_norm": 27.17943443175996,
      "learning_rate": 1.4495670018405125e-11,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.640625,
      "logps/chosen": -328.0,
      "logps/rejected": -700.0,
      "loss": 0.3672,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.21875,
      "rewards/margins": 3.75,
      "rewards/rejected": -5.96875,
      "step": 13250
    },
    {
      "epoch": 0.9976675946128959,
      "grad_norm": 27.914559228102014,
      "learning_rate": 8.286969987086888e-12,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.578125,
      "logps/chosen": -338.0,
      "logps/rejected": -708.0,
      "loss": 0.3704,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.09375,
      "rewards/margins": 3.65625,
      "rewards/rejected": -5.75,
      "step": 13260
    },
    {
      "epoch": 0.9984199834474456,
      "grad_norm": 29.679143987554053,
      "learning_rate": 3.802876887948336e-12,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.5625,
      "logps/chosen": -320.0,
      "logps/rejected": -668.0,
      "loss": 0.3575,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.03125,
      "rewards/margins": 3.46875,
      "rewards/rejected": -5.5,
      "step": 13270
    },
    {
      "epoch": 0.9991723722819953,
      "grad_norm": 26.967267425639182,
      "learning_rate": 1.0434216552168695e-12,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.390625,
      "logps/chosen": -326.0,
      "logps/rejected": -724.0,
      "loss": 0.3283,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.953125,
      "rewards/margins": 4.125,
      "rewards/rejected": -6.09375,
      "step": 13280
    },
    {
      "epoch": 0.999924761116545,
      "grad_norm": 25.01214779575203,
      "learning_rate": 8.623325414847116e-15,
      "logits/chosen": -2.625,
      "logits/rejected": -2.65625,
      "logps/chosen": -334.0,
      "logps/rejected": -660.0,
      "loss": 0.36,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.078125,
      "rewards/margins": 3.40625,
      "rewards/rejected": -5.46875,
      "step": 13290
    },
    {
      "epoch": 1.0,
      "step": 13291,
      "total_flos": 0.0,
      "train_loss": 0.5075161390510498,
      "train_runtime": 153689.2585,
      "train_samples_per_second": 11.069,
      "train_steps_per_second": 0.086
    }
  ],
  "logging_steps": 10,
  "max_steps": 13291,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 10000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}