{
  "best_metric": 0.2576223611831665,
  "best_model_checkpoint": "models/llama3.2-3b-orpo-mini-fp/checkpoint-5000",
  "epoch": 1.9998721963064732,
  "eval_steps": 5000,
  "global_step": 15648,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.00012780369352674293,
      "grad_norm": 26.16355068979081,
      "learning_rate": 8e-08,
      "logits/chosen": -0.8046875,
      "logits/rejected": -0.875,
      "logps/chosen": -238.0,
      "logps/rejected": -242.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0012780369352674292,
      "grad_norm": 35.255934290430766,
      "learning_rate": 8e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.6953125,
      "logps/chosen": -270.0,
      "logps/rejected": -222.0,
      "loss": 0.6905,
      "rewards/accuracies": 0.2638888955116272,
      "rewards/chosen": 0.03125,
      "rewards/margins": 0.016357421875,
      "rewards/rejected": 0.01495361328125,
      "step": 10
    },
    {
      "epoch": 0.0025560738705348585,
      "grad_norm": 18.31723646106629,
      "learning_rate": 1.6e-06,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.87890625,
      "logps/chosen": -278.0,
      "logps/rejected": -223.0,
      "loss": 0.6059,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.5625,
      "rewards/margins": 0.25390625,
      "rewards/rejected": 0.30859375,
      "step": 20
    },
    {
      "epoch": 0.0038341108058022877,
      "grad_norm": 16.18307081214992,
      "learning_rate": 2.4e-06,
      "logits/chosen": -1.4453125,
      "logits/rejected": -1.1953125,
      "logps/chosen": -268.0,
      "logps/rejected": -227.0,
      "loss": 0.5234,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.5234375,
      "rewards/margins": 0.9375,
      "rewards/rejected": 0.58203125,
      "step": 30
    },
    {
      "epoch": 0.005112147741069717,
      "grad_norm": 20.377358400504054,
      "learning_rate": 3.2e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -1.1484375,
      "logps/chosen": -245.0,
      "logps/rejected": -217.0,
      "loss": 0.5055,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.9140625,
      "rewards/margins": 1.2109375,
      "rewards/rejected": 0.69921875,
      "step": 40
    },
    {
      "epoch": 0.006390184676337146,
      "grad_norm": 14.564840991583996,
      "learning_rate": 4e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.8828125,
      "logps/chosen": -256.0,
      "logps/rejected": -217.0,
      "loss": 0.4621,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 1.9921875,
      "rewards/margins": 1.640625,
      "rewards/rejected": 0.353515625,
      "step": 50
    },
    {
      "epoch": 0.007668221611604575,
      "grad_norm": 16.318860657272133,
      "learning_rate": 4.8e-06,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.8125,
      "logps/chosen": -254.0,
      "logps/rejected": -216.0,
      "loss": 0.4719,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.1328125,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -0.23046875,
      "step": 60
    },
    {
      "epoch": 0.008946258546872005,
      "grad_norm": 17.36087809629676,
      "learning_rate": 5.6e-06,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.703125,
      "logps/chosen": -256.0,
      "logps/rejected": -232.0,
      "loss": 0.4521,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 1.6484375,
      "rewards/margins": 1.75,
      "rewards/rejected": -0.10009765625,
      "step": 70
    },
    {
      "epoch": 0.010224295482139434,
      "grad_norm": 17.523844379925357,
      "learning_rate": 6.4e-06,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.640625,
      "logps/chosen": -264.0,
      "logps/rejected": -236.0,
      "loss": 0.4084,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 0.9921875,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -0.91015625,
      "step": 80
    },
    {
      "epoch": 0.011502332417406863,
      "grad_norm": 15.811506364509809,
      "learning_rate": 7.2e-06,
      "logits/chosen": -0.8359375,
      "logits/rejected": -0.609375,
      "logps/chosen": -270.0,
      "logps/rejected": -256.0,
      "loss": 0.4311,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.671875,
      "rewards/margins": 2.0625,
      "rewards/rejected": -2.734375,
      "step": 90
    },
    {
      "epoch": 0.012780369352674292,
      "grad_norm": 13.287493742992458,
      "learning_rate": 8e-06,
      "logits/chosen": -0.83203125,
      "logits/rejected": -0.369140625,
      "logps/chosen": -272.0,
      "logps/rejected": -250.0,
      "loss": 0.4365,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.251953125,
      "rewards/margins": 2.234375,
      "rewards/rejected": -2.484375,
      "step": 100
    },
    {
      "epoch": 0.014058406287941722,
      "grad_norm": 16.707746729759,
      "learning_rate": 7.627700713964738e-06,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.6015625,
      "logps/chosen": -280.0,
      "logps/rejected": -260.0,
      "loss": 0.3997,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.2890625,
      "rewards/margins": 2.421875,
      "rewards/rejected": -3.703125,
      "step": 110
    },
    {
      "epoch": 0.01533644322320915,
      "grad_norm": 15.652966658465445,
      "learning_rate": 7.3029674334022146e-06,
      "logits/chosen": -0.8125,
      "logits/rejected": -0.439453125,
      "logps/chosen": -276.0,
      "logps/rejected": -262.0,
      "loss": 0.4335,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 2.21875,
      "rewards/rejected": -3.828125,
      "step": 120
    },
    {
      "epoch": 0.01661448015847658,
      "grad_norm": 14.981051965738123,
      "learning_rate": 7.016464154456233e-06,
      "logits/chosen": -0.5859375,
      "logits/rejected": -0.341796875,
      "logps/chosen": -268.0,
      "logps/rejected": -284.0,
      "loss": 0.448,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.3828125,
      "rewards/margins": 1.890625,
      "rewards/rejected": -3.28125,
      "step": 130
    },
    {
      "epoch": 0.01789251709374401,
      "grad_norm": 16.727785144560304,
      "learning_rate": 6.7612340378281325e-06,
      "logits/chosen": -0.9140625,
      "logits/rejected": -0.56640625,
      "logps/chosen": -288.0,
      "logps/rejected": -266.0,
      "loss": 0.4179,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.77734375,
      "rewards/margins": 2.171875,
      "rewards/rejected": -2.953125,
      "step": 140
    },
    {
      "epoch": 0.01917055402901144,
      "grad_norm": 14.246878678556142,
      "learning_rate": 6.531972647421809e-06,
      "logits/chosen": -0.61328125,
      "logits/rejected": -0.412109375,
      "logps/chosen": -282.0,
      "logps/rejected": -268.0,
      "loss": 0.3751,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.265625,
      "rewards/margins": 3.0,
      "rewards/rejected": -4.25,
      "step": 150
    },
    {
      "epoch": 0.020448590964278868,
      "grad_norm": 15.87675912819951,
      "learning_rate": 6.3245553203367575e-06,
      "logits/chosen": -0.69921875,
      "logits/rejected": -0.283203125,
      "logps/chosen": -310.0,
      "logps/rejected": -302.0,
      "loss": 0.4075,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.5,
      "rewards/margins": 3.671875,
      "rewards/rejected": -6.1875,
      "step": 160
    },
    {
      "epoch": 0.021726627899546297,
      "grad_norm": 14.27613894881428,
      "learning_rate": 6.135719910778963e-06,
      "logits/chosen": -0.7890625,
      "logits/rejected": -0.2138671875,
      "logps/chosen": -304.0,
      "logps/rejected": -306.0,
      "loss": 0.364,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.015625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -6.78125,
      "step": 170
    },
    {
      "epoch": 0.023004664834813726,
      "grad_norm": 15.597494857838177,
      "learning_rate": 5.962847939999439e-06,
      "logits/chosen": -0.828125,
      "logits/rejected": -0.28125,
      "logps/chosen": -288.0,
      "logps/rejected": -280.0,
      "loss": 0.4248,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.734375,
      "rewards/margins": 2.96875,
      "rewards/rejected": -4.6875,
      "step": 180
    },
    {
      "epoch": 0.024282701770081155,
      "grad_norm": 13.651425467446431,
      "learning_rate": 5.803810000880094e-06,
      "logits/chosen": -0.70703125,
      "logits/rejected": -0.294921875,
      "logps/chosen": -320.0,
      "logps/rejected": -304.0,
      "loss": 0.409,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.859375,
      "rewards/margins": 2.703125,
      "rewards/rejected": -6.5625,
      "step": 190
    },
    {
      "epoch": 0.025560738705348585,
      "grad_norm": 9.342851088545837,
      "learning_rate": 5.65685424949238e-06,
      "logits/chosen": -0.451171875,
      "logits/rejected": -0.0888671875,
      "logps/chosen": -280.0,
      "logps/rejected": -270.0,
      "loss": 0.33,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.828125,
      "rewards/margins": 2.453125,
      "rewards/rejected": -6.28125,
      "step": 200
    },
    {
      "epoch": 0.026838775640616014,
      "grad_norm": 16.591231515404967,
      "learning_rate": 5.5205244747388325e-06,
      "logits/chosen": -0.67578125,
      "logits/rejected": -0.2060546875,
      "logps/chosen": -310.0,
      "logps/rejected": -302.0,
      "loss": 0.398,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.21875,
      "rewards/margins": 3.375,
      "rewards/rejected": -7.59375,
      "step": 210
    },
    {
      "epoch": 0.028116812575883443,
      "grad_norm": 14.99355352482897,
      "learning_rate": 5.393598899705936e-06,
      "logits/chosen": -0.5625,
      "logits/rejected": -0.1083984375,
      "logps/chosen": -310.0,
      "logps/rejected": -306.0,
      "loss": 0.4061,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.828125,
      "rewards/margins": 2.875,
      "rewards/rejected": -5.71875,
      "step": 220
    },
    {
      "epoch": 0.029394849511150872,
      "grad_norm": 12.960679306389352,
      "learning_rate": 5.275043787166296e-06,
      "logits/chosen": -0.462890625,
      "logits/rejected": -0.03466796875,
      "logps/chosen": -306.0,
      "logps/rejected": -296.0,
      "loss": 0.3681,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.703125,
      "rewards/margins": 2.6875,
      "rewards/rejected": -6.40625,
      "step": 230
    },
    {
      "epoch": 0.0306728864464183,
      "grad_norm": 16.566331388920407,
      "learning_rate": 5.163977794943223e-06,
      "logits/chosen": -0.5546875,
      "logits/rejected": -0.07275390625,
      "logps/chosen": -338.0,
      "logps/rejected": -312.0,
      "loss": 0.4252,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.359375,
      "rewards/rejected": -8.5,
      "step": 240
    },
    {
      "epoch": 0.03195092338168573,
      "grad_norm": 13.003802882397766,
      "learning_rate": 5.059644256269407e-06,
      "logits/chosen": -0.3671875,
      "logits/rejected": 0.00665283203125,
      "logps/chosen": -318.0,
      "logps/rejected": -314.0,
      "loss": 0.3364,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -5.1875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -8.375,
      "step": 250
    },
    {
      "epoch": 0.03322896031695316,
      "grad_norm": 13.130685114041277,
      "learning_rate": 4.961389383568338e-06,
      "logits/chosen": -0.369140625,
      "logits/rejected": 0.046875,
      "logps/chosen": -316.0,
      "logps/rejected": -318.0,
      "loss": 0.3233,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -4.375,
      "rewards/margins": 3.703125,
      "rewards/rejected": -8.0625,
      "step": 260
    },
    {
      "epoch": 0.03450699725222059,
      "grad_norm": 13.810098997797867,
      "learning_rate": 4.8686449556014755e-06,
      "logits/chosen": -0.30078125,
      "logits/rejected": 0.34375,
      "logps/chosen": -336.0,
      "logps/rejected": -328.0,
      "loss": 0.3351,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -6.5,
      "rewards/margins": 4.0625,
      "rewards/rejected": -10.5625,
      "step": 270
    },
    {
      "epoch": 0.03578503418748802,
      "grad_norm": 13.154087557787248,
      "learning_rate": 4.780914437337574e-06,
      "logits/chosen": -0.17578125,
      "logits/rejected": 0.33203125,
      "logps/chosen": -332.0,
      "logps/rejected": -330.0,
      "loss": 0.3512,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -6.59375,
      "rewards/margins": 3.453125,
      "rewards/rejected": -10.0,
      "step": 280
    },
    {
      "epoch": 0.03706307112275545,
      "grad_norm": 14.281012072732354,
      "learning_rate": 4.697761756117627e-06,
      "logits/chosen": -0.04736328125,
      "logits/rejected": 0.55859375,
      "logps/chosen": -336.0,
      "logps/rejected": -320.0,
      "loss": 0.3997,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -7.0625,
      "rewards/margins": 3.21875,
      "rewards/rejected": -10.25,
      "step": 290
    },
    {
      "epoch": 0.03834110805802288,
      "grad_norm": 8.682598036905006,
      "learning_rate": 4.618802153517006e-06,
      "logits/chosen": -0.267578125,
      "logits/rejected": 0.337890625,
      "logps/chosen": -338.0,
      "logps/rejected": -332.0,
      "loss": 0.3031,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -9.375,
      "step": 300
    },
    {
      "epoch": 0.039619144993290306,
      "grad_norm": 15.277270766116551,
      "learning_rate": 4.543694673976518e-06,
      "logits/chosen": -0.28125,
      "logits/rejected": 0.349609375,
      "logps/chosen": -322.0,
      "logps/rejected": -336.0,
      "loss": 0.35,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -10.1875,
      "step": 310
    },
    {
      "epoch": 0.040897181928557735,
      "grad_norm": 13.852022381105478,
      "learning_rate": 4.472135954999579e-06,
      "logits/chosen": -0.208984375,
      "logits/rejected": 0.212890625,
      "logps/chosen": -306.0,
      "logps/rejected": -318.0,
      "loss": 0.3199,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -4.625,
      "rewards/margins": 3.5,
      "rewards/rejected": -8.125,
      "step": 320
    },
    {
      "epoch": 0.042175218863825165,
      "grad_norm": 9.457680843536519,
      "learning_rate": 4.403855060505443e-06,
      "logits/chosen": -0.29296875,
      "logits/rejected": 0.2373046875,
      "logps/chosen": -334.0,
      "logps/rejected": -336.0,
      "loss": 0.3466,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.125,
      "rewards/rejected": -9.8125,
      "step": 330
    },
    {
      "epoch": 0.043453255799092594,
      "grad_norm": 13.035251295550905,
      "learning_rate": 4.338609156373123e-06,
      "logits/chosen": -0.322265625,
      "logits/rejected": 0.41015625,
      "logps/chosen": -352.0,
      "logps/rejected": -328.0,
      "loss": 0.3935,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -6.4375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -10.3125,
      "step": 340
    },
    {
      "epoch": 0.04473129273436002,
      "grad_norm": 11.912920595945469,
      "learning_rate": 4.27617987059879e-06,
      "logits/chosen": -0.103515625,
      "logits/rejected": 0.400390625,
      "logps/chosen": -320.0,
      "logps/rejected": -342.0,
      "loss": 0.3156,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -6.75,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.9375,
      "step": 350
    },
    {
      "epoch": 0.04600932966962745,
      "grad_norm": 13.446253645828373,
      "learning_rate": 4.216370213557839e-06,
      "logits/chosen": -0.09814453125,
      "logits/rejected": 0.546875,
      "logps/chosen": -370.0,
      "logps/rejected": -352.0,
      "loss": 0.3938,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -8.875,
      "rewards/margins": 4.125,
      "rewards/rejected": -13.0625,
      "step": 360
    },
    {
      "epoch": 0.04728736660489488,
      "grad_norm": 10.87294198936051,
      "learning_rate": 4.15900195928029e-06,
      "logits/chosen": -0.07470703125,
      "logits/rejected": 0.55078125,
      "logps/chosen": -370.0,
      "logps/rejected": -362.0,
      "loss": 0.3161,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -8.5,
      "rewards/margins": 4.46875,
      "rewards/rejected": -12.9375,
      "step": 370
    },
    {
      "epoch": 0.04856540354016231,
      "grad_norm": 11.01141778789033,
      "learning_rate": 4.103913408340617e-06,
      "logits/chosen": 0.07421875,
      "logits/rejected": 0.6875,
      "logps/chosen": -324.0,
      "logps/rejected": -342.0,
      "loss": 0.2891,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -7.8125,
      "rewards/margins": 4.125,
      "rewards/rejected": -11.9375,
      "step": 380
    },
    {
      "epoch": 0.04984344047542974,
      "grad_norm": 15.697505391883448,
      "learning_rate": 4.050957468334666e-06,
      "logits/chosen": 0.02783203125,
      "logits/rejected": 0.7421875,
      "logps/chosen": -366.0,
      "logps/rejected": -332.0,
      "loss": 0.3477,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -8.3125,
      "rewards/margins": 3.734375,
      "rewards/rejected": -12.0625,
      "step": 390
    },
    {
      "epoch": 0.05112147741069717,
      "grad_norm": 12.278657215770064,
      "learning_rate": 4e-06,
      "logits/chosen": -0.0576171875,
      "logits/rejected": 0.302734375,
      "logps/chosen": -342.0,
      "logps/rejected": -354.0,
      "loss": 0.314,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -6.6875,
      "rewards/margins": 3.84375,
      "rewards/rejected": -10.5,
      "step": 400
    },
    {
      "epoch": 0.0523995143459646,
      "grad_norm": 11.964640891236618,
      "learning_rate": 3.950918386598359e-06,
      "logits/chosen": -0.189453125,
      "logits/rejected": 0.361328125,
      "logps/chosen": -336.0,
      "logps/rejected": -340.0,
      "loss": 0.3146,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -6.53125,
      "rewards/margins": 4.0,
      "rewards/rejected": -10.5625,
      "step": 410
    },
    {
      "epoch": 0.05367755128123203,
      "grad_norm": 13.488671096633563,
      "learning_rate": 3.903600291794132e-06,
      "logits/chosen": -0.130859375,
      "logits/rejected": 0.44140625,
      "logps/chosen": -344.0,
      "logps/rejected": -340.0,
      "loss": 0.3131,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -7.0625,
      "rewards/margins": 3.515625,
      "rewards/rejected": -10.5625,
      "step": 420
    },
    {
      "epoch": 0.05495558821649946,
      "grad_norm": 7.322925823164035,
      "learning_rate": 3.857942577363297e-06,
      "logits/chosen": -0.1376953125,
      "logits/rejected": 0.392578125,
      "logps/chosen": -348.0,
      "logps/rejected": -328.0,
      "loss": 0.3016,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -7.875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -11.625,
      "step": 430
    },
    {
      "epoch": 0.056233625151766886,
      "grad_norm": 12.523599319039954,
      "learning_rate": 3.813850356982369e-06,
      "logits/chosen": -0.08984375,
      "logits/rejected": 0.453125,
      "logps/chosen": -346.0,
      "logps/rejected": -356.0,
      "loss": 0.3883,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -7.28125,
      "rewards/margins": 4.34375,
      "rewards/rejected": -11.625,
      "step": 440
    },
    {
      "epoch": 0.057511662087034315,
      "grad_norm": 13.194504079931475,
      "learning_rate": 3.7712361663282537e-06,
      "logits/chosen": 0.0517578125,
      "logits/rejected": 0.486328125,
      "logps/chosen": -338.0,
      "logps/rejected": -340.0,
      "loss": 0.2982,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -6.8125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -10.875,
      "step": 450
    },
    {
      "epoch": 0.058789699022301745,
      "grad_norm": 11.543901702099564,
      "learning_rate": 3.730019232961255e-06,
      "logits/chosen": 0.095703125,
      "logits/rejected": 0.57421875,
      "logps/chosen": -320.0,
      "logps/rejected": -338.0,
      "loss": 0.31,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -7.46875,
      "rewards/margins": 4.0,
      "rewards/rejected": -11.4375,
      "step": 460
    },
    {
      "epoch": 0.060067735957569174,
      "grad_norm": 8.657243376745845,
      "learning_rate": 3.6901248321155403e-06,
      "logits/chosen": 0.0986328125,
      "logits/rejected": 0.65625,
      "logps/chosen": -354.0,
      "logps/rejected": -370.0,
      "loss": 0.3016,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -9.6875,
      "rewards/margins": 4.5,
      "rewards/rejected": -14.1875,
      "step": 470
    },
    {
      "epoch": 0.0613457728928366,
      "grad_norm": 14.991841349219749,
      "learning_rate": 3.6514837167011073e-06,
      "logits/chosen": -0.03466796875,
      "logits/rejected": 0.671875,
      "logps/chosen": -384.0,
      "logps/rejected": -378.0,
      "loss": 0.3115,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -9.625,
      "rewards/margins": 4.53125,
      "rewards/rejected": -14.1875,
      "step": 480
    },
    {
      "epoch": 0.06262380982810403,
      "grad_norm": 9.092509356896064,
      "learning_rate": 3.6140316116210052e-06,
      "logits/chosen": -0.0257568359375,
      "logits/rejected": 0.59375,
      "logps/chosen": -362.0,
      "logps/rejected": -362.0,
      "loss": 0.2815,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -9.25,
      "rewards/margins": 4.5625,
      "rewards/rejected": -13.8125,
      "step": 490
    },
    {
      "epoch": 0.06390184676337146,
      "grad_norm": 11.689827835120473,
      "learning_rate": 3.5777087639996634e-06,
      "logits/chosen": 0.0537109375,
      "logits/rejected": 0.59375,
      "logps/chosen": -336.0,
      "logps/rejected": -344.0,
      "loss": 0.3242,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -7.96875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -11.5625,
      "step": 500
    },
    {
      "epoch": 0.06517988369863889,
      "grad_norm": 13.471708876236821,
      "learning_rate": 3.5424595421603814e-06,
      "logits/chosen": 0.232421875,
      "logits/rejected": 0.7734375,
      "logps/chosen": -366.0,
      "logps/rejected": -352.0,
      "loss": 0.3051,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -9.5625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -13.625,
      "step": 510
    },
    {
      "epoch": 0.06645792063390632,
      "grad_norm": 12.465629624281892,
      "learning_rate": 3.5082320772281165e-06,
      "logits/chosen": 0.376953125,
      "logits/rejected": 0.71484375,
      "logps/chosen": -338.0,
      "logps/rejected": -358.0,
      "loss": 0.323,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -8.9375,
      "rewards/margins": 3.359375,
      "rewards/rejected": -12.3125,
      "step": 520
    },
    {
      "epoch": 0.06773595756917375,
      "grad_norm": 9.058999713745413,
      "learning_rate": 3.474977942104555e-06,
      "logits/chosen": 0.142578125,
      "logits/rejected": 0.73828125,
      "logps/chosen": -354.0,
      "logps/rejected": -352.0,
      "loss": 0.3741,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -7.875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -11.9375,
      "step": 530
    },
    {
      "epoch": 0.06901399450444118,
      "grad_norm": 13.301924780038847,
      "learning_rate": 3.442651863295481e-06,
      "logits/chosen": 0.05712890625,
      "logits/rejected": 0.478515625,
      "logps/chosen": -358.0,
      "logps/rejected": -366.0,
      "loss": 0.2859,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -7.71875,
      "rewards/margins": 4.8125,
      "rewards/rejected": -12.5,
      "step": 540
    },
    {
      "epoch": 0.07029203143970861,
      "grad_norm": 9.480288752896776,
      "learning_rate": 3.4112114616897665e-06,
      "logits/chosen": 0.3671875,
      "logits/rejected": 0.87109375,
      "logps/chosen": -372.0,
      "logps/rejected": -388.0,
      "loss": 0.2903,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -10.8125,
      "rewards/margins": 4.5,
      "rewards/rejected": -15.3125,
      "step": 550
    },
    {
      "epoch": 0.07157006837497604,
      "grad_norm": 10.331917280886977,
      "learning_rate": 3.3806170189140663e-06,
      "logits/chosen": 0.2451171875,
      "logits/rejected": 0.6875,
      "logps/chosen": -398.0,
      "logps/rejected": -398.0,
      "loss": 0.2756,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -11.25,
      "rewards/margins": 4.5,
      "rewards/rejected": -15.75,
      "step": 560
    },
    {
      "epoch": 0.07284810531024347,
      "grad_norm": 14.818176817976159,
      "learning_rate": 3.350831266333564e-06,
      "logits/chosen": -0.020263671875,
      "logits/rejected": 0.68359375,
      "logps/chosen": -346.0,
      "logps/rejected": -350.0,
      "loss": 0.3079,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -8.8125,
      "rewards/margins": 4.59375,
      "rewards/rejected": -13.4375,
      "step": 570
    },
    {
      "epoch": 0.0741261422455109,
      "grad_norm": 11.61850429561708,
      "learning_rate": 3.3218191941495984e-06,
      "logits/chosen": 0.1787109375,
      "logits/rejected": 0.53515625,
      "logps/chosen": -348.0,
      "logps/rejected": -344.0,
      "loss": 0.2646,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -7.40625,
      "rewards/margins": 4.625,
      "rewards/rejected": -12.0,
      "step": 580
    },
    {
      "epoch": 0.07540417918077832,
      "grad_norm": 10.691351043890407,
      "learning_rate": 3.293547878370473e-06,
      "logits/chosen": -0.2890625,
      "logits/rejected": 0.36328125,
      "logps/chosen": -360.0,
      "logps/rejected": -376.0,
      "loss": 0.3187,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -8.1875,
      "rewards/margins": 4.9375,
      "rewards/rejected": -13.125,
      "step": 590
    },
    {
      "epoch": 0.07668221611604575,
      "grad_norm": 8.935257206387947,
      "learning_rate": 3.2659863237109044e-06,
      "logits/chosen": -0.330078125,
      "logits/rejected": 0.3046875,
      "logps/chosen": -360.0,
      "logps/rejected": -376.0,
      "loss": 0.2655,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -7.21875,
      "rewards/margins": 5.09375,
      "rewards/rejected": -12.3125,
      "step": 600
    },
    {
      "epoch": 0.07796025305131318,
      "grad_norm": 10.126363333603178,
      "learning_rate": 3.239105320715664e-06,
      "logits/chosen": -0.09326171875,
      "logits/rejected": 0.375,
      "logps/chosen": -342.0,
      "logps/rejected": -356.0,
      "loss": 0.2901,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -6.65625,
      "rewards/margins": 4.125,
      "rewards/rejected": -10.8125,
      "step": 610
    },
    {
      "epoch": 0.07923828998658061,
      "grad_norm": 7.8240580189954425,
      "learning_rate": 3.2128773156099956e-06,
      "logits/chosen": 0.059326171875,
      "logits/rejected": 0.55859375,
      "logps/chosen": -366.0,
      "logps/rejected": -358.0,
      "loss": 0.2676,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -8.375,
      "rewards/margins": 4.125,
      "rewards/rejected": -12.5,
      "step": 620
    },
    {
      "epoch": 0.08051632692184804,
      "grad_norm": 11.370656922740451,
      "learning_rate": 3.187276291558383e-06,
      "logits/chosen": 0.302734375,
      "logits/rejected": 0.9140625,
      "logps/chosen": -368.0,
      "logps/rejected": -366.0,
      "loss": 0.291,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -10.5,
      "rewards/margins": 4.5,
      "rewards/rejected": -15.0625,
      "step": 630
    },
    {
      "epoch": 0.08179436385711547,
      "grad_norm": 11.854640494745686,
      "learning_rate": 3.1622776601683788e-06,
      "logits/chosen": 0.1494140625,
      "logits/rejected": 0.7265625,
      "logps/chosen": -388.0,
      "logps/rejected": -398.0,
      "loss": 0.274,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -10.1875,
      "rewards/margins": 4.8125,
      "rewards/rejected": -15.0,
      "step": 640
    },
    {
      "epoch": 0.0830724007923829,
      "grad_norm": 9.960974279092786,
      "learning_rate": 3.1378581622109444e-06,
      "logits/chosen": 0.2734375,
      "logits/rejected": 0.68359375,
      "logps/chosen": -332.0,
      "logps/rejected": -364.0,
      "loss": 0.2396,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -7.78125,
      "rewards/margins": 4.46875,
      "rewards/rejected": -12.25,
      "step": 650
    },
    {
      "epoch": 0.08435043772765033,
      "grad_norm": 7.9576363975144515,
      "learning_rate": 3.113995776646092e-06,
      "logits/chosen": 0.25390625,
      "logits/rejected": 0.734375,
      "logps/chosen": -336.0,
      "logps/rejected": -342.0,
      "loss": 0.2952,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -7.3125,
      "rewards/margins": 4.9375,
      "rewards/rejected": -12.25,
      "step": 660
    },
    {
      "epoch": 0.08562847466291776,
      "grad_norm": 10.347643954866788,
      "learning_rate": 3.090669637145023e-06,
      "logits/chosen": 0.04541015625,
      "logits/rejected": 0.5,
      "logps/chosen": -332.0,
      "logps/rejected": -352.0,
      "loss": 0.297,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -6.53125,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.875,
      "step": 670
    },
    {
      "epoch": 0.08690651159818519,
      "grad_norm": 10.78393136017663,
      "learning_rate": 3.0678599553894814e-06,
      "logits/chosen": -0.1005859375,
      "logits/rejected": 0.6015625,
      "logps/chosen": -366.0,
      "logps/rejected": -374.0,
      "loss": 0.2797,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -7.40625,
      "rewards/margins": 5.09375,
      "rewards/rejected": -12.5,
      "step": 680
    },
    {
      "epoch": 0.08818454853345262,
      "grad_norm": 11.44830107580474,
      "learning_rate": 3.0455479505075235e-06,
      "logits/chosen": 0.0634765625,
      "logits/rejected": 0.45703125,
      "logps/chosen": -350.0,
      "logps/rejected": -354.0,
      "loss": 0.2918,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -8.25,
      "rewards/margins": 3.890625,
      "rewards/rejected": -12.125,
      "step": 690
    },
    {
      "epoch": 0.08946258546872005,
      "grad_norm": 11.862894296459416,
      "learning_rate": 3.0237157840738173e-06,
      "logits/chosen": -0.047607421875,
      "logits/rejected": 0.58203125,
      "logps/chosen": -352.0,
      "logps/rejected": -368.0,
      "loss": 0.2766,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -8.8125,
      "rewards/margins": 4.625,
      "rewards/rejected": -13.4375,
      "step": 700
    },
    {
      "epoch": 0.09074062240398748,
      "grad_norm": 11.178122760940434,
      "learning_rate": 3.002346500163206e-06,
      "logits/chosen": 0.047119140625,
      "logits/rejected": 0.6171875,
      "logps/chosen": -344.0,
      "logps/rejected": -350.0,
      "loss": 0.3273,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -7.96875,
      "rewards/margins": 4.375,
      "rewards/rejected": -12.375,
      "step": 710
    },
    {
      "epoch": 0.0920186593392549,
      "grad_norm": 14.084419766619328,
      "learning_rate": 2.9814239699997195e-06,
      "logits/chosen": 0.09765625,
      "logits/rejected": 0.62890625,
      "logps/chosen": -356.0,
      "logps/rejected": -360.0,
      "loss": 0.2667,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -7.5,
      "rewards/margins": 4.9375,
      "rewards/rejected": -12.375,
      "step": 720
    },
    {
      "epoch": 0.09329669627452233,
      "grad_norm": 15.053502381365979,
      "learning_rate": 2.9609328407904207e-06,
      "logits/chosen": 0.0291748046875,
      "logits/rejected": 0.7734375,
      "logps/chosen": -336.0,
      "logps/rejected": -336.0,
      "loss": 0.2773,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -7.90625,
      "rewards/margins": 5.34375,
      "rewards/rejected": -13.25,
      "step": 730
    },
    {
      "epoch": 0.09457473320978976,
      "grad_norm": 13.594273971959737,
      "learning_rate": 2.940858488375231e-06,
      "logits/chosen": 0.07861328125,
      "logits/rejected": 0.546875,
      "logps/chosen": -360.0,
      "logps/rejected": -358.0,
      "loss": 0.2878,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.9375,
      "rewards/margins": 5.25,
      "rewards/rejected": -12.1875,
      "step": 740
    },
    {
      "epoch": 0.09585277014505719,
      "grad_norm": 11.633730602034111,
      "learning_rate": 2.9211869733608857e-06,
      "logits/chosen": 0.048828125,
      "logits/rejected": 0.5546875,
      "logps/chosen": -318.0,
      "logps/rejected": -328.0,
      "loss": 0.2801,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -6.4375,
      "rewards/margins": 4.71875,
      "rewards/rejected": -11.125,
      "step": 750
    },
    {
      "epoch": 0.09713080708032462,
      "grad_norm": 7.704507257326408,
      "learning_rate": 2.901905000440047e-06,
      "logits/chosen": -0.11572265625,
      "logits/rejected": 0.359375,
      "logps/chosen": -336.0,
      "logps/rejected": -362.0,
      "loss": 0.2646,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.71875,
      "rewards/rejected": -10.5625,
      "step": 760
    },
    {
      "epoch": 0.09840884401559205,
      "grad_norm": 9.641603953890037,
      "learning_rate": 2.8829998806257885e-06,
      "logits/chosen": 0.03369140625,
      "logits/rejected": 0.484375,
      "logps/chosen": -346.0,
      "logps/rejected": -364.0,
      "loss": 0.2943,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -7.625,
      "rewards/margins": 4.625,
      "rewards/rejected": -12.25,
      "step": 770
    },
    {
      "epoch": 0.09968688095085948,
      "grad_norm": 8.998644220442728,
      "learning_rate": 2.8644594961577314e-06,
      "logits/chosen": -0.2001953125,
      "logits/rejected": 0.357421875,
      "logps/chosen": -350.0,
      "logps/rejected": -362.0,
      "loss": 0.2977,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -7.6875,
      "rewards/margins": 5.1875,
      "rewards/rejected": -12.875,
      "step": 780
    },
    {
      "epoch": 0.10096491788612691,
      "grad_norm": 8.338973976445722,
      "learning_rate": 2.84627226785928e-06,
      "logits/chosen": -0.0037384033203125,
      "logits/rejected": 0.462890625,
      "logps/chosen": -350.0,
      "logps/rejected": -356.0,
      "loss": 0.3026,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -7.375,
      "rewards/margins": 4.9375,
      "rewards/rejected": -12.3125,
      "step": 790
    },
    {
      "epoch": 0.10224295482139434,
      "grad_norm": 12.465887361797773,
      "learning_rate": 2.82842712474619e-06,
      "logits/chosen": 0.0247802734375,
      "logits/rejected": 0.6484375,
      "logps/chosen": -344.0,
      "logps/rejected": -356.0,
      "loss": 0.318,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -7.6875,
      "rewards/margins": 4.4375,
      "rewards/rejected": -12.125,
      "step": 800
    },
    {
      "epoch": 0.10352099175666177,
      "grad_norm": 10.197983188521123,
      "learning_rate": 2.810913475705226e-06,
      "logits/chosen": 0.0947265625,
      "logits/rejected": 0.6640625,
      "logps/chosen": -364.0,
      "logps/rejected": -372.0,
      "loss": 0.2628,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -9.5,
      "rewards/margins": 5.40625,
      "rewards/rejected": -14.9375,
      "step": 810
    },
    {
      "epoch": 0.1047990286919292,
      "grad_norm": 8.751265057729666,
      "learning_rate": 2.7937211830783128e-06,
      "logits/chosen": 0.17578125,
      "logits/rejected": 0.62890625,
      "logps/chosen": -358.0,
      "logps/rejected": -414.0,
      "loss": 0.2809,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -9.4375,
      "rewards/margins": 8.125,
      "rewards/rejected": -17.625,
      "step": 820
    },
    {
      "epoch": 0.10607706562719663,
      "grad_norm": 9.8380815512954,
      "learning_rate": 2.776840538002493e-06,
      "logits/chosen": -0.11474609375,
      "logits/rejected": 0.5859375,
      "logps/chosen": -338.0,
      "logps/rejected": -342.0,
      "loss": 0.2856,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -6.875,
      "rewards/margins": 5.1875,
      "rewards/rejected": -12.0625,
      "step": 830
    },
    {
      "epoch": 0.10735510256246406,
      "grad_norm": 14.828358221650914,
      "learning_rate": 2.7602622373694163e-06,
      "logits/chosen": -0.2138671875,
      "logits/rejected": 0.416015625,
      "logps/chosen": -340.0,
      "logps/rejected": -354.0,
      "loss": 0.2641,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.59375,
      "rewards/margins": 5.1875,
      "rewards/rejected": -10.75,
      "step": 840
    },
    {
      "epoch": 0.10863313949773148,
      "grad_norm": 13.586421581520623,
      "learning_rate": 2.743977362280141e-06,
      "logits/chosen": 0.04931640625,
      "logits/rejected": 0.41015625,
      "logps/chosen": -338.0,
      "logps/rejected": -374.0,
      "loss": 0.2952,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -6.46875,
      "rewards/margins": 4.84375,
      "rewards/rejected": -11.3125,
      "step": 850
    },
    {
      "epoch": 0.10991117643299891,
      "grad_norm": 13.737005231984735,
      "learning_rate": 2.7279773578818937e-06,
      "logits/chosen": 0.07177734375,
      "logits/rejected": 0.5703125,
      "logps/chosen": -322.0,
      "logps/rejected": -350.0,
      "loss": 0.2804,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.15625,
      "rewards/margins": 5.1875,
      "rewards/rejected": -11.375,
      "step": 860
    },
    {
      "epoch": 0.11118921336826634,
      "grad_norm": 10.468036167455898,
      "learning_rate": 2.7122540144832417e-06,
      "logits/chosen": 0.11279296875,
      "logits/rejected": 0.671875,
      "logps/chosen": -326.0,
      "logps/rejected": -336.0,
      "loss": 0.2864,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.6875,
      "rewards/margins": 4.625,
      "rewards/rejected": -11.3125,
      "step": 870
    },
    {
      "epoch": 0.11246725030353377,
      "grad_norm": 10.988144338010212,
      "learning_rate": 2.696799449852968e-06,
      "logits/chosen": 0.01251220703125,
      "logits/rejected": 0.51953125,
      "logps/chosen": -336.0,
      "logps/rejected": -360.0,
      "loss": 0.2743,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.3125,
      "rewards/margins": 4.6875,
      "rewards/rejected": -11.0,
      "step": 880
    },
    {
      "epoch": 0.1137452872388012,
      "grad_norm": 19.62788350622451,
      "learning_rate": 2.6816060926159636e-06,
      "logits/chosen": -0.158203125,
      "logits/rejected": 0.5546875,
      "logps/chosen": -376.0,
      "logps/rejected": -384.0,
      "loss": 0.2825,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -7.5,
      "rewards/margins": 5.5625,
      "rewards/rejected": -13.0625,
      "step": 890
    },
    {
      "epoch": 0.11502332417406863,
      "grad_norm": 8.748337503109397,
      "learning_rate": 2.6666666666666664e-06,
      "logits/chosen": 0.11572265625,
      "logits/rejected": 0.78515625,
      "logps/chosen": -352.0,
      "logps/rejected": -368.0,
      "loss": 0.3107,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -8.3125,
      "rewards/margins": 4.71875,
      "rewards/rejected": -13.0,
      "step": 900
    },
    {
      "epoch": 0.11630136110933606,
      "grad_norm": 14.134132471585803,
      "learning_rate": 2.6519741765271837e-06,
      "logits/chosen": 0.08251953125,
      "logits/rejected": 0.68359375,
      "logps/chosen": -366.0,
      "logps/rejected": -370.0,
      "loss": 0.278,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -7.90625,
      "rewards/margins": 4.53125,
      "rewards/rejected": -12.4375,
      "step": 910
    },
    {
      "epoch": 0.11757939804460349,
      "grad_norm": 13.263542183883226,
      "learning_rate": 2.637521893583148e-06,
      "logits/chosen": 0.2578125,
      "logits/rejected": 1.0,
      "logps/chosen": -342.0,
      "logps/rejected": -332.0,
      "loss": 0.3354,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -7.8125,
      "rewards/margins": 4.75,
      "rewards/rejected": -12.5625,
      "step": 920
    },
    {
      "epoch": 0.11885743497987092,
      "grad_norm": 9.948649799485873,
      "learning_rate": 2.6233033431358115e-06,
      "logits/chosen": 0.314453125,
      "logits/rejected": 0.87109375,
      "logps/chosen": -348.0,
      "logps/rejected": -368.0,
      "loss": 0.273,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -8.0625,
      "rewards/margins": 4.46875,
      "rewards/rejected": -12.5625,
      "step": 930
    },
    {
      "epoch": 0.12013547191513835,
      "grad_norm": 12.88899904863435,
      "learning_rate": 2.6093122922137685e-06,
      "logits/chosen": 0.349609375,
      "logits/rejected": 0.91015625,
      "logps/chosen": -350.0,
      "logps/rejected": -376.0,
      "loss": 0.2738,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -8.5625,
      "rewards/margins": 4.84375,
      "rewards/rejected": -13.4375,
      "step": 940
    },
    {
      "epoch": 0.12141350885040578,
      "grad_norm": 13.814107015267178,
      "learning_rate": 2.5955427380922006e-06,
      "logits/chosen": 0.37109375,
      "logits/rejected": 1.0546875,
      "logps/chosen": -378.0,
      "logps/rejected": -412.0,
      "loss": 0.2774,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -10.625,
      "rewards/margins": 5.40625,
      "rewards/rejected": -16.0,
      "step": 950
    },
    {
      "epoch": 0.1226915457856732,
      "grad_norm": 9.892996149549207,
      "learning_rate": 2.5819888974716113e-06,
      "logits/chosen": 0.34765625,
      "logits/rejected": 1.0,
      "logps/chosen": -364.0,
      "logps/rejected": -374.0,
      "loss": 0.3011,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -10.8125,
      "rewards/margins": 4.40625,
      "rewards/rejected": -15.25,
      "step": 960
    },
    {
      "epoch": 0.12396958272094064,
      "grad_norm": 12.532214017031396,
      "learning_rate": 2.5686451962717425e-06,
      "logits/chosen": 0.341796875,
      "logits/rejected": 0.76953125,
      "logps/chosen": -358.0,
      "logps/rejected": -380.0,
      "loss": 0.232,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -9.5,
      "rewards/margins": 5.03125,
      "rewards/rejected": -14.5625,
      "step": 970
    },
    {
      "epoch": 0.12524761965620806,
      "grad_norm": 15.176388296719109,
      "learning_rate": 2.5555062599997596e-06,
      "logits/chosen": 0.2255859375,
      "logits/rejected": 0.95703125,
      "logps/chosen": -386.0,
      "logps/rejected": -384.0,
      "loss": 0.2622,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -10.375,
      "rewards/margins": 4.875,
      "rewards/rejected": -15.3125,
      "step": 980
    },
    {
      "epoch": 0.1265256565914755,
      "grad_norm": 9.673481100974936,
      "learning_rate": 2.5425669046549126e-06,
      "logits/chosen": 0.54296875,
      "logits/rejected": 0.953125,
      "logps/chosen": -332.0,
      "logps/rejected": -364.0,
      "loss": 0.2411,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -8.75,
      "rewards/margins": 5.34375,
      "rewards/rejected": -14.125,
      "step": 990
    },
    {
      "epoch": 0.12780369352674292,
      "grad_norm": 43.28828787568225,
      "learning_rate": 2.5298221281347034e-06,
      "logits/chosen": 0.2470703125,
      "logits/rejected": 0.84375,
      "logps/chosen": -372.0,
      "logps/rejected": -388.0,
      "loss": 0.2903,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -10.0,
      "rewards/margins": 5.4375,
      "rewards/rejected": -15.4375,
      "step": 1000
    },
    {
      "epoch": 0.12908173046201035,
      "grad_norm": 11.953553681695043,
      "learning_rate": 2.5172671021102103e-06,
      "logits/chosen": 0.251953125,
      "logits/rejected": 1.0625,
      "logps/chosen": -364.0,
      "logps/rejected": -356.0,
      "loss": 0.2725,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -9.75,
      "rewards/margins": 4.65625,
      "rewards/rejected": -14.4375,
      "step": 1010
    },
    {
      "epoch": 0.13035976739727778,
      "grad_norm": 10.55510744999636,
      "learning_rate": 2.504897164340598e-06,
      "logits/chosen": 0.349609375,
      "logits/rejected": 0.7421875,
      "logps/chosen": -344.0,
      "logps/rejected": -360.0,
      "loss": 0.3023,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -8.9375,
      "rewards/margins": 4.375,
      "rewards/rejected": -13.3125,
      "step": 1020
    },
    {
      "epoch": 0.1316378043325452,
      "grad_norm": 11.107825471840425,
      "learning_rate": 2.492707811399023e-06,
      "logits/chosen": 0.40625,
      "logits/rejected": 0.84765625,
      "logps/chosen": -342.0,
      "logps/rejected": -360.0,
      "loss": 0.2488,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -8.625,
      "rewards/margins": 4.375,
      "rewards/rejected": -13.0,
      "step": 1030
    },
    {
      "epoch": 0.13291584126781264,
      "grad_norm": 9.856931485574579,
      "learning_rate": 2.480694691784169e-06,
      "logits/chosen": 0.310546875,
      "logits/rejected": 0.7578125,
      "logps/chosen": -348.0,
      "logps/rejected": -386.0,
      "loss": 0.2215,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -9.375,
      "rewards/margins": 4.75,
      "rewards/rejected": -14.125,
      "step": 1040
    },
    {
      "epoch": 0.13419387820308007,
      "grad_norm": 11.688881347601551,
      "learning_rate": 2.4688535993934706e-06,
      "logits/chosen": 0.11376953125,
      "logits/rejected": 0.6953125,
      "logps/chosen": -390.0,
      "logps/rejected": -388.0,
      "loss": 0.2608,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -10.0,
      "rewards/margins": 4.71875,
      "rewards/rejected": -14.75,
      "step": 1050
    },
    {
      "epoch": 0.1354719151383475,
      "grad_norm": 10.404930314285414,
      "learning_rate": 2.457180467335805e-06,
      "logits/chosen": 0.1630859375,
      "logits/rejected": 0.69140625,
      "logps/chosen": -374.0,
      "logps/rejected": -380.0,
      "loss": 0.235,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -9.375,
      "rewards/margins": 5.5,
      "rewards/rejected": -14.875,
      "step": 1060
    },
    {
      "epoch": 0.13674995207361493,
      "grad_norm": 7.113796128473081,
      "learning_rate": 2.4456713620629725e-06,
      "logits/chosen": 0.310546875,
      "logits/rejected": 0.81640625,
      "logps/chosen": -378.0,
      "logps/rejected": -382.0,
      "loss": 0.2711,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -10.625,
      "rewards/margins": 4.53125,
      "rewards/rejected": -15.125,
      "step": 1070
    },
    {
      "epoch": 0.13802798900888236,
      "grad_norm": 14.95891466937915,
      "learning_rate": 2.4343224778007378e-06,
      "logits/chosen": 0.1962890625,
      "logits/rejected": 0.81640625,
      "logps/chosen": -360.0,
      "logps/rejected": -386.0,
      "loss": 0.236,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -10.625,
      "rewards/margins": 4.96875,
      "rewards/rejected": -15.625,
      "step": 1080
    },
    {
      "epoch": 0.13930602594414979,
      "grad_norm": 8.562657407914706,
      "learning_rate": 2.4231301312615306e-06,
      "logits/chosen": 0.361328125,
      "logits/rejected": 0.87890625,
      "logps/chosen": -382.0,
      "logps/rejected": -398.0,
      "loss": 0.2737,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -11.0,
      "rewards/margins": 5.25,
      "rewards/rejected": -16.25,
      "step": 1090
    },
    {
      "epoch": 0.14058406287941722,
      "grad_norm": 11.353774611730962,
      "learning_rate": 2.412090756622109e-06,
      "logits/chosen": 0.388671875,
      "logits/rejected": 0.94140625,
      "logps/chosen": -380.0,
      "logps/rejected": -388.0,
      "loss": 0.213,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -11.0,
      "rewards/margins": 5.15625,
      "rewards/rejected": -16.25,
      "step": 1100
    },
    {
      "epoch": 0.14186209981468464,
      "grad_norm": 11.558043187752448,
      "learning_rate": 2.401200900750657e-06,
      "logits/chosen": 0.427734375,
      "logits/rejected": 1.1328125,
      "logps/chosen": -346.0,
      "logps/rejected": -364.0,
      "loss": 0.2718,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -9.625,
      "rewards/margins": 5.625,
      "rewards/rejected": -15.25,
      "step": 1110
    },
    {
      "epoch": 0.14314013674995207,
      "grad_norm": 10.051167628545407,
      "learning_rate": 2.390457218668787e-06,
      "logits/chosen": 0.443359375,
      "logits/rejected": 0.95703125,
      "logps/chosen": -378.0,
      "logps/rejected": -380.0,
      "loss": 0.2349,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -11.25,
      "rewards/margins": 4.4375,
      "rewards/rejected": -15.6875,
      "step": 1120
    },
    {
      "epoch": 0.1444181736852195,
      "grad_norm": 9.62111752846776,
      "learning_rate": 2.379856469234918e-06,
      "logits/chosen": 0.2578125,
      "logits/rejected": 1.046875,
      "logps/chosen": -402.0,
      "logps/rejected": -392.0,
      "loss": 0.2332,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -12.125,
      "rewards/margins": 5.03125,
      "rewards/rejected": -17.25,
      "step": 1130
    },
    {
      "epoch": 0.14569621062048693,
      "grad_norm": 10.478849690548095,
      "learning_rate": 2.369395511036369e-06,
      "logits/chosen": 0.1953125,
      "logits/rejected": 0.828125,
      "logps/chosen": -376.0,
      "logps/rejected": -384.0,
      "loss": 0.2746,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -10.375,
      "rewards/margins": 5.25,
      "rewards/rejected": -15.625,
      "step": 1140
    },
    {
      "epoch": 0.14697424755575436,
      "grad_norm": 9.536913688745726,
      "learning_rate": 2.359071298478354e-06,
      "logits/chosen": 0.275390625,
      "logits/rejected": 0.69140625,
      "logps/chosen": -356.0,
      "logps/rejected": -364.0,
      "loss": 0.2482,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -8.375,
      "rewards/margins": 4.3125,
      "rewards/rejected": -12.75,
      "step": 1150
    },
    {
      "epoch": 0.1482522844910218,
      "grad_norm": 9.79966793976328,
      "learning_rate": 2.3488808780588137e-06,
      "logits/chosen": 0.271484375,
      "logits/rejected": 0.87890625,
      "logps/chosen": -366.0,
      "logps/rejected": -390.0,
      "loss": 0.2463,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -9.875,
      "rewards/margins": 5.1875,
      "rewards/rejected": -15.0625,
      "step": 1160
    },
    {
      "epoch": 0.14953032142628922,
      "grad_norm": 8.636426343933884,
      "learning_rate": 2.3388213848187446e-06,
      "logits/chosen": 0.2890625,
      "logits/rejected": 0.7890625,
      "logps/chosen": -372.0,
      "logps/rejected": -384.0,
      "loss": 0.2273,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -11.25,
      "rewards/margins": 4.625,
      "rewards/rejected": -15.875,
      "step": 1170
    },
    {
      "epoch": 0.15080835836155665,
      "grad_norm": 7.727026741370242,
      "learning_rate": 2.328890038958328e-06,
      "logits/chosen": 0.28125,
      "logits/rejected": 0.81640625,
      "logps/chosen": -372.0,
      "logps/rejected": -384.0,
      "loss": 0.2348,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -10.125,
      "rewards/margins": 4.84375,
      "rewards/rejected": -15.0,
      "step": 1180
    },
    {
      "epoch": 0.15208639529682408,
      "grad_norm": 9.010602964368038,
      "learning_rate": 2.3190841426097937e-06,
      "logits/chosen": 0.408203125,
      "logits/rejected": 0.65625,
      "logps/chosen": -364.0,
      "logps/rejected": -384.0,
      "loss": 0.2597,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -10.625,
      "rewards/margins": 4.875,
      "rewards/rejected": -15.5,
      "step": 1190
    },
    {
      "epoch": 0.1533644322320915,
      "grad_norm": 16.28689878941296,
      "learning_rate": 2.309401076758503e-06,
      "logits/chosen": 0.337890625,
      "logits/rejected": 1.015625,
      "logps/chosen": -374.0,
      "logps/rejected": -376.0,
      "loss": 0.2464,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -9.5,
      "rewards/margins": 5.03125,
      "rewards/rejected": -14.5625,
      "step": 1200
    },
    {
      "epoch": 0.15464246916735894,
      "grad_norm": 8.363359201186526,
      "learning_rate": 2.299838298304276e-06,
      "logits/chosen": 0.259765625,
      "logits/rejected": 0.80078125,
      "logps/chosen": -356.0,
      "logps/rejected": -380.0,
      "loss": 0.2198,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -9.3125,
      "rewards/margins": 5.15625,
      "rewards/rejected": -14.4375,
      "step": 1210
    },
    {
      "epoch": 0.15592050610262637,
      "grad_norm": 6.434324910305247,
      "learning_rate": 2.2903933372554728e-06,
      "logits/chosen": 0.267578125,
      "logits/rejected": 0.80859375,
      "logps/chosen": -350.0,
      "logps/rejected": -382.0,
      "loss": 0.263,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -9.25,
      "rewards/margins": 4.875,
      "rewards/rejected": -14.0625,
      "step": 1220
    },
    {
      "epoch": 0.1571985430378938,
      "grad_norm": 10.399964886904428,
      "learning_rate": 2.281063794048804e-06,
      "logits/chosen": 0.126953125,
      "logits/rejected": 0.66796875,
      "logps/chosen": -358.0,
      "logps/rejected": -392.0,
      "loss": 0.2501,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -8.5,
      "rewards/margins": 4.8125,
      "rewards/rejected": -13.3125,
      "step": 1230
    },
    {
      "epoch": 0.15847657997316122,
      "grad_norm": 10.456212430528767,
      "learning_rate": 2.271847336988259e-06,
      "logits/chosen": 0.416015625,
      "logits/rejected": 0.921875,
      "logps/chosen": -366.0,
      "logps/rejected": -376.0,
      "loss": 0.2246,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -9.625,
      "rewards/margins": 5.375,
      "rewards/rejected": -15.0,
      "step": 1240
    },
    {
      "epoch": 0.15975461690842865,
      "grad_norm": 15.527896349954354,
      "learning_rate": 2.262741699796952e-06,
      "logits/chosen": 0.3671875,
      "logits/rejected": 1.0546875,
      "logps/chosen": -384.0,
      "logps/rejected": -400.0,
      "loss": 0.2387,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -10.6875,
      "rewards/margins": 5.9375,
      "rewards/rejected": -16.625,
      "step": 1250
    },
    {
      "epoch": 0.16103265384369608,
      "grad_norm": 10.176587459415696,
      "learning_rate": 2.253744679276044e-06,
      "logits/chosen": 0.4921875,
      "logits/rejected": 1.0390625,
      "logps/chosen": -378.0,
      "logps/rejected": -394.0,
      "loss": 0.2511,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -11.0625,
      "rewards/margins": 5.21875,
      "rewards/rejected": -16.25,
      "step": 1260
    },
    {
      "epoch": 0.1623106907789635,
      "grad_norm": 8.9437834821559,
      "learning_rate": 2.244854133065255e-06,
      "logits/chosen": 0.427734375,
      "logits/rejected": 1.046875,
      "logps/chosen": -394.0,
      "logps/rejected": -410.0,
      "loss": 0.2291,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -10.875,
      "rewards/margins": 4.96875,
      "rewards/rejected": -15.875,
      "step": 1270
    },
    {
      "epoch": 0.16358872771423094,
      "grad_norm": 6.993496092066978,
      "learning_rate": 2.2360679774997895e-06,
      "logits/chosen": 0.6796875,
      "logits/rejected": 1.1484375,
      "logps/chosen": -366.0,
      "logps/rejected": -366.0,
      "loss": 0.2401,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -10.3125,
      "rewards/margins": 4.40625,
      "rewards/rejected": -14.75,
      "step": 1280
    },
    {
      "epoch": 0.16486676464949837,
      "grad_norm": 9.736897333070834,
      "learning_rate": 2.2273841855588183e-06,
      "logits/chosen": 0.37109375,
      "logits/rejected": 0.890625,
      "logps/chosen": -372.0,
      "logps/rejected": -386.0,
      "loss": 0.2453,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -10.125,
      "rewards/margins": 5.8125,
      "rewards/rejected": -15.9375,
      "step": 1290
    },
    {
      "epoch": 0.1661448015847658,
      "grad_norm": 10.208688395317598,
      "learning_rate": 2.2188007849009167e-06,
      "logits/chosen": 0.357421875,
      "logits/rejected": 0.90625,
      "logps/chosen": -354.0,
      "logps/rejected": -372.0,
      "loss": 0.2257,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -8.75,
      "rewards/margins": 5.25,
      "rewards/rejected": -14.0,
      "step": 1300
    },
    {
      "epoch": 0.16742283852003323,
      "grad_norm": 10.260678977466272,
      "learning_rate": 2.2103158559821502e-06,
      "logits/chosen": 0.408203125,
      "logits/rejected": 0.87890625,
      "logps/chosen": -368.0,
      "logps/rejected": -392.0,
      "loss": 0.2153,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -9.3125,
      "rewards/margins": 5.78125,
      "rewards/rejected": -15.125,
      "step": 1310
    },
    {
      "epoch": 0.16870087545530066,
      "grad_norm": 9.808535437811537,
      "learning_rate": 2.2019275302527213e-06,
      "logits/chosen": 0.5,
      "logits/rejected": 1.046875,
      "logps/chosen": -352.0,
      "logps/rejected": -382.0,
      "loss": 0.2384,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -10.5625,
      "rewards/margins": 4.84375,
      "rewards/rejected": -15.375,
      "step": 1320
    },
    {
      "epoch": 0.1699789123905681,
      "grad_norm": 10.400128745889681,
      "learning_rate": 2.193633988428327e-06,
      "logits/chosen": 0.357421875,
      "logits/rejected": 1.1640625,
      "logps/chosen": -384.0,
      "logps/rejected": -406.0,
      "loss": 0.2377,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -11.8125,
      "rewards/margins": 7.15625,
      "rewards/rejected": -18.875,
      "step": 1330
    },
    {
      "epoch": 0.17125694932583552,
      "grad_norm": 11.709946715496155,
      "learning_rate": 2.185433458832612e-06,
      "logits/chosen": 0.267578125,
      "logits/rejected": 0.828125,
      "logps/chosen": -402.0,
      "logps/rejected": -420.0,
      "loss": 0.2196,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -12.1875,
      "rewards/margins": 5.6875,
      "rewards/rejected": -17.875,
      "step": 1340
    },
    {
      "epoch": 0.17253498626110295,
      "grad_norm": 8.659998254996506,
      "learning_rate": 2.177324215807269e-06,
      "logits/chosen": 0.54296875,
      "logits/rejected": 1.03125,
      "logps/chosen": -406.0,
      "logps/rejected": -430.0,
      "loss": 0.2548,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -12.125,
      "rewards/margins": 6.6875,
      "rewards/rejected": -18.875,
      "step": 1350
    },
    {
      "epoch": 0.17381302319637038,
      "grad_norm": 9.41710582704418,
      "learning_rate": 2.1693045781865616e-06,
      "logits/chosen": 0.439453125,
      "logits/rejected": 0.96875,
      "logps/chosen": -410.0,
      "logps/rejected": -424.0,
      "loss": 0.235,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -11.0625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -17.5,
      "step": 1360
    },
    {
      "epoch": 0.1750910601316378,
      "grad_norm": 8.288006148090526,
      "learning_rate": 2.1613729078331965e-06,
      "logits/chosen": 0.58203125,
      "logits/rejected": 1.25,
      "logps/chosen": -376.0,
      "logps/rejected": -388.0,
      "loss": 0.2008,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -12.125,
      "rewards/margins": 4.90625,
      "rewards/rejected": -17.0,
      "step": 1370
    },
    {
      "epoch": 0.17636909706690523,
      "grad_norm": 6.616544900609601,
      "learning_rate": 2.1535276082326617e-06,
      "logits/chosen": 0.58984375,
      "logits/rejected": 1.046875,
      "logps/chosen": -384.0,
      "logps/rejected": -404.0,
      "loss": 0.2323,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -11.875,
      "rewards/margins": 5.28125,
      "rewards/rejected": -17.125,
      "step": 1380
    },
    {
      "epoch": 0.17764713400217266,
      "grad_norm": 13.348736574474547,
      "learning_rate": 2.14576712314328e-06,
      "logits/chosen": 0.55078125,
      "logits/rejected": 1.171875,
      "logps/chosen": -368.0,
      "logps/rejected": -384.0,
      "loss": 0.2429,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -10.8125,
      "rewards/margins": 5.0625,
      "rewards/rejected": -15.875,
      "step": 1390
    },
    {
      "epoch": 0.1789251709374401,
      "grad_norm": 11.640552621443618,
      "learning_rate": 2.138089935299395e-06,
      "logits/chosen": 0.3125,
      "logits/rejected": 0.95703125,
      "logps/chosen": -420.0,
      "logps/rejected": -422.0,
      "loss": 0.2311,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -12.125,
      "rewards/margins": 5.59375,
      "rewards/rejected": -17.75,
      "step": 1400
    },
    {
      "epoch": 0.18020320787270752,
      "grad_norm": 11.264326998261202,
      "learning_rate": 2.1304945651652297e-06,
      "logits/chosen": 0.51171875,
      "logits/rejected": 1.109375,
      "logps/chosen": -398.0,
      "logps/rejected": -428.0,
      "loss": 0.2215,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -12.25,
      "rewards/margins": 5.53125,
      "rewards/rejected": -17.875,
      "step": 1410
    },
    {
      "epoch": 0.18148124480797495,
      "grad_norm": 9.157537049266336,
      "learning_rate": 2.122979569737101e-06,
      "logits/chosen": 0.5,
      "logits/rejected": 1.140625,
      "logps/chosen": -394.0,
      "logps/rejected": -416.0,
      "loss": 0.1854,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -12.375,
      "rewards/margins": 6.0625,
      "rewards/rejected": -18.375,
      "step": 1420
    },
    {
      "epoch": 0.18275928174324238,
      "grad_norm": 9.53032861640571,
      "learning_rate": 2.11554354139178e-06,
      "logits/chosen": 0.46484375,
      "logits/rejected": 1.09375,
      "logps/chosen": -376.0,
      "logps/rejected": -406.0,
      "loss": 0.2261,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -12.375,
      "rewards/margins": 5.90625,
      "rewards/rejected": -18.25,
      "step": 1430
    },
    {
      "epoch": 0.1840373186785098,
      "grad_norm": 9.673516594776435,
      "learning_rate": 2.1081851067789196e-06,
      "logits/chosen": 0.34765625,
      "logits/rejected": 0.890625,
      "logps/chosen": -412.0,
      "logps/rejected": -434.0,
      "loss": 0.2321,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -13.0,
      "rewards/margins": 5.96875,
      "rewards/rejected": -18.875,
      "step": 1440
    },
    {
      "epoch": 0.18531535561377724,
      "grad_norm": 11.764030057640724,
      "learning_rate": 2.1009029257555606e-06,
      "logits/chosen": 0.404296875,
      "logits/rejected": 0.87109375,
      "logps/chosen": -374.0,
      "logps/rejected": -394.0,
      "loss": 0.2122,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -10.4375,
      "rewards/margins": 4.75,
      "rewards/rejected": -15.25,
      "step": 1450
    },
    {
      "epoch": 0.18659339254904467,
      "grad_norm": 11.155826304078195,
      "learning_rate": 2.0936956903608545e-06,
      "logits/chosen": 0.2734375,
      "logits/rejected": 0.9609375,
      "logps/chosen": -364.0,
      "logps/rejected": -378.0,
      "loss": 0.2532,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -10.5,
      "rewards/margins": 5.0625,
      "rewards/rejected": -15.5625,
      "step": 1460
    },
    {
      "epoch": 0.1878714294843121,
      "grad_norm": 14.23345371198682,
      "learning_rate": 2.0865621238292046e-06,
      "logits/chosen": 0.1689453125,
      "logits/rejected": 0.8515625,
      "logps/chosen": -406.0,
      "logps/rejected": -416.0,
      "loss": 0.2814,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -11.4375,
      "rewards/margins": 5.21875,
      "rewards/rejected": -16.625,
      "step": 1470
    },
    {
      "epoch": 0.18914946641957953,
      "grad_norm": 10.897412595365932,
      "learning_rate": 2.079500979640145e-06,
      "logits/chosen": 0.283203125,
      "logits/rejected": 1.0,
      "logps/chosen": -398.0,
      "logps/rejected": -404.0,
      "loss": 0.2152,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -11.375,
      "rewards/margins": 5.0625,
      "rewards/rejected": -16.375,
      "step": 1480
    },
    {
      "epoch": 0.19042750335484696,
      "grad_norm": 9.409514780720869,
      "learning_rate": 2.072511040603359e-06,
      "logits/chosen": 0.439453125,
      "logits/rejected": 1.078125,
      "logps/chosen": -394.0,
      "logps/rejected": -406.0,
      "loss": 0.2122,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -11.3125,
      "rewards/margins": 6.28125,
      "rewards/rejected": -17.625,
      "step": 1490
    },
    {
      "epoch": 0.19170554029011438,
      "grad_norm": 10.893449322272971,
      "learning_rate": 2.065591117977289e-06,
      "logits/chosen": 0.50390625,
      "logits/rejected": 0.99609375,
      "logps/chosen": -382.0,
      "logps/rejected": -402.0,
      "loss": 0.2199,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -11.75,
      "rewards/margins": 6.0625,
      "rewards/rejected": -17.75,
      "step": 1500
    },
    {
      "epoch": 0.1929835772253818,
      "grad_norm": 9.661869248869076,
      "learning_rate": 2.058740050619915e-06,
      "logits/chosen": 0.388671875,
      "logits/rejected": 1.140625,
      "logps/chosen": -388.0,
      "logps/rejected": -386.0,
      "loss": 0.1943,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -11.6875,
      "rewards/margins": 5.90625,
      "rewards/rejected": -17.625,
      "step": 1510
    },
    {
      "epoch": 0.19426161416064924,
      "grad_norm": 7.772098124342362,
      "learning_rate": 2.0519567041703083e-06,
      "logits/chosen": 0.369140625,
      "logits/rejected": 0.90234375,
      "logps/chosen": -386.0,
      "logps/rejected": -418.0,
      "loss": 0.2397,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -11.3125,
      "rewards/margins": 5.625,
      "rewards/rejected": -16.875,
      "step": 1520
    },
    {
      "epoch": 0.19553965109591667,
      "grad_norm": 8.716523082089784,
      "learning_rate": 2.0452399702596544e-06,
      "logits/chosen": 0.4375,
      "logits/rejected": 1.03125,
      "logps/chosen": -368.0,
      "logps/rejected": -386.0,
      "loss": 0.2204,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -10.875,
      "rewards/margins": 4.78125,
      "rewards/rejected": -15.625,
      "step": 1530
    },
    {
      "epoch": 0.1968176880311841,
      "grad_norm": 10.565297917029323,
      "learning_rate": 2.0385887657505017e-06,
      "logits/chosen": 0.26953125,
      "logits/rejected": 1.0078125,
      "logps/chosen": -390.0,
      "logps/rejected": -394.0,
      "loss": 0.2338,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -10.75,
      "rewards/margins": 5.375,
      "rewards/rejected": -16.125,
      "step": 1540
    },
    {
      "epoch": 0.19809572496645153,
      "grad_norm": 18.753318406301545,
      "learning_rate": 2.032002032003048e-06,
      "logits/chosen": 0.3125,
      "logits/rejected": 0.796875,
      "logps/chosen": -392.0,
      "logps/rejected": -408.0,
      "loss": 0.1873,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -10.3125,
      "rewards/margins": 5.375,
      "rewards/rejected": -15.6875,
      "step": 1550
    },
    {
      "epoch": 0.19937376190171896,
      "grad_norm": 9.751550776871808,
      "learning_rate": 2.025478734167333e-06,
      "logits/chosen": 0.29296875,
      "logits/rejected": 0.9453125,
      "logps/chosen": -380.0,
      "logps/rejected": -402.0,
      "loss": 0.23,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -10.1875,
      "rewards/margins": 5.84375,
      "rewards/rejected": -16.0,
      "step": 1560
    },
    {
      "epoch": 0.2006517988369864,
      "grad_norm": 10.603369776088993,
      "learning_rate": 2.0190178605002747e-06,
      "logits/chosen": 0.484375,
      "logits/rejected": 1.0625,
      "logps/chosen": -386.0,
      "logps/rejected": -406.0,
      "loss": 0.2821,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -11.6875,
      "rewards/margins": 5.71875,
      "rewards/rejected": -17.375,
      "step": 1570
    },
    {
      "epoch": 0.20192983577225382,
      "grad_norm": 10.438658014579774,
      "learning_rate": 2.0126184217065104e-06,
      "logits/chosen": 0.490234375,
      "logits/rejected": 1.3046875,
      "logps/chosen": -388.0,
      "logps/rejected": -398.0,
      "loss": 0.2321,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -12.8125,
      "rewards/margins": 5.09375,
      "rewards/rejected": -17.875,
      "step": 1580
    },
    {
      "epoch": 0.20320787270752125,
      "grad_norm": 10.356330858305313,
      "learning_rate": 2.0062794503020765e-06,
      "logits/chosen": 0.5859375,
      "logits/rejected": 1.0546875,
      "logps/chosen": -390.0,
      "logps/rejected": -394.0,
      "loss": 0.236,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -11.25,
      "rewards/margins": 4.65625,
      "rewards/rejected": -15.9375,
      "step": 1590
    },
    {
      "epoch": 0.20448590964278868,
      "grad_norm": 9.625796873248342,
      "learning_rate": 2e-06,
      "logits/chosen": 0.4765625,
      "logits/rejected": 1.140625,
      "logps/chosen": -376.0,
      "logps/rejected": -392.0,
      "loss": 0.1907,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -11.5625,
      "rewards/margins": 5.4375,
      "rewards/rejected": -17.0,
      "step": 1600
    },
    {
      "epoch": 0.2057639465780561,
      "grad_norm": 56.19431650709996,
      "learning_rate": 1.993779145116907e-06,
      "logits/chosen": 0.53515625,
      "logits/rejected": 1.1328125,
      "logps/chosen": -386.0,
      "logps/rejected": -418.0,
      "loss": 0.2581,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -12.5625,
      "rewards/margins": 5.9375,
      "rewards/rejected": -18.5,
      "step": 1610
    },
    {
      "epoch": 0.20704198351332354,
      "grad_norm": 6.775382702080136,
      "learning_rate": 1.987615979999813e-06,
      "logits/chosen": 0.6171875,
      "logits/rejected": 1.0703125,
      "logps/chosen": -388.0,
      "logps/rejected": -452.0,
      "loss": 0.2257,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -12.125,
      "rewards/margins": 5.9375,
      "rewards/rejected": -18.125,
      "step": 1620
    },
    {
      "epoch": 0.20832002044859096,
      "grad_norm": 11.355781432791957,
      "learning_rate": 1.9815096184722797e-06,
      "logits/chosen": 0.6171875,
      "logits/rejected": 1.21875,
      "logps/chosen": -384.0,
      "logps/rejected": -396.0,
      "loss": 0.2199,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -11.8125,
      "rewards/margins": 5.28125,
      "rewards/rejected": -17.125,
      "step": 1630
    },
    {
      "epoch": 0.2095980573838584,
      "grad_norm": 8.536420741475725,
      "learning_rate": 1.9754591932991793e-06,
      "logits/chosen": 0.498046875,
      "logits/rejected": 1.1015625,
      "logps/chosen": -392.0,
      "logps/rejected": -422.0,
      "loss": 0.1868,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -11.625,
      "rewards/margins": 6.25,
      "rewards/rejected": -17.875,
      "step": 1640
    },
    {
      "epoch": 0.21087609431912582,
      "grad_norm": 7.837326679562043,
      "learning_rate": 1.9694638556693235e-06,
      "logits/chosen": 0.6875,
      "logits/rejected": 1.296875,
      "logps/chosen": -392.0,
      "logps/rejected": -400.0,
      "loss": 0.1942,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -12.625,
      "rewards/margins": 6.25,
      "rewards/rejected": -18.875,
      "step": 1650
    },
    {
      "epoch": 0.21215413125439325,
      "grad_norm": 10.626949014519813,
      "learning_rate": 1.963522774695264e-06,
      "logits/chosen": 0.330078125,
      "logits/rejected": 0.9453125,
      "logps/chosen": -398.0,
      "logps/rejected": -416.0,
      "loss": 0.2296,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -11.875,
      "rewards/margins": 5.78125,
      "rewards/rejected": -17.625,
      "step": 1660
    },
    {
      "epoch": 0.21343216818966068,
      "grad_norm": 8.324088637928815,
      "learning_rate": 1.9576351369295853e-06,
      "logits/chosen": 0.357421875,
      "logits/rejected": 0.890625,
      "logps/chosen": -404.0,
      "logps/rejected": -428.0,
      "loss": 0.1847,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -11.5625,
      "rewards/margins": 5.59375,
      "rewards/rejected": -17.125,
      "step": 1670
    },
    {
      "epoch": 0.2147102051249281,
      "grad_norm": 9.391802087767184,
      "learning_rate": 1.951800145897066e-06,
      "logits/chosen": 0.51171875,
      "logits/rejected": 1.203125,
      "logps/chosen": -350.0,
      "logps/rejected": -352.0,
      "loss": 0.2347,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -9.9375,
      "rewards/margins": 5.125,
      "rewards/rejected": -15.125,
      "step": 1680
    },
    {
      "epoch": 0.21598824206019554,
      "grad_norm": 6.654802785516758,
      "learning_rate": 1.9460170216420797e-06,
      "logits/chosen": 0.353515625,
      "logits/rejected": 0.87890625,
      "logps/chosen": -370.0,
      "logps/rejected": -366.0,
      "loss": 0.2065,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -9.1875,
      "rewards/margins": 4.71875,
      "rewards/rejected": -13.875,
      "step": 1690
    },
    {
      "epoch": 0.21726627899546297,
      "grad_norm": 11.666865270866497,
      "learning_rate": 1.9402850002906637e-06,
      "logits/chosen": 0.369140625,
      "logits/rejected": 1.046875,
      "logps/chosen": -376.0,
      "logps/rejected": -388.0,
      "loss": 0.2384,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -11.0,
      "rewards/margins": 5.40625,
      "rewards/rejected": -16.375,
      "step": 1700
    },
    {
      "epoch": 0.2185443159307304,
      "grad_norm": 8.194573629374778,
      "learning_rate": 1.9346033336266974e-06,
      "logits/chosen": 0.58984375,
      "logits/rejected": 1.046875,
      "logps/chosen": -392.0,
      "logps/rejected": -408.0,
      "loss": 0.2274,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -12.0625,
      "rewards/margins": 5.15625,
      "rewards/rejected": -17.25,
      "step": 1710
    },
    {
      "epoch": 0.21982235286599783,
      "grad_norm": 6.593621490273498,
      "learning_rate": 1.9289712886816486e-06,
      "logits/chosen": 0.390625,
      "logits/rejected": 1.046875,
      "logps/chosen": -382.0,
      "logps/rejected": -400.0,
      "loss": 0.1884,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -10.9375,
      "rewards/margins": 5.8125,
      "rewards/rejected": -16.75,
      "step": 1720
    },
    {
      "epoch": 0.22110038980126526,
      "grad_norm": 12.07847111692444,
      "learning_rate": 1.92338814733738e-06,
      "logits/chosen": 0.5390625,
      "logits/rejected": 0.99609375,
      "logps/chosen": -374.0,
      "logps/rejected": -398.0,
      "loss": 0.2228,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -11.3125,
      "rewards/margins": 5.71875,
      "rewards/rejected": -17.0,
      "step": 1730
    },
    {
      "epoch": 0.22237842673653269,
      "grad_norm": 13.527321618937918,
      "learning_rate": 1.9178532059415367e-06,
      "logits/chosen": 0.416015625,
      "logits/rejected": 1.015625,
      "logps/chosen": -416.0,
      "logps/rejected": -432.0,
      "loss": 0.1988,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -13.9375,
      "rewards/margins": 5.75,
      "rewards/rejected": -19.75,
      "step": 1740
    },
    {
      "epoch": 0.22365646367180012,
      "grad_norm": 8.996443187662518,
      "learning_rate": 1.9123657749350298e-06,
      "logits/chosen": 0.447265625,
      "logits/rejected": 1.0078125,
      "logps/chosen": -382.0,
      "logps/rejected": -386.0,
      "loss": 0.2362,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -11.9375,
      "rewards/margins": 5.28125,
      "rewards/rejected": -17.25,
      "step": 1750
    },
    {
      "epoch": 0.22493450060706754,
      "grad_norm": 6.876056068057693,
      "learning_rate": 1.9069251784911844e-06,
      "logits/chosen": 0.349609375,
      "logits/rejected": 0.8984375,
      "logps/chosen": -428.0,
      "logps/rejected": -430.0,
      "loss": 0.2166,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -14.1875,
      "rewards/margins": 4.9375,
      "rewards/rejected": -19.125,
      "step": 1760
    },
    {
      "epoch": 0.22621253754233497,
      "grad_norm": 5.407313159684597,
      "learning_rate": 1.9015307541661132e-06,
      "logits/chosen": 0.328125,
      "logits/rejected": 1.0859375,
      "logps/chosen": -422.0,
      "logps/rejected": -436.0,
      "loss": 0.1755,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -13.625,
      "rewards/margins": 5.5625,
      "rewards/rejected": -19.125,
      "step": 1770
    },
    {
      "epoch": 0.2274905744776024,
      "grad_norm": 5.651213400035844,
      "learning_rate": 1.8961818525599089e-06,
      "logits/chosen": 0.482421875,
      "logits/rejected": 1.0,
      "logps/chosen": -390.0,
      "logps/rejected": -422.0,
      "loss": 0.2067,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -12.25,
      "rewards/margins": 5.34375,
      "rewards/rejected": -17.625,
      "step": 1780
    },
    {
      "epoch": 0.22876861141286983,
      "grad_norm": 8.420064042520776,
      "learning_rate": 1.890877836988262e-06,
      "logits/chosen": 0.423828125,
      "logits/rejected": 1.078125,
      "logps/chosen": -380.0,
      "logps/rejected": -416.0,
      "loss": 0.1853,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -11.625,
      "rewards/margins": 6.625,
      "rewards/rejected": -18.25,
      "step": 1790
    },
    {
      "epoch": 0.23004664834813726,
      "grad_norm": 9.845876786996106,
      "learning_rate": 1.8856180831641269e-06,
      "logits/chosen": 0.42578125,
      "logits/rejected": 1.0703125,
      "logps/chosen": -420.0,
      "logps/rejected": -420.0,
      "loss": 0.2063,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -13.0625,
      "rewards/margins": 5.25,
      "rewards/rejected": -18.25,
      "step": 1800
    },
    {
      "epoch": 0.2313246852834047,
      "grad_norm": 6.2780087333278765,
      "learning_rate": 1.8804019788890737e-06,
      "logits/chosen": 0.5,
      "logits/rejected": 1.1953125,
      "logps/chosen": -384.0,
      "logps/rejected": -392.0,
      "loss": 0.182,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -11.0625,
      "rewards/margins": 5.625,
      "rewards/rejected": -16.625,
      "step": 1810
    },
    {
      "epoch": 0.23260272221867212,
      "grad_norm": 9.456042007005376,
      "learning_rate": 1.8752289237539816e-06,
      "logits/chosen": 0.478515625,
      "logits/rejected": 0.78125,
      "logps/chosen": -370.0,
      "logps/rejected": -402.0,
      "loss": 0.1936,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -11.25,
      "rewards/margins": 5.5625,
      "rewards/rejected": -16.75,
      "step": 1820
    },
    {
      "epoch": 0.23388075915393955,
      "grad_norm": 14.671541103050929,
      "learning_rate": 1.8700983288487376e-06,
      "logits/chosen": 0.4765625,
      "logits/rejected": 1.1328125,
      "logps/chosen": -388.0,
      "logps/rejected": -420.0,
      "loss": 0.2058,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -12.1875,
      "rewards/margins": 5.65625,
      "rewards/rejected": -17.875,
      "step": 1830
    },
    {
      "epoch": 0.23515879608920698,
      "grad_norm": 10.249547357691537,
      "learning_rate": 1.8650096164806275e-06,
      "logits/chosen": 0.46875,
      "logits/rejected": 1.078125,
      "logps/chosen": -388.0,
      "logps/rejected": -408.0,
      "loss": 0.1843,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -11.9375,
      "rewards/margins": 6.15625,
      "rewards/rejected": -18.125,
      "step": 1840
    },
    {
      "epoch": 0.2364368330244744,
      "grad_norm": 8.446908830272301,
      "learning_rate": 1.8599622199011084e-06,
      "logits/chosen": 0.38671875,
      "logits/rejected": 1.1953125,
      "logps/chosen": -400.0,
      "logps/rejected": -416.0,
      "loss": 0.1977,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -12.375,
      "rewards/margins": 6.25,
      "rewards/rejected": -18.625,
      "step": 1850
    },
    {
      "epoch": 0.23771486995974184,
      "grad_norm": 10.340484868236555,
      "learning_rate": 1.854955583040673e-06,
      "logits/chosen": 0.38671875,
      "logits/rejected": 1.1015625,
      "logps/chosen": -402.0,
      "logps/rejected": -394.0,
      "loss": 0.21,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -11.75,
      "rewards/margins": 5.40625,
      "rewards/rejected": -17.125,
      "step": 1860
    },
    {
      "epoch": 0.23899290689500927,
      "grad_norm": 11.32643539399373,
      "learning_rate": 1.849989160251521e-06,
      "logits/chosen": 0.5625,
      "logits/rejected": 1.15625,
      "logps/chosen": -382.0,
      "logps/rejected": -424.0,
      "loss": 0.2056,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -11.5,
      "rewards/margins": 5.3125,
      "rewards/rejected": -16.75,
      "step": 1870
    },
    {
      "epoch": 0.2402709438302767,
      "grad_norm": 7.80126499237845,
      "learning_rate": 1.8450624160577701e-06,
      "logits/chosen": 0.396484375,
      "logits/rejected": 1.1015625,
      "logps/chosen": -400.0,
      "logps/rejected": -422.0,
      "loss": 0.1866,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -12.4375,
      "rewards/margins": 6.375,
      "rewards/rejected": -18.75,
      "step": 1880
    },
    {
      "epoch": 0.24154898076554412,
      "grad_norm": 8.639669238463219,
      "learning_rate": 1.8401748249129445e-06,
      "logits/chosen": 0.55859375,
      "logits/rejected": 1.3984375,
      "logps/chosen": -384.0,
      "logps/rejected": -398.0,
      "loss": 0.2089,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -12.4375,
      "rewards/margins": 6.4375,
      "rewards/rejected": -18.875,
      "step": 1890
    },
    {
      "epoch": 0.24282701770081155,
      "grad_norm": 12.571808439414198,
      "learning_rate": 1.8353258709644938e-06,
      "logits/chosen": 0.60546875,
      "logits/rejected": 1.3515625,
      "logps/chosen": -406.0,
      "logps/rejected": -410.0,
      "loss": 0.2405,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -14.1875,
      "rewards/margins": 5.4375,
      "rewards/rejected": -19.625,
      "step": 1900
    },
    {
      "epoch": 0.24410505463607898,
      "grad_norm": 10.022504407927672,
      "learning_rate": 1.830515047825102e-06,
      "logits/chosen": 0.6171875,
      "logits/rejected": 1.203125,
      "logps/chosen": -392.0,
      "logps/rejected": -402.0,
      "loss": 0.2349,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -11.625,
      "rewards/margins": 5.125,
      "rewards/rejected": -16.75,
      "step": 1910
    },
    {
      "epoch": 0.2453830915713464,
      "grad_norm": 11.983619150767659,
      "learning_rate": 1.8257418583505536e-06,
      "logits/chosen": 0.48046875,
      "logits/rejected": 1.140625,
      "logps/chosen": -386.0,
      "logps/rejected": -376.0,
      "loss": 0.2276,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -9.75,
      "rewards/margins": 4.59375,
      "rewards/rejected": -14.375,
      "step": 1920
    },
    {
      "epoch": 0.24666112850661384,
      "grad_norm": 10.013229529395735,
      "learning_rate": 1.8210058144239416e-06,
      "logits/chosen": 0.466796875,
      "logits/rejected": 1.0234375,
      "logps/chosen": -378.0,
      "logps/rejected": -378.0,
      "loss": 0.2192,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -9.9375,
      "rewards/margins": 4.875,
      "rewards/rejected": -14.75,
      "step": 1930
    },
    {
      "epoch": 0.24793916544188127,
      "grad_norm": 9.869246155146344,
      "learning_rate": 1.816306436745999e-06,
      "logits/chosen": 0.38671875,
      "logits/rejected": 0.8828125,
      "logps/chosen": -360.0,
      "logps/rejected": -384.0,
      "loss": 0.2291,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -9.375,
      "rewards/margins": 5.34375,
      "rewards/rejected": -14.75,
      "step": 1940
    },
    {
      "epoch": 0.2492172023771487,
      "grad_norm": 9.752026338549422,
      "learning_rate": 1.8116432546313529e-06,
      "logits/chosen": 0.51171875,
      "logits/rejected": 0.984375,
      "logps/chosen": -372.0,
      "logps/rejected": -394.0,
      "loss": 0.2135,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -9.75,
      "rewards/margins": 5.59375,
      "rewards/rejected": -15.375,
      "step": 1950
    },
    {
      "epoch": 0.25049523931241613,
      "grad_norm": 6.5544574140785485,
      "learning_rate": 1.8070158058105026e-06,
      "logits/chosen": 0.478515625,
      "logits/rejected": 1.03125,
      "logps/chosen": -362.0,
      "logps/rejected": -366.0,
      "loss": 0.2239,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -10.0,
      "rewards/margins": 4.8125,
      "rewards/rejected": -14.8125,
      "step": 1960
    },
    {
      "epoch": 0.25177327624768353,
      "grad_norm": 9.34575327164808,
      "learning_rate": 1.8024236362373315e-06,
      "logits/chosen": 0.349609375,
      "logits/rejected": 1.1171875,
      "logps/chosen": -358.0,
      "logps/rejected": -368.0,
      "loss": 0.2099,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -9.5625,
      "rewards/margins": 6.0625,
      "rewards/rejected": -15.625,
      "step": 1970
    },
    {
      "epoch": 0.253051313182951,
      "grad_norm": 10.997727931514152,
      "learning_rate": 1.7978662999019787e-06,
      "logits/chosen": 0.392578125,
      "logits/rejected": 0.78125,
      "logps/chosen": -356.0,
      "logps/rejected": -396.0,
      "loss": 0.2221,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -10.125,
      "rewards/margins": 5.1875,
      "rewards/rejected": -15.3125,
      "step": 1980
    },
    {
      "epoch": 0.2543293501182184,
      "grad_norm": 13.000707469514587,
      "learning_rate": 1.793343358648881e-06,
      "logits/chosen": 0.4375,
      "logits/rejected": 0.88671875,
      "logps/chosen": -362.0,
      "logps/rejected": -398.0,
      "loss": 0.2272,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -9.6875,
      "rewards/margins": 5.28125,
      "rewards/rejected": -14.9375,
      "step": 1990
    },
    {
      "epoch": 0.25560738705348585,
      "grad_norm": 10.03385120671223,
      "learning_rate": 1.7888543819998317e-06,
      "logits/chosen": 0.365234375,
      "logits/rejected": 0.96484375,
      "logps/chosen": -364.0,
      "logps/rejected": -390.0,
      "loss": 0.2098,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -9.4375,
      "rewards/margins": 5.75,
      "rewards/rejected": -15.1875,
      "step": 2000
    },
    {
      "epoch": 0.25688542398875325,
      "grad_norm": 12.382933145010032,
      "learning_rate": 1.7843989469818819e-06,
      "logits/chosen": 0.412109375,
      "logits/rejected": 0.859375,
      "logps/chosen": -370.0,
      "logps/rejected": -394.0,
      "loss": 0.2108,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -9.0,
      "rewards/margins": 7.03125,
      "rewards/rejected": -16.0,
      "step": 2010
    },
    {
      "epoch": 0.2581634609240207,
      "grad_norm": 9.69861435637343,
      "learning_rate": 1.779976637959939e-06,
      "logits/chosen": 0.2392578125,
      "logits/rejected": 0.82421875,
      "logps/chosen": -374.0,
      "logps/rejected": -390.0,
      "loss": 0.1999,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -9.75,
      "rewards/margins": 5.53125,
      "rewards/rejected": -15.3125,
      "step": 2020
    },
    {
      "epoch": 0.2594414978592881,
      "grad_norm": 8.03317969262773,
      "learning_rate": 1.7755870464739012e-06,
      "logits/chosen": 0.396484375,
      "logits/rejected": 1.0625,
      "logps/chosen": -362.0,
      "logps/rejected": -394.0,
      "loss": 0.1932,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -10.4375,
      "rewards/margins": 6.09375,
      "rewards/rejected": -16.5,
      "step": 2030
    },
    {
      "epoch": 0.26071953479455556,
      "grad_norm": 9.066287607764782,
      "learning_rate": 1.7712297710801907e-06,
      "logits/chosen": 0.6015625,
      "logits/rejected": 1.140625,
      "logps/chosen": -374.0,
      "logps/rejected": -398.0,
      "loss": 0.2183,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -11.25,
      "rewards/margins": 5.34375,
      "rewards/rejected": -16.625,
      "step": 2040
    },
    {
      "epoch": 0.26199757172982296,
      "grad_norm": 11.2882279001223,
      "learning_rate": 1.7669044171975444e-06,
      "logits/chosen": 0.490234375,
      "logits/rejected": 1.1171875,
      "logps/chosen": -400.0,
      "logps/rejected": -408.0,
      "loss": 0.2023,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -11.0,
      "rewards/margins": 5.8125,
      "rewards/rejected": -16.75,
      "step": 2050
    },
    {
      "epoch": 0.2632756086650904,
      "grad_norm": 7.562140331091214,
      "learning_rate": 1.7626105969569268e-06,
      "logits/chosen": 0.365234375,
      "logits/rejected": 0.8359375,
      "logps/chosen": -386.0,
      "logps/rejected": -398.0,
      "loss": 0.2117,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -10.125,
      "rewards/margins": 5.375,
      "rewards/rejected": -15.5,
      "step": 2060
    },
    {
      "epoch": 0.2645536456003578,
      "grad_norm": 6.738006348102007,
      "learning_rate": 1.758347929055432e-06,
      "logits/chosen": 0.353515625,
      "logits/rejected": 0.890625,
      "logps/chosen": -366.0,
      "logps/rejected": -414.0,
      "loss": 0.2111,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -9.3125,
      "rewards/margins": 6.125,
      "rewards/rejected": -15.4375,
      "step": 2070
    },
    {
      "epoch": 0.2658316825356253,
      "grad_norm": 8.552524446882419,
      "learning_rate": 1.7541160386140582e-06,
      "logits/chosen": 0.404296875,
      "logits/rejected": 1.1015625,
      "logps/chosen": -386.0,
      "logps/rejected": -368.0,
      "loss": 0.2056,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -9.6875,
      "rewards/margins": 5.46875,
      "rewards/rejected": -15.1875,
      "step": 2080
    },
    {
      "epoch": 0.2671097194708927,
      "grad_norm": 10.544988748330061,
      "learning_rate": 1.7499145570392284e-06,
      "logits/chosen": 0.416015625,
      "logits/rejected": 1.09375,
      "logps/chosen": -372.0,
      "logps/rejected": -374.0,
      "loss": 0.1973,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -10.1875,
      "rewards/margins": 5.625,
      "rewards/rejected": -15.8125,
      "step": 2090
    },
    {
      "epoch": 0.26838775640616014,
      "grad_norm": 10.400154068318201,
      "learning_rate": 1.745743121887939e-06,
      "logits/chosen": 0.4296875,
      "logits/rejected": 1.1015625,
      "logps/chosen": -380.0,
      "logps/rejected": -390.0,
      "loss": 0.1971,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -10.375,
      "rewards/margins": 5.9375,
      "rewards/rejected": -16.25,
      "step": 2100
    },
    {
      "epoch": 0.26966579334142754,
      "grad_norm": 12.849868803613106,
      "learning_rate": 1.7416013767364324e-06,
      "logits/chosen": 0.65625,
      "logits/rejected": 1.171875,
      "logps/chosen": -388.0,
      "logps/rejected": -398.0,
      "loss": 0.1993,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -11.9375,
      "rewards/margins": 5.5625,
      "rewards/rejected": -17.5,
      "step": 2110
    },
    {
      "epoch": 0.270943830276695,
      "grad_norm": 10.134753785464733,
      "learning_rate": 1.7374889710522776e-06,
      "logits/chosen": 0.60546875,
      "logits/rejected": 1.1875,
      "logps/chosen": -362.0,
      "logps/rejected": -406.0,
      "loss": 0.2073,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -9.875,
      "rewards/margins": 6.03125,
      "rewards/rejected": -15.875,
      "step": 2120
    },
    {
      "epoch": 0.2722218672119624,
      "grad_norm": 5.975884503503696,
      "learning_rate": 1.7334055600697579e-06,
      "logits/chosen": 0.375,
      "logits/rejected": 1.1015625,
      "logps/chosen": -398.0,
      "logps/rejected": -408.0,
      "loss": 0.163,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -11.5,
      "rewards/margins": 6.40625,
      "rewards/rejected": -17.875,
      "step": 2130
    },
    {
      "epoch": 0.27349990414722986,
      "grad_norm": 8.969952975822274,
      "learning_rate": 1.7293508046684678e-06,
      "logits/chosen": 0.671875,
      "logits/rejected": 1.2890625,
      "logps/chosen": -372.0,
      "logps/rejected": -408.0,
      "loss": 0.1836,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -12.0,
      "rewards/margins": 5.78125,
      "rewards/rejected": -17.75,
      "step": 2140
    },
    {
      "epoch": 0.27477794108249726,
      "grad_norm": 8.260418148523808,
      "learning_rate": 1.7253243712550145e-06,
      "logits/chosen": 0.62890625,
      "logits/rejected": 1.25,
      "logps/chosen": -400.0,
      "logps/rejected": -428.0,
      "loss": 0.1826,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -12.5625,
      "rewards/margins": 6.1875,
      "rewards/rejected": -18.75,
      "step": 2150
    },
    {
      "epoch": 0.2760559780177647,
      "grad_norm": 10.112611971792418,
      "learning_rate": 1.7213259316477406e-06,
      "logits/chosen": 0.5703125,
      "logits/rejected": 1.0859375,
      "logps/chosen": -408.0,
      "logps/rejected": -440.0,
      "loss": 0.235,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -12.9375,
      "rewards/margins": 6.1875,
      "rewards/rejected": -19.125,
      "step": 2160
    },
    {
      "epoch": 0.2773340149530321,
      "grad_norm": 12.196146581904246,
      "learning_rate": 1.7173551629643674e-06,
      "logits/chosen": 0.61328125,
      "logits/rejected": 1.25,
      "logps/chosen": -380.0,
      "logps/rejected": -400.0,
      "loss": 0.1953,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -11.9375,
      "rewards/margins": 5.6875,
      "rewards/rejected": -17.625,
      "step": 2170
    },
    {
      "epoch": 0.27861205188829957,
      "grad_norm": 11.582179520164495,
      "learning_rate": 1.713411747512477e-06,
      "logits/chosen": 0.6484375,
      "logits/rejected": 1.296875,
      "logps/chosen": -404.0,
      "logps/rejected": -426.0,
      "loss": 0.219,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -12.5,
      "rewards/margins": 6.09375,
      "rewards/rejected": -18.625,
      "step": 2180
    },
    {
      "epoch": 0.279890088823567,
      "grad_norm": 10.247754097874962,
      "learning_rate": 1.709495372682753e-06,
      "logits/chosen": 0.59765625,
      "logits/rejected": 1.1953125,
      "logps/chosen": -410.0,
      "logps/rejected": -432.0,
      "loss": 0.216,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -14.0625,
      "rewards/margins": 5.625,
      "rewards/rejected": -19.75,
      "step": 2190
    },
    {
      "epoch": 0.28116812575883443,
      "grad_norm": 7.642903889795623,
      "learning_rate": 1.7056057308448832e-06,
      "logits/chosen": 0.439453125,
      "logits/rejected": 1.0546875,
      "logps/chosen": -382.0,
      "logps/rejected": -398.0,
      "loss": 0.1638,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -11.125,
      "rewards/margins": 5.5,
      "rewards/rejected": -16.625,
      "step": 2200
    },
    {
      "epoch": 0.28244616269410183,
      "grad_norm": 6.351396104674788,
      "learning_rate": 1.701742519246068e-06,
      "logits/chosen": 0.341796875,
      "logits/rejected": 0.953125,
      "logps/chosen": -404.0,
      "logps/rejected": -432.0,
      "loss": 0.1634,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -12.5625,
      "rewards/margins": 6.875,
      "rewards/rejected": -19.375,
      "step": 2210
    },
    {
      "epoch": 0.2837241996293693,
      "grad_norm": 8.092904427079562,
      "learning_rate": 1.6979054399120355e-06,
      "logits/chosen": 0.30859375,
      "logits/rejected": 0.9375,
      "logps/chosen": -394.0,
      "logps/rejected": -418.0,
      "loss": 0.2136,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -11.9375,
      "rewards/margins": 6.71875,
      "rewards/rejected": -18.625,
      "step": 2220
    },
    {
      "epoch": 0.2850022365646367,
      "grad_norm": 7.866533998279605,
      "learning_rate": 1.6940941995505069e-06,
      "logits/chosen": 0.36328125,
      "logits/rejected": 1.015625,
      "logps/chosen": -388.0,
      "logps/rejected": -398.0,
      "loss": 0.2025,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -11.375,
      "rewards/margins": 5.5625,
      "rewards/rejected": -17.0,
      "step": 2230
    },
    {
      "epoch": 0.28628027349990415,
      "grad_norm": 9.125191797058465,
      "learning_rate": 1.6903085094570331e-06,
      "logits/chosen": 0.34375,
      "logits/rejected": 0.82421875,
      "logps/chosen": -394.0,
      "logps/rejected": -424.0,
      "loss": 0.1916,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -11.8125,
      "rewards/margins": 5.59375,
      "rewards/rejected": -17.375,
      "step": 2240
    },
    {
      "epoch": 0.28755831043517155,
      "grad_norm": 12.944444743152847,
      "learning_rate": 1.6865480854231356e-06,
      "logits/chosen": 0.48046875,
      "logits/rejected": 1.078125,
      "logps/chosen": -376.0,
      "logps/rejected": -394.0,
      "loss": 0.1975,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -10.3125,
      "rewards/margins": 6.3125,
      "rewards/rejected": -16.625,
      "step": 2250
    },
    {
      "epoch": 0.288836347370439,
      "grad_norm": 8.923753690921213,
      "learning_rate": 1.682812647646685e-06,
      "logits/chosen": 0.4453125,
      "logits/rejected": 0.9921875,
      "logps/chosen": -344.0,
      "logps/rejected": -372.0,
      "loss": 0.1584,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -9.25,
      "rewards/margins": 5.625,
      "rewards/rejected": -14.875,
      "step": 2260
    },
    {
      "epoch": 0.2901143843057064,
      "grad_norm": 9.267592885957715,
      "learning_rate": 1.6791019206444541e-06,
      "logits/chosen": 0.330078125,
      "logits/rejected": 1.09375,
      "logps/chosen": -388.0,
      "logps/rejected": -384.0,
      "loss": 0.1707,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -10.75,
      "rewards/margins": 5.90625,
      "rewards/rejected": -16.625,
      "step": 2270
    },
    {
      "epoch": 0.29139242124097386,
      "grad_norm": 6.26502692899196,
      "learning_rate": 1.675415633166782e-06,
      "logits/chosen": 0.6015625,
      "logits/rejected": 1.1875,
      "logps/chosen": -384.0,
      "logps/rejected": -402.0,
      "loss": 0.177,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -11.6875,
      "rewards/margins": 6.15625,
      "rewards/rejected": -17.875,
      "step": 2280
    },
    {
      "epoch": 0.29267045817624127,
      "grad_norm": 10.762460243795994,
      "learning_rate": 1.6717535181142914e-06,
      "logits/chosen": 0.2734375,
      "logits/rejected": 0.70703125,
      "logps/chosen": -412.0,
      "logps/rejected": -444.0,
      "loss": 0.21,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -12.1875,
      "rewards/margins": 6.53125,
      "rewards/rejected": -18.75,
      "step": 2290
    },
    {
      "epoch": 0.2939484951115087,
      "grad_norm": 8.630817615013783,
      "learning_rate": 1.668115312456598e-06,
      "logits/chosen": 0.3828125,
      "logits/rejected": 0.83203125,
      "logps/chosen": -398.0,
      "logps/rejected": -434.0,
      "loss": 0.1418,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -12.1875,
      "rewards/margins": 7.3125,
      "rewards/rejected": -19.5,
      "step": 2300
    },
    {
      "epoch": 0.2952265320467761,
      "grad_norm": 7.6703160331800335,
      "learning_rate": 1.6645007571529578e-06,
      "logits/chosen": 0.55078125,
      "logits/rejected": 1.2578125,
      "logps/chosen": -378.0,
      "logps/rejected": -398.0,
      "loss": 0.1841,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -12.5,
      "rewards/margins": 6.0625,
      "rewards/rejected": -18.5,
      "step": 2310
    },
    {
      "epoch": 0.2965045689820436,
      "grad_norm": 10.943031979197396,
      "learning_rate": 1.6609095970747992e-06,
      "logits/chosen": 0.427734375,
      "logits/rejected": 0.95703125,
      "logps/chosen": -390.0,
      "logps/rejected": -394.0,
      "loss": 0.2381,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -11.125,
      "rewards/margins": 5.8125,
      "rewards/rejected": -17.0,
      "step": 2320
    },
    {
      "epoch": 0.297782605917311,
      "grad_norm": 7.7998341966043006,
      "learning_rate": 1.6573415809300833e-06,
      "logits/chosen": 0.58203125,
      "logits/rejected": 1.1484375,
      "logps/chosen": -372.0,
      "logps/rejected": -414.0,
      "loss": 0.176,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -11.1875,
      "rewards/margins": 6.03125,
      "rewards/rejected": -17.25,
      "step": 2330
    },
    {
      "epoch": 0.29906064285257844,
      "grad_norm": 10.47807805395261,
      "learning_rate": 1.6537964611894462e-06,
      "logits/chosen": 0.609375,
      "logits/rejected": 1.1953125,
      "logps/chosen": -388.0,
      "logps/rejected": -402.0,
      "loss": 0.1744,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -11.5,
      "rewards/margins": 5.59375,
      "rewards/rejected": -17.125,
      "step": 2340
    },
    {
      "epoch": 0.30033867978784584,
      "grad_norm": 9.90326865169702,
      "learning_rate": 1.6502739940140692e-06,
      "logits/chosen": 0.341796875,
      "logits/rejected": 0.96875,
      "logps/chosen": -408.0,
      "logps/rejected": -430.0,
      "loss": 0.2172,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -12.25,
      "rewards/margins": 6.40625,
      "rewards/rejected": -18.625,
      "step": 2350
    },
    {
      "epoch": 0.3016167167231133,
      "grad_norm": 10.636770768123503,
      "learning_rate": 1.6467739391852364e-06,
      "logits/chosen": 0.640625,
      "logits/rejected": 1.125,
      "logps/chosen": -384.0,
      "logps/rejected": -398.0,
      "loss": 0.1989,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -10.9375,
      "rewards/margins": 6.0625,
      "rewards/rejected": -17.0,
      "step": 2360
    },
    {
      "epoch": 0.3028947536583807,
      "grad_norm": 9.292492408762836,
      "learning_rate": 1.6432960600355221e-06,
      "logits/chosen": 0.51953125,
      "logits/rejected": 0.98046875,
      "logps/chosen": -398.0,
      "logps/rejected": -438.0,
      "loss": 0.1918,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -11.875,
      "rewards/margins": 5.90625,
      "rewards/rejected": -17.75,
      "step": 2370
    },
    {
      "epoch": 0.30417279059364816,
      "grad_norm": 8.360660462094847,
      "learning_rate": 1.6398401233815756e-06,
      "logits/chosen": 0.57421875,
      "logits/rejected": 1.1171875,
      "logps/chosen": -370.0,
      "logps/rejected": -378.0,
      "loss": 0.1895,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -10.5,
      "rewards/margins": 5.9375,
      "rewards/rejected": -16.5,
      "step": 2380
    },
    {
      "epoch": 0.30545082752891556,
      "grad_norm": 5.43135655548482,
      "learning_rate": 1.6364058994584524e-06,
      "logits/chosen": 0.3203125,
      "logits/rejected": 0.87890625,
      "logps/chosen": -362.0,
      "logps/rejected": -374.0,
      "loss": 1.0535,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -9.375,
      "rewards/margins": 5.6875,
      "rewards/rejected": -15.0625,
      "step": 2390
    },
    {
      "epoch": 0.306728864464183,
      "grad_norm": 5.812596169588797,
      "learning_rate": 1.6329931618554522e-06,
      "logits/chosen": 0.435546875,
      "logits/rejected": 1.03125,
      "logps/chosen": -344.0,
      "logps/rejected": -402.0,
      "loss": 0.1824,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -9.25,
      "rewards/margins": 8.9375,
      "rewards/rejected": -18.125,
      "step": 2400
    },
    {
      "epoch": 0.3080069013994504,
      "grad_norm": 5.202118481787796,
      "learning_rate": 1.6296016874534209e-06,
      "logits/chosen": 0.40625,
      "logits/rejected": 1.109375,
      "logps/chosen": -394.0,
      "logps/rejected": -376.0,
      "loss": 0.2078,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -10.4375,
      "rewards/margins": 5.75,
      "rewards/rejected": -16.25,
      "step": 2410
    },
    {
      "epoch": 0.3092849383347179,
      "grad_norm": 10.716391972424226,
      "learning_rate": 1.6262312563634835e-06,
      "logits/chosen": 0.478515625,
      "logits/rejected": 0.99609375,
      "logps/chosen": -372.0,
      "logps/rejected": -396.0,
      "loss": 0.1923,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -10.6875,
      "rewards/margins": 6.6875,
      "rewards/rejected": -17.375,
      "step": 2420
    },
    {
      "epoch": 0.3105629752699853,
      "grad_norm": 9.980810442686792,
      "learning_rate": 1.6228816518671587e-06,
      "logits/chosen": 0.4609375,
      "logits/rejected": 1.25,
      "logps/chosen": -388.0,
      "logps/rejected": -402.0,
      "loss": 0.1673,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -11.75,
      "rewards/margins": 6.28125,
      "rewards/rejected": -18.0,
      "step": 2430
    },
    {
      "epoch": 0.31184101220525273,
      "grad_norm": 9.330279522127043,
      "learning_rate": 1.619552660357832e-06,
      "logits/chosen": 0.47265625,
      "logits/rejected": 0.91796875,
      "logps/chosen": -396.0,
      "logps/rejected": -416.0,
      "loss": 0.2038,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -11.375,
      "rewards/margins": 5.625,
      "rewards/rejected": -17.0,
      "step": 2440
    },
    {
      "epoch": 0.31311904914052013,
      "grad_norm": 10.35953725571517,
      "learning_rate": 1.616244071283537e-06,
      "logits/chosen": 0.59375,
      "logits/rejected": 0.91015625,
      "logps/chosen": -358.0,
      "logps/rejected": -416.0,
      "loss": 0.1946,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -10.625,
      "rewards/margins": 5.71875,
      "rewards/rejected": -16.375,
      "step": 2450
    },
    {
      "epoch": 0.3143970860757876,
      "grad_norm": 9.135292770049318,
      "learning_rate": 1.6129556770910235e-06,
      "logits/chosen": 0.578125,
      "logits/rejected": 1.171875,
      "logps/chosen": -364.0,
      "logps/rejected": -392.0,
      "loss": 0.169,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -11.625,
      "rewards/margins": 5.75,
      "rewards/rejected": -17.375,
      "step": 2460
    },
    {
      "epoch": 0.315675123011055,
      "grad_norm": 9.807902773503795,
      "learning_rate": 1.6096872731710673e-06,
      "logits/chosen": 0.3359375,
      "logits/rejected": 1.0390625,
      "logps/chosen": -414.0,
      "logps/rejected": -430.0,
      "loss": 0.1694,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -12.625,
      "rewards/margins": 6.75,
      "rewards/rejected": -19.375,
      "step": 2470
    },
    {
      "epoch": 0.31695315994632245,
      "grad_norm": 6.143214537918317,
      "learning_rate": 1.6064386578049978e-06,
      "logits/chosen": 0.22265625,
      "logits/rejected": 0.8046875,
      "logps/chosen": -392.0,
      "logps/rejected": -410.0,
      "loss": 0.1883,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -10.1875,
      "rewards/margins": 6.03125,
      "rewards/rejected": -16.25,
      "step": 2480
    },
    {
      "epoch": 0.31823119688158985,
      "grad_norm": 8.415676594938274,
      "learning_rate": 1.6032096321124046e-06,
      "logits/chosen": 0.416015625,
      "logits/rejected": 0.90625,
      "logps/chosen": -368.0,
      "logps/rejected": -376.0,
      "loss": 0.1682,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -9.875,
      "rewards/margins": 5.5625,
      "rewards/rejected": -15.4375,
      "step": 2490
    },
    {
      "epoch": 0.3195092338168573,
      "grad_norm": 9.997850768899943,
      "learning_rate": 1.6e-06,
      "logits/chosen": 0.3359375,
      "logits/rejected": 0.87890625,
      "logps/chosen": -404.0,
      "logps/rejected": -440.0,
      "loss": 0.2231,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -12.0,
      "rewards/margins": 6.78125,
      "rewards/rejected": -18.875,
      "step": 2500
    },
    {
      "epoch": 0.3207872707521247,
      "grad_norm": 9.862095494481318,
      "learning_rate": 1.5968095681115984e-06,
      "logits/chosen": 0.404296875,
      "logits/rejected": 0.79296875,
      "logps/chosen": -382.0,
      "logps/rejected": -442.0,
      "loss": 0.2074,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -11.5,
      "rewards/margins": 7.3125,
      "rewards/rejected": -18.75,
      "step": 2510
    },
    {
      "epoch": 0.32206530768739217,
      "grad_norm": 7.2484321073962334,
      "learning_rate": 1.5936381457791914e-06,
      "logits/chosen": 0.255859375,
      "logits/rejected": 0.8828125,
      "logps/chosen": -390.0,
      "logps/rejected": -384.0,
      "loss": 0.1821,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -10.6875,
      "rewards/margins": 5.84375,
      "rewards/rejected": -16.5,
      "step": 2520
    },
    {
      "epoch": 0.32334334462265957,
      "grad_norm": 6.48866539120617,
      "learning_rate": 1.590485544975088e-06,
      "logits/chosen": 0.1904296875,
      "logits/rejected": 0.78125,
      "logps/chosen": -376.0,
      "logps/rejected": -392.0,
      "loss": 0.1709,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -10.625,
      "rewards/margins": 5.5625,
      "rewards/rejected": -16.25,
      "step": 2530
    },
    {
      "epoch": 0.324621381557927,
      "grad_norm": 10.609198507732819,
      "learning_rate": 1.5873515802650901e-06,
      "logits/chosen": 0.353515625,
      "logits/rejected": 1.0546875,
      "logps/chosen": -386.0,
      "logps/rejected": -390.0,
      "loss": 0.1879,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -11.0625,
      "rewards/margins": 5.84375,
      "rewards/rejected": -16.875,
      "step": 2540
    },
    {
      "epoch": 0.3258994184931944,
      "grad_norm": 10.699811710326099,
      "learning_rate": 1.584236068762679e-06,
      "logits/chosen": 0.42578125,
      "logits/rejected": 0.99609375,
      "logps/chosen": -412.0,
      "logps/rejected": -444.0,
      "loss": 0.1576,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -14.4375,
      "rewards/margins": 6.1875,
      "rewards/rejected": -20.625,
      "step": 2550
    },
    {
      "epoch": 0.3271774554284619,
      "grad_norm": 8.66762193443968,
      "learning_rate": 1.5811388300841894e-06,
      "logits/chosen": 0.328125,
      "logits/rejected": 0.8671875,
      "logps/chosen": -440.0,
      "logps/rejected": -434.0,
      "loss": 0.2164,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.4375,
      "rewards/margins": 6.375,
      "rewards/rejected": -20.875,
      "step": 2560
    },
    {
      "epoch": 0.3284554923637293,
      "grad_norm": 10.589300011742601,
      "learning_rate": 1.5780596863049431e-06,
      "logits/chosen": 0.376953125,
      "logits/rejected": 1.0546875,
      "logps/chosen": -390.0,
      "logps/rejected": -412.0,
      "loss": 0.1861,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -12.25,
      "rewards/margins": 5.75,
      "rewards/rejected": -18.0,
      "step": 2570
    },
    {
      "epoch": 0.32973352929899674,
      "grad_norm": 7.232667960066554,
      "learning_rate": 1.5749984619163156e-06,
      "logits/chosen": 0.2890625,
      "logits/rejected": 0.85546875,
      "logps/chosen": -372.0,
      "logps/rejected": -386.0,
      "loss": 0.1621,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -10.375,
      "rewards/margins": 6.28125,
      "rewards/rejected": -16.625,
      "step": 2580
    },
    {
      "epoch": 0.33101156623426414,
      "grad_norm": 10.069004371649715,
      "learning_rate": 1.5719549837837187e-06,
      "logits/chosen": 0.31640625,
      "logits/rejected": 0.8203125,
      "logps/chosen": -388.0,
      "logps/rejected": -386.0,
      "loss": 0.1872,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -9.875,
      "rewards/margins": 5.84375,
      "rewards/rejected": -15.75,
      "step": 2590
    },
    {
      "epoch": 0.3322896031695316,
      "grad_norm": 5.497514150396675,
      "learning_rate": 1.5689290811054722e-06,
      "logits/chosen": 0.1865234375,
      "logits/rejected": 0.83984375,
      "logps/chosen": -382.0,
      "logps/rejected": -394.0,
      "loss": 0.1889,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -10.4375,
      "rewards/margins": 5.75,
      "rewards/rejected": -16.125,
      "step": 2600
    },
    {
      "epoch": 0.333567640104799,
      "grad_norm": 8.6223509804558,
      "learning_rate": 1.5659205853725426e-06,
      "logits/chosen": 0.322265625,
      "logits/rejected": 0.85546875,
      "logps/chosen": -384.0,
      "logps/rejected": -420.0,
      "loss": 0.1843,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -11.1875,
      "rewards/margins": 5.6875,
      "rewards/rejected": -16.875,
      "step": 2610
    },
    {
      "epoch": 0.33484567704006646,
      "grad_norm": 3.6942421260609546,
      "learning_rate": 1.562929330329127e-06,
      "logits/chosen": 0.421875,
      "logits/rejected": 1.0703125,
      "logps/chosen": -358.0,
      "logps/rejected": -396.0,
      "loss": 0.1416,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -11.25,
      "rewards/margins": 6.21875,
      "rewards/rejected": -17.5,
      "step": 2620
    },
    {
      "epoch": 0.33612371397533386,
      "grad_norm": 11.808223800553867,
      "learning_rate": 1.5599551519340636e-06,
      "logits/chosen": 0.31640625,
      "logits/rejected": 1.0078125,
      "logps/chosen": -386.0,
      "logps/rejected": -418.0,
      "loss": 0.1837,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -11.75,
      "rewards/margins": 6.46875,
      "rewards/rejected": -18.25,
      "step": 2630
    },
    {
      "epoch": 0.3374017509106013,
      "grad_norm": 11.667344119282047,
      "learning_rate": 1.556997888323046e-06,
      "logits/chosen": 0.27734375,
      "logits/rejected": 0.859375,
      "logps/chosen": -398.0,
      "logps/rejected": -404.0,
      "loss": 0.2187,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -11.625,
      "rewards/margins": 5.40625,
      "rewards/rejected": -17.0,
      "step": 2640
    },
    {
      "epoch": 0.3386797878458687,
      "grad_norm": 11.149279815982254,
      "learning_rate": 1.5540573797716226e-06,
      "logits/chosen": 0.177734375,
      "logits/rejected": 0.7734375,
      "logps/chosen": -356.0,
      "logps/rejected": -386.0,
      "loss": 0.1666,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -9.625,
      "rewards/margins": 5.875,
      "rewards/rejected": -15.5,
      "step": 2650
    },
    {
      "epoch": 0.3399578247811362,
      "grad_norm": 8.972538465024622,
      "learning_rate": 1.5511334686589623e-06,
      "logits/chosen": 0.263671875,
      "logits/rejected": 0.7265625,
      "logps/chosen": -414.0,
      "logps/rejected": -438.0,
      "loss": 0.1744,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -13.5625,
      "rewards/margins": 5.78125,
      "rewards/rejected": -19.375,
      "step": 2660
    },
    {
      "epoch": 0.3412358617164036,
      "grad_norm": 15.846273451695808,
      "learning_rate": 1.548225999432367e-06,
      "logits/chosen": 0.482421875,
      "logits/rejected": 1.0390625,
      "logps/chosen": -404.0,
      "logps/rejected": -420.0,
      "loss": 0.1834,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -13.4375,
      "rewards/margins": 5.59375,
      "rewards/rejected": -19.0,
      "step": 2670
    },
    {
      "epoch": 0.34251389865167103,
      "grad_norm": 8.009496897062526,
      "learning_rate": 1.5453348185725114e-06,
      "logits/chosen": 0.357421875,
      "logits/rejected": 0.98828125,
      "logps/chosen": -402.0,
      "logps/rejected": -414.0,
      "loss": 0.2121,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -13.4375,
      "rewards/margins": 5.96875,
      "rewards/rejected": -19.375,
      "step": 2680
    },
    {
      "epoch": 0.34379193558693844,
      "grad_norm": 10.504859142182582,
      "learning_rate": 1.542459774559398e-06,
      "logits/chosen": 0.396484375,
      "logits/rejected": 1.03125,
      "logps/chosen": -386.0,
      "logps/rejected": -402.0,
      "loss": 0.1711,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -11.5625,
      "rewards/margins": 6.40625,
      "rewards/rejected": -18.0,
      "step": 2690
    },
    {
      "epoch": 0.3450699725222059,
      "grad_norm": 9.31166429399796,
      "learning_rate": 1.539600717839002e-06,
      "logits/chosen": 0.2373046875,
      "logits/rejected": 0.88671875,
      "logps/chosen": -378.0,
      "logps/rejected": -402.0,
      "loss": 0.1844,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -11.0625,
      "rewards/margins": 6.09375,
      "rewards/rejected": -17.125,
      "step": 2700
    },
    {
      "epoch": 0.3463480094574733,
      "grad_norm": 10.061837162462728,
      "learning_rate": 1.536757500790597e-06,
      "logits/chosen": 0.34765625,
      "logits/rejected": 1.046875,
      "logps/chosen": -378.0,
      "logps/rejected": -390.0,
      "loss": 0.2012,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -11.375,
      "rewards/margins": 6.21875,
      "rewards/rejected": -17.625,
      "step": 2710
    },
    {
      "epoch": 0.34762604639274075,
      "grad_norm": 10.763956036606048,
      "learning_rate": 1.5339299776947407e-06,
      "logits/chosen": 0.412109375,
      "logits/rejected": 0.92578125,
      "logps/chosen": -444.0,
      "logps/rejected": -452.0,
      "loss": 0.2016,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.75,
      "rewards/margins": 6.71875,
      "rewards/rejected": -21.5,
      "step": 2720
    },
    {
      "epoch": 0.34890408332800815,
      "grad_norm": 3.7956990961819113,
      "learning_rate": 1.5311180047019054e-06,
      "logits/chosen": 0.462890625,
      "logits/rejected": 1.1015625,
      "logps/chosen": -390.0,
      "logps/rejected": -414.0,
      "loss": 0.1717,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -13.1875,
      "rewards/margins": 6.78125,
      "rewards/rejected": -20.0,
      "step": 2730
    },
    {
      "epoch": 0.3501821202632756,
      "grad_norm": 11.23299268622459,
      "learning_rate": 1.5283214398017402e-06,
      "logits/chosen": 0.4609375,
      "logits/rejected": 0.875,
      "logps/chosen": -370.0,
      "logps/rejected": -414.0,
      "loss": 0.1944,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -12.375,
      "rewards/margins": 7.0625,
      "rewards/rejected": -19.375,
      "step": 2740
    },
    {
      "epoch": 0.351460157198543,
      "grad_norm": 11.356574041725917,
      "learning_rate": 1.5255401427929477e-06,
      "logits/chosen": 0.5078125,
      "logits/rejected": 1.0078125,
      "logps/chosen": -366.0,
      "logps/rejected": -406.0,
      "loss": 0.1872,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -10.5,
      "rewards/margins": 6.46875,
      "rewards/rejected": -17.0,
      "step": 2750
    },
    {
      "epoch": 0.35273819413381047,
      "grad_norm": 8.727243899842758,
      "learning_rate": 1.5227739752537617e-06,
      "logits/chosen": 0.3359375,
      "logits/rejected": 1.015625,
      "logps/chosen": -392.0,
      "logps/rejected": -402.0,
      "loss": 0.195,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -10.9375,
      "rewards/margins": 6.125,
      "rewards/rejected": -17.0,
      "step": 2760
    },
    {
      "epoch": 0.35401623106907787,
      "grad_norm": 7.391730652692325,
      "learning_rate": 1.5200228005130127e-06,
      "logits/chosen": 0.40234375,
      "logits/rejected": 0.859375,
      "logps/chosen": -388.0,
      "logps/rejected": -404.0,
      "loss": 0.1503,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -11.125,
      "rewards/margins": 6.0,
      "rewards/rejected": -17.125,
      "step": 2770
    },
    {
      "epoch": 0.3552942680043453,
      "grad_norm": 7.186410341660446,
      "learning_rate": 1.5172864836217631e-06,
      "logits/chosen": 0.408203125,
      "logits/rejected": 0.859375,
      "logps/chosen": -400.0,
      "logps/rejected": -450.0,
      "loss": 0.1475,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.25,
      "rewards/margins": 6.125,
      "rewards/rejected": -19.375,
      "step": 2780
    },
    {
      "epoch": 0.3565723049396127,
      "grad_norm": 8.413303376685455,
      "learning_rate": 1.514564891325506e-06,
      "logits/chosen": 0.40625,
      "logits/rejected": 1.015625,
      "logps/chosen": -436.0,
      "logps/rejected": -444.0,
      "loss": 0.1861,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -15.25,
      "rewards/margins": 5.8125,
      "rewards/rejected": -21.0,
      "step": 2790
    },
    {
      "epoch": 0.3578503418748802,
      "grad_norm": 9.261290949896116,
      "learning_rate": 1.5118578920369086e-06,
      "logits/chosen": 0.404296875,
      "logits/rejected": 0.96875,
      "logps/chosen": -446.0,
      "logps/rejected": -456.0,
      "loss": 0.1647,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.625,
      "rewards/margins": 6.34375,
      "rewards/rejected": -21.0,
      "step": 2800
    },
    {
      "epoch": 0.3591283788101476,
      "grad_norm": 9.664097082797042,
      "learning_rate": 1.5091653558090898e-06,
      "logits/chosen": 0.466796875,
      "logits/rejected": 1.1328125,
      "logps/chosen": -406.0,
      "logps/rejected": -444.0,
      "loss": 0.2023,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.875,
      "rewards/margins": 5.90625,
      "rewards/rejected": -20.75,
      "step": 2810
    },
    {
      "epoch": 0.36040641574541504,
      "grad_norm": 8.20712352636203,
      "learning_rate": 1.506487154309419e-06,
      "logits/chosen": 0.302734375,
      "logits/rejected": 1.1484375,
      "logps/chosen": -458.0,
      "logps/rejected": -464.0,
      "loss": 0.1851,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.0,
      "rewards/margins": 6.40625,
      "rewards/rejected": -22.375,
      "step": 2820
    },
    {
      "epoch": 0.36168445268068244,
      "grad_norm": 9.659109752066405,
      "learning_rate": 1.5038231607938247e-06,
      "logits/chosen": 0.42578125,
      "logits/rejected": 1.1796875,
      "logps/chosen": -426.0,
      "logps/rejected": -444.0,
      "loss": 0.1686,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -15.0,
      "rewards/margins": 6.15625,
      "rewards/rejected": -21.125,
      "step": 2830
    },
    {
      "epoch": 0.3629624896159499,
      "grad_norm": 10.439857036475965,
      "learning_rate": 1.501173250081603e-06,
      "logits/chosen": 0.439453125,
      "logits/rejected": 1.09375,
      "logps/chosen": -434.0,
      "logps/rejected": -452.0,
      "loss": 0.1978,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.0625,
      "rewards/margins": 6.75,
      "rewards/rejected": -21.75,
      "step": 2840
    },
    {
      "epoch": 0.3642405265512173,
      "grad_norm": 5.788575676488457,
      "learning_rate": 1.4985372985307103e-06,
      "logits/chosen": 0.33203125,
      "logits/rejected": 0.984375,
      "logps/chosen": -432.0,
      "logps/rejected": -460.0,
      "loss": 0.161,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -13.5,
      "rewards/margins": 6.6875,
      "rewards/rejected": -20.125,
      "step": 2850
    },
    {
      "epoch": 0.36551856348648476,
      "grad_norm": 4.837332998448716,
      "learning_rate": 1.4959151840135313e-06,
      "logits/chosen": 0.251953125,
      "logits/rejected": 0.84765625,
      "logps/chosen": -424.0,
      "logps/rejected": -460.0,
      "loss": 0.1545,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.375,
      "rewards/margins": 7.0,
      "rewards/rejected": -21.375,
      "step": 2860
    },
    {
      "epoch": 0.36679660042175216,
      "grad_norm": 10.523517518196611,
      "learning_rate": 1.4933067858931148e-06,
      "logits/chosen": 0.42578125,
      "logits/rejected": 0.9609375,
      "logps/chosen": -360.0,
      "logps/rejected": -420.0,
      "loss": 0.1697,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -11.875,
      "rewards/margins": 6.71875,
      "rewards/rejected": -18.625,
      "step": 2870
    },
    {
      "epoch": 0.3680746373570196,
      "grad_norm": 15.711896894476958,
      "learning_rate": 1.4907119849998597e-06,
      "logits/chosen": 0.328125,
      "logits/rejected": 0.8515625,
      "logps/chosen": -368.0,
      "logps/rejected": -400.0,
      "loss": 0.1771,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -10.8125,
      "rewards/margins": 6.09375,
      "rewards/rejected": -16.875,
      "step": 2880
    },
    {
      "epoch": 0.369352674292287,
      "grad_norm": 6.7192026006567405,
      "learning_rate": 1.488130663608649e-06,
      "logits/chosen": 0.3515625,
      "logits/rejected": 0.83203125,
      "logps/chosen": -368.0,
      "logps/rejected": -406.0,
      "loss": 0.1247,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -11.25,
      "rewards/margins": 6.09375,
      "rewards/rejected": -17.375,
      "step": 2890
    },
    {
      "epoch": 0.3706307112275545,
      "grad_norm": 7.809374300701405,
      "learning_rate": 1.4855627054164149e-06,
      "logits/chosen": 0.291015625,
      "logits/rejected": 0.94140625,
      "logps/chosen": -392.0,
      "logps/rejected": -440.0,
      "loss": 0.1733,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -13.6875,
      "rewards/margins": 6.25,
      "rewards/rejected": -19.875,
      "step": 2900
    },
    {
      "epoch": 0.3719087481628219,
      "grad_norm": 11.128128353468712,
      "learning_rate": 1.4830079955201294e-06,
      "logits/chosen": 0.310546875,
      "logits/rejected": 1.015625,
      "logps/chosen": -412.0,
      "logps/rejected": -442.0,
      "loss": 0.1772,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -12.9375,
      "rewards/margins": 7.6875,
      "rewards/rejected": -20.625,
      "step": 2910
    },
    {
      "epoch": 0.37318678509808934,
      "grad_norm": 18.299513826178178,
      "learning_rate": 1.4804664203952103e-06,
      "logits/chosen": 0.5546875,
      "logits/rejected": 1.2265625,
      "logps/chosen": -396.0,
      "logps/rejected": -450.0,
      "loss": 0.1714,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.125,
      "rewards/margins": 7.03125,
      "rewards/rejected": -21.125,
      "step": 2920
    },
    {
      "epoch": 0.37446482203335674,
      "grad_norm": 4.812342118636766,
      "learning_rate": 1.4779378678743327e-06,
      "logits/chosen": 0.390625,
      "logits/rejected": 1.0546875,
      "logps/chosen": -416.0,
      "logps/rejected": -432.0,
      "loss": 0.1452,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.25,
      "rewards/margins": 6.34375,
      "rewards/rejected": -20.625,
      "step": 2930
    },
    {
      "epoch": 0.3757428589686242,
      "grad_norm": 14.541509901071656,
      "learning_rate": 1.4754222271266348e-06,
      "logits/chosen": 0.376953125,
      "logits/rejected": 1.0390625,
      "logps/chosen": -410.0,
      "logps/rejected": -420.0,
      "loss": 0.1731,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -12.4375,
      "rewards/margins": 6.28125,
      "rewards/rejected": -18.75,
      "step": 2940
    },
    {
      "epoch": 0.3770208959038916,
      "grad_norm": 5.247893810720422,
      "learning_rate": 1.4729193886373175e-06,
      "logits/chosen": 0.28515625,
      "logits/rejected": 0.8125,
      "logps/chosen": -400.0,
      "logps/rejected": -438.0,
      "loss": 0.1327,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -12.625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -19.375,
      "step": 2950
    },
    {
      "epoch": 0.37829893283915905,
      "grad_norm": 8.599933543926108,
      "learning_rate": 1.4704292441876156e-06,
      "logits/chosen": 0.337890625,
      "logits/rejected": 0.9375,
      "logps/chosen": -428.0,
      "logps/rejected": -442.0,
      "loss": 0.1878,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.5625,
      "rewards/margins": 8.0625,
      "rewards/rejected": -21.625,
      "step": 2960
    },
    {
      "epoch": 0.37957696977442645,
      "grad_norm": 11.043920341388738,
      "learning_rate": 1.4679516868351474e-06,
      "logits/chosen": 0.5625,
      "logits/rejected": 1.0625,
      "logps/chosen": -394.0,
      "logps/rejected": -438.0,
      "loss": 0.1972,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -14.1875,
      "rewards/margins": 6.71875,
      "rewards/rejected": -20.875,
      "step": 2970
    },
    {
      "epoch": 0.3808550067096939,
      "grad_norm": 8.394660895022735,
      "learning_rate": 1.4654866108946234e-06,
      "logits/chosen": 0.4609375,
      "logits/rejected": 1.1484375,
      "logps/chosen": -404.0,
      "logps/rejected": -442.0,
      "loss": 0.1979,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -13.875,
      "rewards/margins": 6.96875,
      "rewards/rejected": -20.875,
      "step": 2980
    },
    {
      "epoch": 0.3821330436449613,
      "grad_norm": 6.667417633483774,
      "learning_rate": 1.4630339119189101e-06,
      "logits/chosen": 0.29296875,
      "logits/rejected": 0.98046875,
      "logps/chosen": -392.0,
      "logps/rejected": -418.0,
      "loss": 0.1595,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -12.5625,
      "rewards/margins": 6.125,
      "rewards/rejected": -18.75,
      "step": 2990
    },
    {
      "epoch": 0.38341108058022877,
      "grad_norm": 6.384883878159698,
      "learning_rate": 1.4605934866804429e-06,
      "logits/chosen": 0.455078125,
      "logits/rejected": 0.9921875,
      "logps/chosen": -388.0,
      "logps/rejected": -432.0,
      "loss": 0.1407,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -13.5625,
      "rewards/margins": 6.0,
      "rewards/rejected": -19.5,
      "step": 3000
    },
    {
      "epoch": 0.38468911751549617,
      "grad_norm": 9.12806834903797,
      "learning_rate": 1.4581652331529784e-06,
      "logits/chosen": 0.2060546875,
      "logits/rejected": 0.91796875,
      "logps/chosen": -414.0,
      "logps/rejected": -444.0,
      "loss": 0.1763,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -13.5,
      "rewards/margins": 6.8125,
      "rewards/rejected": -20.25,
      "step": 3010
    },
    {
      "epoch": 0.3859671544507636,
      "grad_norm": 9.314373827635102,
      "learning_rate": 1.4557490504936778e-06,
      "logits/chosen": 0.255859375,
      "logits/rejected": 0.99609375,
      "logps/chosen": -404.0,
      "logps/rejected": -442.0,
      "loss": 0.1772,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -12.9375,
      "rewards/margins": 7.21875,
      "rewards/rejected": -20.125,
      "step": 3020
    },
    {
      "epoch": 0.38724519138603103,
      "grad_norm": 8.890729779478567,
      "learning_rate": 1.453344839025519e-06,
      "logits/chosen": 0.3828125,
      "logits/rejected": 1.0078125,
      "logps/chosen": -428.0,
      "logps/rejected": -462.0,
      "loss": 0.153,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.125,
      "rewards/margins": 6.75,
      "rewards/rejected": -20.875,
      "step": 3030
    },
    {
      "epoch": 0.3885232283212985,
      "grad_norm": 7.145085147504701,
      "learning_rate": 1.4509525002200234e-06,
      "logits/chosen": 0.30859375,
      "logits/rejected": 0.796875,
      "logps/chosen": -408.0,
      "logps/rejected": -412.0,
      "loss": 0.1802,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -12.8125,
      "rewards/margins": 5.78125,
      "rewards/rejected": -18.625,
      "step": 3040
    },
    {
      "epoch": 0.3898012652565659,
      "grad_norm": 9.139041467395005,
      "learning_rate": 1.4485719366802965e-06,
      "logits/chosen": 0.388671875,
      "logits/rejected": 0.88671875,
      "logps/chosen": -394.0,
      "logps/rejected": -424.0,
      "loss": 0.2007,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -11.5,
      "rewards/margins": 6.4375,
      "rewards/rejected": -17.875,
      "step": 3050
    },
    {
      "epoch": 0.39107930219183334,
      "grad_norm": 6.241306567771513,
      "learning_rate": 1.4462030521243742e-06,
      "logits/chosen": 0.322265625,
      "logits/rejected": 0.96484375,
      "logps/chosen": -408.0,
      "logps/rejected": -404.0,
      "loss": 0.1248,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -11.375,
      "rewards/margins": 6.1875,
      "rewards/rejected": -17.5,
      "step": 3060
    },
    {
      "epoch": 0.39235733912710075,
      "grad_norm": 7.762748216230055,
      "learning_rate": 1.443845751368867e-06,
      "logits/chosen": 0.2138671875,
      "logits/rejected": 0.91796875,
      "logps/chosen": -408.0,
      "logps/rejected": -438.0,
      "loss": 0.1764,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -12.6875,
      "rewards/margins": 6.5,
      "rewards/rejected": -19.25,
      "step": 3070
    },
    {
      "epoch": 0.3936353760623682,
      "grad_norm": 6.322245690951376,
      "learning_rate": 1.4414999403128943e-06,
      "logits/chosen": 0.42578125,
      "logits/rejected": 0.984375,
      "logps/chosen": -380.0,
      "logps/rejected": -410.0,
      "loss": 0.1836,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -11.4375,
      "rewards/margins": 6.3125,
      "rewards/rejected": -17.75,
      "step": 3080
    },
    {
      "epoch": 0.3949134129976356,
      "grad_norm": 7.250517529371249,
      "learning_rate": 1.439165525922309e-06,
      "logits/chosen": 0.279296875,
      "logits/rejected": 0.92578125,
      "logps/chosen": -380.0,
      "logps/rejected": -406.0,
      "loss": 0.1631,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -11.0,
      "rewards/margins": 6.15625,
      "rewards/rejected": -17.125,
      "step": 3090
    },
    {
      "epoch": 0.39619144993290306,
      "grad_norm": 5.815912333742044,
      "learning_rate": 1.4368424162141992e-06,
      "logits/chosen": 0.3671875,
      "logits/rejected": 0.9921875,
      "logps/chosen": -390.0,
      "logps/rejected": -408.0,
      "loss": 0.2023,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -12.0625,
      "rewards/margins": 6.09375,
      "rewards/rejected": -18.25,
      "step": 3100
    },
    {
      "epoch": 0.39746948686817046,
      "grad_norm": 7.174724559451219,
      "learning_rate": 1.434530520241665e-06,
      "logits/chosen": 0.359375,
      "logits/rejected": 0.84765625,
      "logps/chosen": -382.0,
      "logps/rejected": -428.0,
      "loss": 0.1497,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -11.9375,
      "rewards/margins": 6.1875,
      "rewards/rejected": -18.125,
      "step": 3110
    },
    {
      "epoch": 0.3987475238034379,
      "grad_norm": 10.55329478998411,
      "learning_rate": 1.4322297480788657e-06,
      "logits/chosen": 0.412109375,
      "logits/rejected": 0.92578125,
      "logps/chosen": -380.0,
      "logps/rejected": -412.0,
      "loss": 0.1751,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -11.625,
      "rewards/margins": 6.5,
      "rewards/rejected": -18.125,
      "step": 3120
    },
    {
      "epoch": 0.4000255607387053,
      "grad_norm": 10.082645318724088,
      "learning_rate": 1.4299400108063247e-06,
      "logits/chosen": 0.453125,
      "logits/rejected": 0.97265625,
      "logps/chosen": -418.0,
      "logps/rejected": -440.0,
      "loss": 0.1689,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.0,
      "rewards/margins": 6.65625,
      "rewards/rejected": -20.625,
      "step": 3130
    },
    {
      "epoch": 0.4013035976739728,
      "grad_norm": 7.229065833350122,
      "learning_rate": 1.4276612204964992e-06,
      "logits/chosen": 0.388671875,
      "logits/rejected": 0.91015625,
      "logps/chosen": -410.0,
      "logps/rejected": -444.0,
      "loss": 0.1436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.875,
      "rewards/margins": 6.75,
      "rewards/rejected": -20.625,
      "step": 3140
    },
    {
      "epoch": 0.4025816346092402,
      "grad_norm": 12.244741394955584,
      "learning_rate": 1.4253932901995967e-06,
      "logits/chosen": 0.232421875,
      "logits/rejected": 0.9375,
      "logps/chosen": -420.0,
      "logps/rejected": -452.0,
      "loss": 0.1783,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.0,
      "rewards/margins": 6.78125,
      "rewards/rejected": -19.75,
      "step": 3150
    },
    {
      "epoch": 0.40385967154450764,
      "grad_norm": 8.527792419711863,
      "learning_rate": 1.42313613392964e-06,
      "logits/chosen": 0.330078125,
      "logits/rejected": 0.97265625,
      "logps/chosen": -412.0,
      "logps/rejected": -432.0,
      "loss": 0.1881,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.6875,
      "rewards/margins": 6.34375,
      "rewards/rejected": -20.0,
      "step": 3160
    },
    {
      "epoch": 0.40513770847977504,
      "grad_norm": 7.51382045600948,
      "learning_rate": 1.4208896666507756e-06,
      "logits/chosen": 0.345703125,
      "logits/rejected": 1.078125,
      "logps/chosen": -412.0,
      "logps/rejected": -428.0,
      "loss": 0.1526,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -13.75,
      "rewards/margins": 6.84375,
      "rewards/rejected": -20.625,
      "step": 3170
    },
    {
      "epoch": 0.4064157454150425,
      "grad_norm": 7.443674660576827,
      "learning_rate": 1.4186538042638173e-06,
      "logits/chosen": 0.390625,
      "logits/rejected": 0.98046875,
      "logps/chosen": -424.0,
      "logps/rejected": -438.0,
      "loss": 0.1546,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.5625,
      "rewards/margins": 6.28125,
      "rewards/rejected": -20.875,
      "step": 3180
    },
    {
      "epoch": 0.4076937823503099,
      "grad_norm": 11.63338715049146,
      "learning_rate": 1.416428463593022e-06,
      "logits/chosen": 0.337890625,
      "logits/rejected": 0.9375,
      "logps/chosen": -402.0,
      "logps/rejected": -438.0,
      "loss": 0.1622,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.6875,
      "rewards/margins": 6.5,
      "rewards/rejected": -21.25,
      "step": 3190
    },
    {
      "epoch": 0.40897181928557735,
      "grad_norm": 8.798688500761584,
      "learning_rate": 1.414213562373095e-06,
      "logits/chosen": 0.30859375,
      "logits/rejected": 1.0234375,
      "logps/chosen": -416.0,
      "logps/rejected": -446.0,
      "loss": 0.1724,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -15.5,
      "rewards/margins": 7.0,
      "rewards/rejected": -22.5,
      "step": 3200
    },
    {
      "epoch": 0.41024985622084476,
      "grad_norm": 9.990285026354506,
      "learning_rate": 1.4120090192364154e-06,
      "logits/chosen": 0.326171875,
      "logits/rejected": 1.0390625,
      "logps/chosen": -410.0,
      "logps/rejected": -444.0,
      "loss": 0.149,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.5625,
      "rewards/margins": 7.15625,
      "rewards/rejected": -21.75,
      "step": 3210
    },
    {
      "epoch": 0.4115278931561122,
      "grad_norm": 11.906169493996083,
      "learning_rate": 1.4098147537004828e-06,
      "logits/chosen": 0.4375,
      "logits/rejected": 1.1171875,
      "logps/chosen": -422.0,
      "logps/rejected": -458.0,
      "loss": 0.1893,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.75,
      "rewards/margins": 7.375,
      "rewards/rejected": -22.125,
      "step": 3220
    },
    {
      "epoch": 0.4128059300913796,
      "grad_norm": 10.688730464851359,
      "learning_rate": 1.4076306861555735e-06,
      "logits/chosen": 0.37890625,
      "logits/rejected": 0.83203125,
      "logps/chosen": -424.0,
      "logps/rejected": -452.0,
      "loss": 0.1628,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.3125,
      "rewards/margins": 6.71875,
      "rewards/rejected": -22.0,
      "step": 3230
    },
    {
      "epoch": 0.41408396702664707,
      "grad_norm": 11.076098831228041,
      "learning_rate": 1.405456737852613e-06,
      "logits/chosen": 0.396484375,
      "logits/rejected": 0.95703125,
      "logps/chosen": -412.0,
      "logps/rejected": -446.0,
      "loss": 0.1541,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.25,
      "rewards/margins": 6.625,
      "rewards/rejected": -20.875,
      "step": 3240
    },
    {
      "epoch": 0.41536200396191447,
      "grad_norm": 5.861320454990625,
      "learning_rate": 1.4032928308912468e-06,
      "logits/chosen": 0.298828125,
      "logits/rejected": 0.8046875,
      "logps/chosen": -428.0,
      "logps/rejected": -462.0,
      "loss": 0.1704,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -13.875,
      "rewards/margins": 6.28125,
      "rewards/rejected": -20.125,
      "step": 3250
    },
    {
      "epoch": 0.41664004089718193,
      "grad_norm": 6.321220828293229,
      "learning_rate": 1.4011388882081175e-06,
      "logits/chosen": 0.490234375,
      "logits/rejected": 0.9375,
      "logps/chosen": -364.0,
      "logps/rejected": -388.0,
      "loss": 0.1734,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -11.1875,
      "rewards/margins": 5.9375,
      "rewards/rejected": -17.125,
      "step": 3260
    },
    {
      "epoch": 0.41791807783244933,
      "grad_norm": 9.72266208394698,
      "learning_rate": 1.3989948335653378e-06,
      "logits/chosen": 0.298828125,
      "logits/rejected": 0.9921875,
      "logps/chosen": -394.0,
      "logps/rejected": -398.0,
      "loss": 0.1588,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -11.6875,
      "rewards/margins": 6.03125,
      "rewards/rejected": -17.75,
      "step": 3270
    },
    {
      "epoch": 0.4191961147677168,
      "grad_norm": 9.17809650998058,
      "learning_rate": 1.3968605915391564e-06,
      "logits/chosen": 0.45703125,
      "logits/rejected": 1.0703125,
      "logps/chosen": -404.0,
      "logps/rejected": -424.0,
      "loss": 0.1594,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.875,
      "rewards/margins": 6.5625,
      "rewards/rejected": -20.375,
      "step": 3280
    },
    {
      "epoch": 0.4204741517029842,
      "grad_norm": 7.4324896929829345,
      "learning_rate": 1.3947360875088132e-06,
      "logits/chosen": 0.353515625,
      "logits/rejected": 1.0546875,
      "logps/chosen": -424.0,
      "logps/rejected": -448.0,
      "loss": 0.161,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.75,
      "rewards/margins": 6.34375,
      "rewards/rejected": -21.125,
      "step": 3290
    },
    {
      "epoch": 0.42175218863825165,
      "grad_norm": 9.329249561017615,
      "learning_rate": 1.3926212476455828e-06,
      "logits/chosen": 0.43359375,
      "logits/rejected": 1.09375,
      "logps/chosen": -406.0,
      "logps/rejected": -454.0,
      "loss": 0.1677,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -15.5625,
      "rewards/margins": 6.125,
      "rewards/rejected": -21.75,
      "step": 3300
    },
    {
      "epoch": 0.42303022557351905,
      "grad_norm": 9.5052776594737,
      "learning_rate": 1.3905159989019964e-06,
      "logits/chosen": 0.404296875,
      "logits/rejected": 0.984375,
      "logps/chosen": -422.0,
      "logps/rejected": -472.0,
      "loss": 0.1879,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -14.25,
      "rewards/margins": 6.21875,
      "rewards/rejected": -20.5,
      "step": 3310
    },
    {
      "epoch": 0.4243082625087865,
      "grad_norm": 8.297054201747507,
      "learning_rate": 1.3884202690012465e-06,
      "logits/chosen": 0.283203125,
      "logits/rejected": 1.1171875,
      "logps/chosen": -418.0,
      "logps/rejected": -428.0,
      "loss": 0.1545,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -13.6875,
      "rewards/margins": 6.34375,
      "rewards/rejected": -20.0,
      "step": 3320
    },
    {
      "epoch": 0.4255862994440539,
      "grad_norm": 11.266813170758867,
      "learning_rate": 1.3863339864267636e-06,
      "logits/chosen": 0.474609375,
      "logits/rejected": 1.078125,
      "logps/chosen": -390.0,
      "logps/rejected": -428.0,
      "loss": 0.1912,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -13.5,
      "rewards/margins": 6.625,
      "rewards/rejected": -20.125,
      "step": 3330
    },
    {
      "epoch": 0.42686433637932136,
      "grad_norm": 7.896300189342659,
      "learning_rate": 1.3842570804119655e-06,
      "logits/chosen": 0.259765625,
      "logits/rejected": 0.85546875,
      "logps/chosen": -404.0,
      "logps/rejected": -446.0,
      "loss": 0.1557,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -13.9375,
      "rewards/margins": 7.03125,
      "rewards/rejected": -21.0,
      "step": 3340
    },
    {
      "epoch": 0.42814237331458876,
      "grad_norm": 7.2806233616891936,
      "learning_rate": 1.3821894809301763e-06,
      "logits/chosen": 0.15625,
      "logits/rejected": 1.15625,
      "logps/chosen": -434.0,
      "logps/rejected": -444.0,
      "loss": 0.1341,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.6875,
      "rewards/margins": 6.78125,
      "rewards/rejected": -21.5,
      "step": 3350
    },
    {
      "epoch": 0.4294204102498562,
      "grad_norm": 7.954767567888139,
      "learning_rate": 1.3801311186847081e-06,
      "logits/chosen": 0.2001953125,
      "logits/rejected": 1.0390625,
      "logps/chosen": -420.0,
      "logps/rejected": -456.0,
      "loss": 0.1395,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.5625,
      "rewards/margins": 7.0,
      "rewards/rejected": -21.5,
      "step": 3360
    },
    {
      "epoch": 0.4306984471851236,
      "grad_norm": 10.816229476265907,
      "learning_rate": 1.378081925099109e-06,
      "logits/chosen": 0.34765625,
      "logits/rejected": 0.984375,
      "logps/chosen": -400.0,
      "logps/rejected": -458.0,
      "loss": 0.1579,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -13.9375,
      "rewards/margins": 7.125,
      "rewards/rejected": -21.0,
      "step": 3370
    },
    {
      "epoch": 0.4319764841203911,
      "grad_norm": 7.5629643005079155,
      "learning_rate": 1.376041832307563e-06,
      "logits/chosen": 0.283203125,
      "logits/rejected": 1.25,
      "logps/chosen": -384.0,
      "logps/rejected": -430.0,
      "loss": 0.1569,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -12.625,
      "rewards/margins": 7.28125,
      "rewards/rejected": -19.875,
      "step": 3380
    },
    {
      "epoch": 0.4332545210556585,
      "grad_norm": 9.520099894472832,
      "learning_rate": 1.3740107731454524e-06,
      "logits/chosen": 0.408203125,
      "logits/rejected": 1.015625,
      "logps/chosen": -384.0,
      "logps/rejected": -412.0,
      "loss": 0.1428,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -12.4375,
      "rewards/margins": 6.1875,
      "rewards/rejected": -18.625,
      "step": 3390
    },
    {
      "epoch": 0.43453255799092594,
      "grad_norm": 9.943198890487507,
      "learning_rate": 1.3719886811400705e-06,
      "logits/chosen": 0.2734375,
      "logits/rejected": 0.875,
      "logps/chosen": -404.0,
      "logps/rejected": -432.0,
      "loss": 0.1834,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -13.0,
      "rewards/margins": 6.71875,
      "rewards/rejected": -19.75,
      "step": 3400
    },
    {
      "epoch": 0.43581059492619334,
      "grad_norm": 5.905337339724567,
      "learning_rate": 1.3699754905014834e-06,
      "logits/chosen": 0.40234375,
      "logits/rejected": 1.0234375,
      "logps/chosen": -390.0,
      "logps/rejected": -414.0,
      "loss": 0.1219,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -13.5625,
      "rewards/margins": 6.40625,
      "rewards/rejected": -20.0,
      "step": 3410
    },
    {
      "epoch": 0.4370886318614608,
      "grad_norm": 8.758868354952035,
      "learning_rate": 1.3679711361135388e-06,
      "logits/chosen": 0.388671875,
      "logits/rejected": 1.0390625,
      "logps/chosen": -398.0,
      "logps/rejected": -440.0,
      "loss": 0.1365,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.9375,
      "rewards/margins": 7.21875,
      "rewards/rejected": -21.125,
      "step": 3420
    },
    {
      "epoch": 0.4383666687967282,
      "grad_norm": 9.369196399306151,
      "learning_rate": 1.3659755535250212e-06,
      "logits/chosen": 0.376953125,
      "logits/rejected": 0.91015625,
      "logps/chosen": -410.0,
      "logps/rejected": -460.0,
      "loss": 0.158,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.0625,
      "rewards/margins": 6.8125,
      "rewards/rejected": -20.875,
      "step": 3430
    },
    {
      "epoch": 0.43964470573199566,
      "grad_norm": 8.32907520938717,
      "learning_rate": 1.3639886789409469e-06,
      "logits/chosen": 0.447265625,
      "logits/rejected": 1.0,
      "logps/chosen": -386.0,
      "logps/rejected": -402.0,
      "loss": 0.1581,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -13.1875,
      "rewards/margins": 5.5625,
      "rewards/rejected": -18.75,
      "step": 3440
    },
    {
      "epoch": 0.44092274266726306,
      "grad_norm": 9.02279454432918,
      "learning_rate": 1.3620104492139977e-06,
      "logits/chosen": 0.349609375,
      "logits/rejected": 1.1796875,
      "logps/chosen": -392.0,
      "logps/rejected": -436.0,
      "loss": 0.1417,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.0625,
      "rewards/margins": 7.0625,
      "rewards/rejected": -21.125,
      "step": 3450
    },
    {
      "epoch": 0.4422007796025305,
      "grad_norm": 10.866043514090565,
      "learning_rate": 1.3600408018360918e-06,
      "logits/chosen": 0.373046875,
      "logits/rejected": 1.015625,
      "logps/chosen": -400.0,
      "logps/rejected": -420.0,
      "loss": 0.1837,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -14.1875,
      "rewards/margins": 6.5625,
      "rewards/rejected": -20.75,
      "step": 3460
    },
    {
      "epoch": 0.4434788165377979,
      "grad_norm": 9.592905681682163,
      "learning_rate": 1.3580796749300878e-06,
      "logits/chosen": 0.2109375,
      "logits/rejected": 0.9453125,
      "logps/chosen": -442.0,
      "logps/rejected": -422.0,
      "loss": 0.1503,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -13.9375,
      "rewards/margins": 6.53125,
      "rewards/rejected": -20.5,
      "step": 3470
    },
    {
      "epoch": 0.44475685347306537,
      "grad_norm": 6.471659876643119,
      "learning_rate": 1.3561270072416209e-06,
      "logits/chosen": 0.306640625,
      "logits/rejected": 0.8828125,
      "logps/chosen": -432.0,
      "logps/rejected": -438.0,
      "loss": 0.16,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.125,
      "rewards/margins": 6.71875,
      "rewards/rejected": -20.875,
      "step": 3480
    },
    {
      "epoch": 0.4460348904083328,
      "grad_norm": 7.67946103819814,
      "learning_rate": 1.3541827381310652e-06,
      "logits/chosen": 0.310546875,
      "logits/rejected": 0.95703125,
      "logps/chosen": -398.0,
      "logps/rejected": -436.0,
      "loss": 0.1249,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -13.5,
      "rewards/margins": 7.09375,
      "rewards/rejected": -20.625,
      "step": 3490
    },
    {
      "epoch": 0.44731292734360023,
      "grad_norm": 7.3861759247828855,
      "learning_rate": 1.3522468075656264e-06,
      "logits/chosen": 0.4921875,
      "logits/rejected": 1.0,
      "logps/chosen": -378.0,
      "logps/rejected": -438.0,
      "loss": 0.1709,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.4375,
      "rewards/margins": 7.09375,
      "rewards/rejected": -20.5,
      "step": 3500
    },
    {
      "epoch": 0.44859096427886763,
      "grad_norm": 9.892398878491829,
      "learning_rate": 1.3503191561115553e-06,
      "logits/chosen": 0.31640625,
      "logits/rejected": 0.98046875,
      "logps/chosen": -404.0,
      "logps/rejected": -444.0,
      "loss": 0.1737,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.375,
      "rewards/margins": 6.6875,
      "rewards/rejected": -21.125,
      "step": 3510
    },
    {
      "epoch": 0.4498690012141351,
      "grad_norm": 8.730640330922256,
      "learning_rate": 1.348399724926484e-06,
      "logits/chosen": 0.369140625,
      "logits/rejected": 1.0,
      "logps/chosen": -428.0,
      "logps/rejected": -436.0,
      "loss": 0.1614,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -13.9375,
      "rewards/margins": 6.78125,
      "rewards/rejected": -20.75,
      "step": 3520
    },
    {
      "epoch": 0.4511470381494025,
      "grad_norm": 6.886605194733343,
      "learning_rate": 1.346488455751882e-06,
      "logits/chosen": 0.419921875,
      "logits/rejected": 1.0234375,
      "logps/chosen": -396.0,
      "logps/rejected": -420.0,
      "loss": 0.1898,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -13.5625,
      "rewards/margins": 5.96875,
      "rewards/rejected": -19.5,
      "step": 3530
    },
    {
      "epoch": 0.45242507508466995,
      "grad_norm": 9.632768225569464,
      "learning_rate": 1.3445852909056286e-06,
      "logits/chosen": 0.396484375,
      "logits/rejected": 0.90234375,
      "logps/chosen": -400.0,
      "logps/rejected": -434.0,
      "loss": 0.1968,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.3125,
      "rewards/margins": 6.125,
      "rewards/rejected": -19.5,
      "step": 3540
    },
    {
      "epoch": 0.45370311201993735,
      "grad_norm": 7.230632072859315,
      "learning_rate": 1.3426901732747024e-06,
      "logits/chosen": 0.29296875,
      "logits/rejected": 0.79296875,
      "logps/chosen": -440.0,
      "logps/rejected": -434.0,
      "loss": 0.1531,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.625,
      "rewards/margins": 6.25,
      "rewards/rejected": -20.875,
      "step": 3550
    },
    {
      "epoch": 0.4549811489552048,
      "grad_norm": 8.639864086749046,
      "learning_rate": 1.3408030463079818e-06,
      "logits/chosen": 0.392578125,
      "logits/rejected": 1.15625,
      "logps/chosen": -404.0,
      "logps/rejected": -412.0,
      "loss": 0.1985,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.3125,
      "rewards/margins": 6.09375,
      "rewards/rejected": -20.375,
      "step": 3560
    },
    {
      "epoch": 0.4562591858904722,
      "grad_norm": 8.260064790579408,
      "learning_rate": 1.3389238540091568e-06,
      "logits/chosen": 0.458984375,
      "logits/rejected": 1.0546875,
      "logps/chosen": -408.0,
      "logps/rejected": -438.0,
      "loss": 0.1508,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.6875,
      "rewards/margins": 6.0625,
      "rewards/rejected": -20.75,
      "step": 3570
    },
    {
      "epoch": 0.45753722282573966,
      "grad_norm": 9.567599612126887,
      "learning_rate": 1.337052540929751e-06,
      "logits/chosen": 0.515625,
      "logits/rejected": 0.91796875,
      "logps/chosen": -410.0,
      "logps/rejected": -466.0,
      "loss": 0.1455,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.9375,
      "rewards/margins": 6.9375,
      "rewards/rejected": -21.875,
      "step": 3580
    },
    {
      "epoch": 0.45881525976100707,
      "grad_norm": 11.044501814992806,
      "learning_rate": 1.33518905216225e-06,
      "logits/chosen": 0.42578125,
      "logits/rejected": 0.984375,
      "logps/chosen": -424.0,
      "logps/rejected": -440.0,
      "loss": 0.1752,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.875,
      "rewards/margins": 6.40625,
      "rewards/rejected": -21.25,
      "step": 3590
    },
    {
      "epoch": 0.4600932966962745,
      "grad_norm": 8.933072486182414,
      "learning_rate": 1.3333333333333332e-06,
      "logits/chosen": 0.4296875,
      "logits/rejected": 0.91796875,
      "logps/chosen": -424.0,
      "logps/rejected": -472.0,
      "loss": 0.1584,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -15.125,
      "rewards/margins": 7.4375,
      "rewards/rejected": -22.625,
      "step": 3600
    },
    {
      "epoch": 0.4613713336315419,
      "grad_norm": 5.051404251650758,
      "learning_rate": 1.3314853305972122e-06,
      "logits/chosen": 0.26171875,
      "logits/rejected": 1.125,
      "logps/chosen": -420.0,
      "logps/rejected": -432.0,
      "loss": 0.1534,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -13.6875,
      "rewards/margins": 7.125,
      "rewards/rejected": -20.75,
      "step": 3610
    },
    {
      "epoch": 0.4626493705668094,
      "grad_norm": 7.54095569738256,
      "learning_rate": 1.3296449906290671e-06,
      "logits/chosen": 0.330078125,
      "logits/rejected": 0.8203125,
      "logps/chosen": -410.0,
      "logps/rejected": -436.0,
      "loss": 0.1538,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -13.6875,
      "rewards/margins": 6.28125,
      "rewards/rejected": -20.0,
      "step": 3620
    },
    {
      "epoch": 0.4639274075020768,
      "grad_norm": 8.304249508003823,
      "learning_rate": 1.3278122606185844e-06,
      "logits/chosen": 0.369140625,
      "logits/rejected": 0.984375,
      "logps/chosen": -394.0,
      "logps/rejected": -424.0,
      "loss": 0.1562,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -13.6875,
      "rewards/margins": 6.9375,
      "rewards/rejected": -20.625,
      "step": 3630
    },
    {
      "epoch": 0.46520544443734424,
      "grad_norm": 7.022927255282503,
      "learning_rate": 1.3259870882635918e-06,
      "logits/chosen": 0.25390625,
      "logits/rejected": 0.7421875,
      "logps/chosen": -422.0,
      "logps/rejected": -448.0,
      "loss": 0.1795,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -13.6875,
      "rewards/margins": 5.53125,
      "rewards/rejected": -19.25,
      "step": 3640
    },
    {
      "epoch": 0.46648348137261164,
      "grad_norm": 4.3837128870226865,
      "learning_rate": 1.3241694217637886e-06,
      "logits/chosen": 0.52734375,
      "logits/rejected": 1.109375,
      "logps/chosen": -402.0,
      "logps/rejected": -418.0,
      "loss": 0.1464,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.5625,
      "rewards/margins": 6.375,
      "rewards/rejected": -20.875,
      "step": 3650
    },
    {
      "epoch": 0.4677615183078791,
      "grad_norm": 8.838296981898608,
      "learning_rate": 1.3223592098145723e-06,
      "logits/chosen": 0.34765625,
      "logits/rejected": 0.7578125,
      "logps/chosen": -444.0,
      "logps/rejected": -492.0,
      "loss": 0.1463,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.6875,
      "rewards/margins": 8.375,
      "rewards/rejected": -23.0,
      "step": 3660
    },
    {
      "epoch": 0.4690395552431465,
      "grad_norm": 7.444702235663592,
      "learning_rate": 1.3205564016009555e-06,
      "logits/chosen": 0.40234375,
      "logits/rejected": 0.83984375,
      "logps/chosen": -420.0,
      "logps/rejected": -468.0,
      "loss": 0.1466,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -15.1875,
      "rewards/margins": 7.09375,
      "rewards/rejected": -22.25,
      "step": 3670
    },
    {
      "epoch": 0.47031759217841396,
      "grad_norm": 7.556726381869826,
      "learning_rate": 1.318760946791574e-06,
      "logits/chosen": 0.361328125,
      "logits/rejected": 1.15625,
      "logps/chosen": -384.0,
      "logps/rejected": -414.0,
      "loss": 0.1373,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -13.9375,
      "rewards/margins": 7.0625,
      "rewards/rejected": -21.0,
      "step": 3680
    },
    {
      "epoch": 0.47159562911368136,
      "grad_norm": 8.283712431054798,
      "learning_rate": 1.316972795532786e-06,
      "logits/chosen": 0.33203125,
      "logits/rejected": 0.83203125,
      "logps/chosen": -432.0,
      "logps/rejected": -454.0,
      "loss": 0.1686,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -14.4375,
      "rewards/margins": 6.875,
      "rewards/rejected": -21.375,
      "step": 3690
    },
    {
      "epoch": 0.4728736660489488,
      "grad_norm": 5.51389513559845,
      "learning_rate": 1.3151918984428582e-06,
      "logits/chosen": 0.396484375,
      "logits/rejected": 0.9453125,
      "logps/chosen": -400.0,
      "logps/rejected": -432.0,
      "loss": 0.1146,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.1875,
      "rewards/margins": 7.25,
      "rewards/rejected": -20.375,
      "step": 3700
    },
    {
      "epoch": 0.4741517029842162,
      "grad_norm": 8.205388024785442,
      "learning_rate": 1.313418206606237e-06,
      "logits/chosen": 0.376953125,
      "logits/rejected": 0.85546875,
      "logps/chosen": -402.0,
      "logps/rejected": -428.0,
      "loss": 0.1519,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -13.9375,
      "rewards/margins": 6.15625,
      "rewards/rejected": -20.125,
      "step": 3710
    },
    {
      "epoch": 0.4754297399194837,
      "grad_norm": 10.41339244145453,
      "learning_rate": 1.3116516715679057e-06,
      "logits/chosen": 0.314453125,
      "logits/rejected": 0.984375,
      "logps/chosen": -416.0,
      "logps/rejected": -432.0,
      "loss": 0.1678,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.4375,
      "rewards/margins": 6.5625,
      "rewards/rejected": -21.0,
      "step": 3720
    },
    {
      "epoch": 0.4767077768547511,
      "grad_norm": 8.387002353296094,
      "learning_rate": 1.3098922453278258e-06,
      "logits/chosen": 0.23046875,
      "logits/rejected": 0.90234375,
      "logps/chosen": -432.0,
      "logps/rejected": -438.0,
      "loss": 0.1508,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -15.1875,
      "rewards/margins": 6.40625,
      "rewards/rejected": -21.625,
      "step": 3730
    },
    {
      "epoch": 0.47798581379001853,
      "grad_norm": 8.151815564177527,
      "learning_rate": 1.3081398803354573e-06,
      "logits/chosen": 0.1728515625,
      "logits/rejected": 0.765625,
      "logps/chosen": -432.0,
      "logps/rejected": -452.0,
      "loss": 0.1299,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.125,
      "rewards/margins": 6.78125,
      "rewards/rejected": -21.0,
      "step": 3740
    },
    {
      "epoch": 0.47926385072528593,
      "grad_norm": 7.782428064552844,
      "learning_rate": 1.3063945294843617e-06,
      "logits/chosen": 0.185546875,
      "logits/rejected": 0.80859375,
      "logps/chosen": -400.0,
      "logps/rejected": -442.0,
      "loss": 0.1458,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -12.75,
      "rewards/margins": 7.28125,
      "rewards/rejected": -20.0,
      "step": 3750
    },
    {
      "epoch": 0.4805418876605534,
      "grad_norm": 4.282478580172844,
      "learning_rate": 1.3046561461068843e-06,
      "logits/chosen": 0.2451171875,
      "logits/rejected": 0.8203125,
      "logps/chosen": -402.0,
      "logps/rejected": -436.0,
      "loss": 0.1531,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -12.9375,
      "rewards/margins": 6.03125,
      "rewards/rejected": -19.0,
      "step": 3760
    },
    {
      "epoch": 0.4818199245958208,
      "grad_norm": 14.548789580535033,
      "learning_rate": 1.3029246839689124e-06,
      "logits/chosen": 0.2421875,
      "logits/rejected": 0.85546875,
      "logps/chosen": -434.0,
      "logps/rejected": -458.0,
      "loss": 0.1664,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.625,
      "rewards/margins": 6.65625,
      "rewards/rejected": -21.25,
      "step": 3770
    },
    {
      "epoch": 0.48309796153108825,
      "grad_norm": 4.778954836982539,
      "learning_rate": 1.3012000972647109e-06,
      "logits/chosen": 0.251953125,
      "logits/rejected": 0.87890625,
      "logps/chosen": -408.0,
      "logps/rejected": -426.0,
      "loss": 0.1678,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -13.0,
      "rewards/margins": 5.96875,
      "rewards/rejected": -19.0,
      "step": 3780
    },
    {
      "epoch": 0.48437599846635565,
      "grad_norm": 7.228107608751014,
      "learning_rate": 1.299482340611832e-06,
      "logits/chosen": 0.4296875,
      "logits/rejected": 0.9296875,
      "logps/chosen": -390.0,
      "logps/rejected": -438.0,
      "loss": 0.1693,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.0625,
      "rewards/margins": 6.0625,
      "rewards/rejected": -19.125,
      "step": 3790
    },
    {
      "epoch": 0.4856540354016231,
      "grad_norm": 7.347272930033195,
      "learning_rate": 1.2977713690461003e-06,
      "logits/chosen": 0.5078125,
      "logits/rejected": 1.03125,
      "logps/chosen": -400.0,
      "logps/rejected": -436.0,
      "loss": 0.1587,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.5625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -21.0,
      "step": 3800
    },
    {
      "epoch": 0.4869320723368905,
      "grad_norm": 10.1399952886653,
      "learning_rate": 1.296067138016669e-06,
      "logits/chosen": 0.3203125,
      "logits/rejected": 0.9765625,
      "logps/chosen": -424.0,
      "logps/rejected": -434.0,
      "loss": 0.146,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.75,
      "rewards/margins": 6.5625,
      "rewards/rejected": -21.25,
      "step": 3810
    },
    {
      "epoch": 0.48821010927215797,
      "grad_norm": 9.947586509485754,
      "learning_rate": 1.294369603381147e-06,
      "logits/chosen": 0.421875,
      "logits/rejected": 0.94921875,
      "logps/chosen": -432.0,
      "logps/rejected": -446.0,
      "loss": 0.1653,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -15.4375,
      "rewards/margins": 6.625,
      "rewards/rejected": -22.125,
      "step": 3820
    },
    {
      "epoch": 0.48948814620742537,
      "grad_norm": 6.970277288872026,
      "learning_rate": 1.2926787214007981e-06,
      "logits/chosen": 0.373046875,
      "logits/rejected": 1.09375,
      "logps/chosen": -452.0,
      "logps/rejected": -458.0,
      "loss": 0.139,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.375,
      "rewards/margins": 6.78125,
      "rewards/rejected": -23.25,
      "step": 3830
    },
    {
      "epoch": 0.4907661831426928,
      "grad_norm": 7.296618153321888,
      "learning_rate": 1.2909944487358056e-06,
      "logits/chosen": 0.43359375,
      "logits/rejected": 1.1484375,
      "logps/chosen": -414.0,
      "logps/rejected": -426.0,
      "loss": 0.1661,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.6875,
      "rewards/margins": 7.09375,
      "rewards/rejected": -21.875,
      "step": 3840
    },
    {
      "epoch": 0.4920442200779602,
      "grad_norm": 7.628725108636832,
      "learning_rate": 1.2893167424406084e-06,
      "logits/chosen": 0.3203125,
      "logits/rejected": 0.93359375,
      "logps/chosen": -426.0,
      "logps/rejected": -442.0,
      "loss": 0.157,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.5,
      "rewards/margins": 6.0625,
      "rewards/rejected": -21.625,
      "step": 3850
    },
    {
      "epoch": 0.4933222570132277,
      "grad_norm": 9.267254871858436,
      "learning_rate": 1.2876455599593008e-06,
      "logits/chosen": 0.244140625,
      "logits/rejected": 0.953125,
      "logps/chosen": -432.0,
      "logps/rejected": -452.0,
      "loss": 0.1661,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.375,
      "rewards/margins": 7.3125,
      "rewards/rejected": -22.75,
      "step": 3860
    },
    {
      "epoch": 0.4946002939484951,
      "grad_norm": 10.023922471719036,
      "learning_rate": 1.285980859121099e-06,
      "logits/chosen": 0.2138671875,
      "logits/rejected": 0.87109375,
      "logps/chosen": -408.0,
      "logps/rejected": -442.0,
      "loss": 0.1527,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -14.4375,
      "rewards/margins": 6.75,
      "rewards/rejected": -21.125,
      "step": 3870
    },
    {
      "epoch": 0.49587833088376254,
      "grad_norm": 6.920332910964864,
      "learning_rate": 1.2843225981358712e-06,
      "logits/chosen": 0.1884765625,
      "logits/rejected": 0.703125,
      "logps/chosen": -416.0,
      "logps/rejected": -450.0,
      "loss": 0.1478,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.0625,
      "rewards/margins": 7.0625,
      "rewards/rejected": -21.125,
      "step": 3880
    },
    {
      "epoch": 0.49715636781902994,
      "grad_norm": 8.396455660997331,
      "learning_rate": 1.2826707355897317e-06,
      "logits/chosen": 0.2470703125,
      "logits/rejected": 0.67578125,
      "logps/chosen": -422.0,
      "logps/rejected": -456.0,
      "loss": 0.1406,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.375,
      "rewards/margins": 6.96875,
      "rewards/rejected": -21.375,
      "step": 3890
    },
    {
      "epoch": 0.4984344047542974,
      "grad_norm": 7.935374411801874,
      "learning_rate": 1.281025230440697e-06,
      "logits/chosen": 0.1669921875,
      "logits/rejected": 0.78125,
      "logps/chosen": -420.0,
      "logps/rejected": -440.0,
      "loss": 0.1726,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.625,
      "rewards/margins": 6.53125,
      "rewards/rejected": -21.25,
      "step": 3900
    },
    {
      "epoch": 0.4997124416895648,
      "grad_norm": 8.061537819311052,
      "learning_rate": 1.2793860420144025e-06,
      "logits/chosen": 0.1376953125,
      "logits/rejected": 0.69921875,
      "logps/chosen": -420.0,
      "logps/rejected": -444.0,
      "loss": 0.1636,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -13.4375,
      "rewards/margins": 6.96875,
      "rewards/rejected": -20.375,
      "step": 3910
    },
    {
      "epoch": 0.5009904786248323,
      "grad_norm": 5.874874146230511,
      "learning_rate": 1.2777531299998798e-06,
      "logits/chosen": -0.033447265625,
      "logits/rejected": 0.51953125,
      "logps/chosen": -440.0,
      "logps/rejected": -460.0,
      "loss": 0.1414,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -13.4375,
      "rewards/margins": 6.71875,
      "rewards/rejected": -20.125,
      "step": 3920
    },
    {
      "epoch": 0.5022685155600997,
      "grad_norm": 9.546271692128585,
      "learning_rate": 1.2761264544453928e-06,
      "logits/chosen": 0.220703125,
      "logits/rejected": 0.6640625,
      "logps/chosen": -406.0,
      "logps/rejected": -430.0,
      "loss": 0.1452,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -12.625,
      "rewards/margins": 5.75,
      "rewards/rejected": -18.375,
      "step": 3930
    },
    {
      "epoch": 0.5035465524953671,
      "grad_norm": 11.074945799321585,
      "learning_rate": 1.2745059757543324e-06,
      "logits/chosen": 0.263671875,
      "logits/rejected": 0.7890625,
      "logps/chosen": -424.0,
      "logps/rejected": -446.0,
      "loss": 0.1596,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.125,
      "rewards/margins": 6.21875,
      "rewards/rejected": -21.375,
      "step": 3940
    },
    {
      "epoch": 0.5048245894306346,
      "grad_norm": 9.37953994257787,
      "learning_rate": 1.272891654681168e-06,
      "logits/chosen": 0.1650390625,
      "logits/rejected": 0.89453125,
      "logps/chosen": -398.0,
      "logps/rejected": -422.0,
      "loss": 0.1421,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -13.5,
      "rewards/margins": 6.15625,
      "rewards/rejected": -19.625,
      "step": 3950
    },
    {
      "epoch": 0.506102626365902,
      "grad_norm": 8.960530129601892,
      "learning_rate": 1.2712834523274563e-06,
      "logits/chosen": 0.1318359375,
      "logits/rejected": 0.765625,
      "logps/chosen": -402.0,
      "logps/rejected": -432.0,
      "loss": 0.144,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.5625,
      "rewards/margins": 6.1875,
      "rewards/rejected": -19.75,
      "step": 3960
    },
    {
      "epoch": 0.5073806633011694,
      "grad_norm": 13.179154125951804,
      "learning_rate": 1.2696813301379032e-06,
      "logits/chosen": 0.31640625,
      "logits/rejected": 0.89453125,
      "logps/chosen": -410.0,
      "logps/rejected": -436.0,
      "loss": 0.1553,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -14.8125,
      "rewards/margins": 6.75,
      "rewards/rejected": -21.5,
      "step": 3970
    },
    {
      "epoch": 0.5086587002364368,
      "grad_norm": 8.62287859910909,
      "learning_rate": 1.2680852498964829e-06,
      "logits/chosen": 0.3203125,
      "logits/rejected": 0.89453125,
      "logps/chosen": -400.0,
      "logps/rejected": -426.0,
      "loss": 0.1762,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.0625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -20.75,
      "step": 3980
    },
    {
      "epoch": 0.5099367371717043,
      "grad_norm": 6.055378036882499,
      "learning_rate": 1.266495173722607e-06,
      "logits/chosen": 0.3203125,
      "logits/rejected": 0.98046875,
      "logps/chosen": -380.0,
      "logps/rejected": -420.0,
      "loss": 0.1506,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.125,
      "rewards/margins": 6.25,
      "rewards/rejected": -19.375,
      "step": 3990
    },
    {
      "epoch": 0.5112147741069717,
      "grad_norm": 7.932146564146819,
      "learning_rate": 1.2649110640673517e-06,
      "logits/chosen": 0.318359375,
      "logits/rejected": 1.0390625,
      "logps/chosen": -400.0,
      "logps/rejected": -410.0,
      "loss": 0.1371,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -14.3125,
      "rewards/margins": 5.875,
      "rewards/rejected": -20.25,
      "step": 4000
    },
    {
      "epoch": 0.5124928110422391,
      "grad_norm": 13.004968212309231,
      "learning_rate": 1.2633328837097308e-06,
      "logits/chosen": 0.357421875,
      "logits/rejected": 0.921875,
      "logps/chosen": -432.0,
      "logps/rejected": -468.0,
      "loss": 0.1357,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -15.75,
      "rewards/margins": 6.8125,
      "rewards/rejected": -22.5,
      "step": 4010
    },
    {
      "epoch": 0.5137708479775065,
      "grad_norm": 10.94246431260662,
      "learning_rate": 1.2617605957530233e-06,
      "logits/chosen": 0.39453125,
      "logits/rejected": 0.94140625,
      "logps/chosen": -436.0,
      "logps/rejected": -456.0,
      "loss": 0.1409,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -16.75,
      "rewards/margins": 6.28125,
      "rewards/rejected": -23.0,
      "step": 4020
    },
    {
      "epoch": 0.515048884912774,
      "grad_norm": 6.689700282912666,
      "learning_rate": 1.2601941636211516e-06,
      "logits/chosen": 0.26171875,
      "logits/rejected": 1.0078125,
      "logps/chosen": -456.0,
      "logps/rejected": -474.0,
      "loss": 0.1568,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -16.75,
      "rewards/margins": 6.9375,
      "rewards/rejected": -23.75,
      "step": 4030
    },
    {
      "epoch": 0.5163269218480414,
      "grad_norm": 8.475709812509852,
      "learning_rate": 1.2586335510551052e-06,
      "logits/chosen": 0.30078125,
      "logits/rejected": 0.82421875,
      "logps/chosen": -436.0,
      "logps/rejected": -458.0,
      "loss": 0.1469,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.25,
      "rewards/margins": 6.25,
      "rewards/rejected": -22.5,
      "step": 4040
    },
    {
      "epoch": 0.5176049587833088,
      "grad_norm": 11.727660469630143,
      "learning_rate": 1.2570787221094177e-06,
      "logits/chosen": 0.19140625,
      "logits/rejected": 0.6640625,
      "logps/chosen": -432.0,
      "logps/rejected": -480.0,
      "loss": 0.1743,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.5625,
      "rewards/margins": 8.6875,
      "rewards/rejected": -23.25,
      "step": 4050
    },
    {
      "epoch": 0.5188829957185762,
      "grad_norm": 9.417199407201368,
      "learning_rate": 1.255529641148689e-06,
      "logits/chosen": 0.1806640625,
      "logits/rejected": 0.7421875,
      "logps/chosen": -418.0,
      "logps/rejected": -438.0,
      "loss": 0.1688,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -14.125,
      "rewards/margins": 6.40625,
      "rewards/rejected": -20.5,
      "step": 4060
    },
    {
      "epoch": 0.5201610326538437,
      "grad_norm": 8.259302654348371,
      "learning_rate": 1.2539862728441536e-06,
      "logits/chosen": 0.37890625,
      "logits/rejected": 0.95703125,
      "logps/chosen": -408.0,
      "logps/rejected": -444.0,
      "loss": 0.1716,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.8125,
      "rewards/margins": 6.4375,
      "rewards/rejected": -21.25,
      "step": 4070
    },
    {
      "epoch": 0.5214390695891111,
      "grad_norm": 11.630948143113944,
      "learning_rate": 1.252448582170299e-06,
      "logits/chosen": 0.326171875,
      "logits/rejected": 1.0078125,
      "logps/chosen": -412.0,
      "logps/rejected": -432.0,
      "loss": 0.1431,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.0,
      "rewards/margins": 5.8125,
      "rewards/rejected": -19.875,
      "step": 4080
    },
    {
      "epoch": 0.5227171065243785,
      "grad_norm": 10.027952432721076,
      "learning_rate": 1.2509165344015243e-06,
      "logits/chosen": 0.216796875,
      "logits/rejected": 0.90234375,
      "logps/chosen": -424.0,
      "logps/rejected": -440.0,
      "loss": 0.1271,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -13.5625,
      "rewards/margins": 7.0,
      "rewards/rejected": -20.5,
      "step": 4090
    },
    {
      "epoch": 0.5239951434596459,
      "grad_norm": 7.7484515063456,
      "learning_rate": 1.2493900951088486e-06,
      "logits/chosen": 0.3828125,
      "logits/rejected": 0.875,
      "logps/chosen": -408.0,
      "logps/rejected": -426.0,
      "loss": 0.1505,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.1875,
      "rewards/margins": 7.0,
      "rewards/rejected": -20.25,
      "step": 4100
    },
    {
      "epoch": 0.5252731803949134,
      "grad_norm": 5.20522645051547,
      "learning_rate": 1.2478692301566601e-06,
      "logits/chosen": 0.296875,
      "logits/rejected": 0.97265625,
      "logps/chosen": -414.0,
      "logps/rejected": -460.0,
      "loss": 0.1438,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -13.5,
      "rewards/margins": 7.15625,
      "rewards/rejected": -20.625,
      "step": 4110
    },
    {
      "epoch": 0.5265512173301808,
      "grad_norm": 4.623413228679383,
      "learning_rate": 1.2463539056995116e-06,
      "logits/chosen": 0.435546875,
      "logits/rejected": 0.9765625,
      "logps/chosen": -392.0,
      "logps/rejected": -440.0,
      "loss": 0.1451,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.0625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -20.75,
      "step": 4120
    },
    {
      "epoch": 0.5278292542654482,
      "grad_norm": 6.1550488901910025,
      "learning_rate": 1.2448440881789541e-06,
      "logits/chosen": 0.234375,
      "logits/rejected": 0.62109375,
      "logps/chosen": -436.0,
      "logps/rejected": -470.0,
      "loss": 0.1356,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.0625,
      "rewards/margins": 7.90625,
      "rewards/rejected": -22.0,
      "step": 4130
    },
    {
      "epoch": 0.5291072912007156,
      "grad_norm": 9.105608468177582,
      "learning_rate": 1.2433397443204184e-06,
      "logits/chosen": 0.1845703125,
      "logits/rejected": 0.84765625,
      "logps/chosen": -418.0,
      "logps/rejected": -462.0,
      "loss": 0.1504,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.1875,
      "rewards/margins": 6.90625,
      "rewards/rejected": -21.125,
      "step": 4140
    },
    {
      "epoch": 0.5303853281359832,
      "grad_norm": 6.757818547677151,
      "learning_rate": 1.2418408411301324e-06,
      "logits/chosen": 0.35546875,
      "logits/rejected": 0.95703125,
      "logps/chosen": -392.0,
      "logps/rejected": -438.0,
      "loss": 0.1403,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.375,
      "rewards/margins": 6.5,
      "rewards/rejected": -20.875,
      "step": 4150
    },
    {
      "epoch": 0.5316633650712506,
      "grad_norm": 8.849303859710535,
      "learning_rate": 1.2403473458920844e-06,
      "logits/chosen": 0.2275390625,
      "logits/rejected": 0.92578125,
      "logps/chosen": -382.0,
      "logps/rejected": -416.0,
      "loss": 0.1578,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -12.4375,
      "rewards/margins": 7.375,
      "rewards/rejected": -19.75,
      "step": 4160
    },
    {
      "epoch": 0.532941402006518,
      "grad_norm": 5.855710732354923,
      "learning_rate": 1.2388592261650217e-06,
      "logits/chosen": 0.443359375,
      "logits/rejected": 1.1796875,
      "logps/chosen": -402.0,
      "logps/rejected": -418.0,
      "loss": 0.1276,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.125,
      "rewards/margins": 6.6875,
      "rewards/rejected": -20.875,
      "step": 4170
    },
    {
      "epoch": 0.5342194389417854,
      "grad_norm": 7.276502206082859,
      "learning_rate": 1.2373764497794918e-06,
      "logits/chosen": 0.1337890625,
      "logits/rejected": 0.734375,
      "logps/chosen": -428.0,
      "logps/rejected": -458.0,
      "loss": 0.1654,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -15.125,
      "rewards/margins": 6.59375,
      "rewards/rejected": -21.75,
      "step": 4180
    },
    {
      "epoch": 0.5354974758770529,
      "grad_norm": 7.405285780180957,
      "learning_rate": 1.2358989848349217e-06,
      "logits/chosen": 0.333984375,
      "logits/rejected": 0.890625,
      "logps/chosen": -384.0,
      "logps/rejected": -408.0,
      "loss": 0.139,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.0625,
      "rewards/margins": 6.28125,
      "rewards/rejected": -19.25,
      "step": 4190
    },
    {
      "epoch": 0.5367755128123203,
      "grad_norm": 7.713685499643182,
      "learning_rate": 1.2344267996967353e-06,
      "logits/chosen": 0.1796875,
      "logits/rejected": 0.7265625,
      "logps/chosen": -400.0,
      "logps/rejected": -440.0,
      "loss": 0.1503,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -13.625,
      "rewards/margins": 6.34375,
      "rewards/rejected": -20.0,
      "step": 4200
    },
    {
      "epoch": 0.5380535497475877,
      "grad_norm": 8.523559967464994,
      "learning_rate": 1.2329598629935076e-06,
      "logits/chosen": 0.265625,
      "logits/rejected": 0.94140625,
      "logps/chosen": -412.0,
      "logps/rejected": -436.0,
      "loss": 0.1523,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -13.4375,
      "rewards/margins": 6.84375,
      "rewards/rejected": -20.25,
      "step": 4210
    },
    {
      "epoch": 0.5393315866828551,
      "grad_norm": 7.586819183509101,
      "learning_rate": 1.2314981436141583e-06,
      "logits/chosen": 0.294921875,
      "logits/rejected": 0.8203125,
      "logps/chosen": -404.0,
      "logps/rejected": -440.0,
      "loss": 0.1161,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.375,
      "rewards/margins": 7.28125,
      "rewards/rejected": -21.625,
      "step": 4220
    },
    {
      "epoch": 0.5406096236181226,
      "grad_norm": 9.989537897153312,
      "learning_rate": 1.2300416107051802e-06,
      "logits/chosen": 0.197265625,
      "logits/rejected": 0.78515625,
      "logps/chosen": -416.0,
      "logps/rejected": -472.0,
      "loss": 0.1512,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.5625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -22.625,
      "step": 4230
    },
    {
      "epoch": 0.54188766055339,
      "grad_norm": 9.543136127855679,
      "learning_rate": 1.2285902336679024e-06,
      "logits/chosen": 0.2353515625,
      "logits/rejected": 0.96484375,
      "logps/chosen": -414.0,
      "logps/rejected": -448.0,
      "loss": 0.133,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.1875,
      "rewards/margins": 6.4375,
      "rewards/rejected": -21.625,
      "step": 4240
    },
    {
      "epoch": 0.5431656974886574,
      "grad_norm": 12.392920315419518,
      "learning_rate": 1.2271439821557926e-06,
      "logits/chosen": 0.3671875,
      "logits/rejected": 0.91015625,
      "logps/chosen": -410.0,
      "logps/rejected": -436.0,
      "loss": 0.1422,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.5,
      "rewards/margins": 6.34375,
      "rewards/rejected": -20.875,
      "step": 4250
    },
    {
      "epoch": 0.5444437344239248,
      "grad_norm": 8.755726499979085,
      "learning_rate": 1.225702826071791e-06,
      "logits/chosen": 0.1103515625,
      "logits/rejected": 0.7578125,
      "logps/chosen": -408.0,
      "logps/rejected": -434.0,
      "loss": 0.165,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -13.4375,
      "rewards/margins": 6.46875,
      "rewards/rejected": -19.875,
      "step": 4260
    },
    {
      "epoch": 0.5457217713591923,
      "grad_norm": 7.048265804450963,
      "learning_rate": 1.2242667355656797e-06,
      "logits/chosen": 0.20703125,
      "logits/rejected": 0.71875,
      "logps/chosen": -420.0,
      "logps/rejected": -444.0,
      "loss": 0.1424,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.3125,
      "rewards/margins": 7.0,
      "rewards/rejected": -20.375,
      "step": 4270
    },
    {
      "epoch": 0.5469998082944597,
      "grad_norm": 7.963643906072645,
      "learning_rate": 1.2228356810314862e-06,
      "logits/chosen": 0.2314453125,
      "logits/rejected": 0.75390625,
      "logps/chosen": -390.0,
      "logps/rejected": -420.0,
      "loss": 0.133,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -12.8125,
      "rewards/margins": 5.9375,
      "rewards/rejected": -18.75,
      "step": 4280
    },
    {
      "epoch": 0.5482778452297271,
      "grad_norm": 9.724179481532149,
      "learning_rate": 1.2214096331049186e-06,
      "logits/chosen": 0.2412109375,
      "logits/rejected": 0.84375,
      "logps/chosen": -390.0,
      "logps/rejected": -424.0,
      "loss": 0.1587,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -11.75,
      "rewards/margins": 7.3125,
      "rewards/rejected": -19.0,
      "step": 4290
    },
    {
      "epoch": 0.5495558821649945,
      "grad_norm": 9.072035456544127,
      "learning_rate": 1.2199885626608373e-06,
      "logits/chosen": 0.345703125,
      "logits/rejected": 0.83203125,
      "logps/chosen": -390.0,
      "logps/rejected": -418.0,
      "loss": 0.1515,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -13.0625,
      "rewards/margins": 6.40625,
      "rewards/rejected": -19.5,
      "step": 4300
    },
    {
      "epoch": 0.550833919100262,
      "grad_norm": 9.932415245703808,
      "learning_rate": 1.2185724408107546e-06,
      "logits/chosen": 0.267578125,
      "logits/rejected": 0.71484375,
      "logps/chosen": -396.0,
      "logps/rejected": -430.0,
      "loss": 0.1534,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -12.5625,
      "rewards/margins": 6.1875,
      "rewards/rejected": -18.75,
      "step": 4310
    },
    {
      "epoch": 0.5521119560355294,
      "grad_norm": 12.130834322471259,
      "learning_rate": 1.2171612389003689e-06,
      "logits/chosen": 0.2890625,
      "logits/rejected": 0.86328125,
      "logps/chosen": -392.0,
      "logps/rejected": -440.0,
      "loss": 0.1427,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.5,
      "rewards/margins": 6.8125,
      "rewards/rejected": -20.375,
      "step": 4320
    },
    {
      "epoch": 0.5533899929707968,
      "grad_norm": 8.871665536276053,
      "learning_rate": 1.2157549285071297e-06,
      "logits/chosen": 0.255859375,
      "logits/rejected": 0.9375,
      "logps/chosen": -416.0,
      "logps/rejected": -428.0,
      "loss": 0.1329,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.3125,
      "rewards/margins": 6.25,
      "rewards/rejected": -20.5,
      "step": 4330
    },
    {
      "epoch": 0.5546680299060642,
      "grad_norm": 8.357146776608477,
      "learning_rate": 1.2143534814378327e-06,
      "logits/chosen": 0.125,
      "logits/rejected": 0.703125,
      "logps/chosen": -412.0,
      "logps/rejected": -436.0,
      "loss": 0.1198,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -13.6875,
      "rewards/margins": 6.125,
      "rewards/rejected": -19.875,
      "step": 4340
    },
    {
      "epoch": 0.5559460668413317,
      "grad_norm": 7.57692738399649,
      "learning_rate": 1.2129568697262454e-06,
      "logits/chosen": 0.2255859375,
      "logits/rejected": 0.80859375,
      "logps/chosen": -398.0,
      "logps/rejected": -426.0,
      "loss": 0.1476,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -12.375,
      "rewards/margins": 7.125,
      "rewards/rejected": -19.5,
      "step": 4350
    },
    {
      "epoch": 0.5572241037765991,
      "grad_norm": 7.293493295257088,
      "learning_rate": 1.2115650656307653e-06,
      "logits/chosen": 0.10107421875,
      "logits/rejected": 0.875,
      "logps/chosen": -412.0,
      "logps/rejected": -402.0,
      "loss": 0.1363,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -12.9375,
      "rewards/margins": 6.5,
      "rewards/rejected": -19.5,
      "step": 4360
    },
    {
      "epoch": 0.5585021407118665,
      "grad_norm": 9.352097716964115,
      "learning_rate": 1.210178041632103e-06,
      "logits/chosen": 0.240234375,
      "logits/rejected": 0.85546875,
      "logps/chosen": -412.0,
      "logps/rejected": -454.0,
      "loss": 0.1639,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.625,
      "rewards/margins": 7.0625,
      "rewards/rejected": -21.625,
      "step": 4370
    },
    {
      "epoch": 0.559780177647134,
      "grad_norm": 7.991618319290518,
      "learning_rate": 1.2087957704309988e-06,
      "logits/chosen": 0.1787109375,
      "logits/rejected": 0.7734375,
      "logps/chosen": -416.0,
      "logps/rejected": -424.0,
      "loss": 0.1213,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -13.875,
      "rewards/margins": 6.5625,
      "rewards/rejected": -20.5,
      "step": 4380
    },
    {
      "epoch": 0.5610582145824015,
      "grad_norm": 12.0437054409563,
      "learning_rate": 1.2074182249459642e-06,
      "logits/chosen": 0.2001953125,
      "logits/rejected": 0.99609375,
      "logps/chosen": -402.0,
      "logps/rejected": -408.0,
      "loss": 0.1631,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.25,
      "rewards/margins": 6.8125,
      "rewards/rejected": -20.0,
      "step": 4390
    },
    {
      "epoch": 0.5623362515176689,
      "grad_norm": 8.685879881529042,
      "learning_rate": 1.2060453783110545e-06,
      "logits/chosen": 0.1123046875,
      "logits/rejected": 0.83203125,
      "logps/chosen": -408.0,
      "logps/rejected": -424.0,
      "loss": 0.1504,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -14.5,
      "rewards/margins": 6.46875,
      "rewards/rejected": -21.0,
      "step": 4400
    },
    {
      "epoch": 0.5636142884529363,
      "grad_norm": 12.649269571698579,
      "learning_rate": 1.2046772038736682e-06,
      "logits/chosen": 0.119140625,
      "logits/rejected": 0.80078125,
      "logps/chosen": -396.0,
      "logps/rejected": -406.0,
      "loss": 0.1184,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -13.5,
      "rewards/margins": 6.53125,
      "rewards/rejected": -20.0,
      "step": 4410
    },
    {
      "epoch": 0.5648923253882037,
      "grad_norm": 7.558521401635597,
      "learning_rate": 1.2033136751923736e-06,
      "logits/chosen": 0.0478515625,
      "logits/rejected": 0.55078125,
      "logps/chosen": -432.0,
      "logps/rejected": -474.0,
      "loss": 0.1503,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.6875,
      "rewards/margins": 7.1875,
      "rewards/rejected": -21.875,
      "step": 4420
    },
    {
      "epoch": 0.5661703623234712,
      "grad_norm": 6.127466754989835,
      "learning_rate": 1.201954766034762e-06,
      "logits/chosen": 0.103515625,
      "logits/rejected": 0.66015625,
      "logps/chosen": -408.0,
      "logps/rejected": -436.0,
      "loss": 0.1434,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -12.75,
      "rewards/margins": 7.53125,
      "rewards/rejected": -20.25,
      "step": 4430
    },
    {
      "epoch": 0.5674483992587386,
      "grad_norm": 8.885939296686109,
      "learning_rate": 1.2006004503753285e-06,
      "logits/chosen": 0.1748046875,
      "logits/rejected": 0.59765625,
      "logps/chosen": -380.0,
      "logps/rejected": -452.0,
      "loss": 0.1376,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -13.875,
      "rewards/margins": 6.59375,
      "rewards/rejected": -20.375,
      "step": 4440
    },
    {
      "epoch": 0.568726436194006,
      "grad_norm": 10.678217137089993,
      "learning_rate": 1.1992507023933782e-06,
      "logits/chosen": 0.134765625,
      "logits/rejected": 0.625,
      "logps/chosen": -432.0,
      "logps/rejected": -440.0,
      "loss": 0.127,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.0625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -20.75,
      "step": 4450
    },
    {
      "epoch": 0.5700044731292734,
      "grad_norm": 5.91200528125887,
      "learning_rate": 1.1979054964709597e-06,
      "logits/chosen": 0.049560546875,
      "logits/rejected": 0.6484375,
      "logps/chosen": -426.0,
      "logps/rejected": -456.0,
      "loss": 0.1463,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.8125,
      "rewards/margins": 7.53125,
      "rewards/rejected": -21.375,
      "step": 4460
    },
    {
      "epoch": 0.5712825100645409,
      "grad_norm": 8.626698056492438,
      "learning_rate": 1.1965648071908207e-06,
      "logits/chosen": 0.322265625,
      "logits/rejected": 0.8203125,
      "logps/chosen": -404.0,
      "logps/rejected": -438.0,
      "loss": 0.1199,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.625,
      "rewards/margins": 6.53125,
      "rewards/rejected": -21.125,
      "step": 4470
    },
    {
      "epoch": 0.5725605469998083,
      "grad_norm": 9.92014832210552,
      "learning_rate": 1.1952286093343935e-06,
      "logits/chosen": 0.30078125,
      "logits/rejected": 0.81640625,
      "logps/chosen": -420.0,
      "logps/rejected": -450.0,
      "loss": 0.1434,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.0,
      "rewards/margins": 7.375,
      "rewards/rejected": -22.375,
      "step": 4480
    },
    {
      "epoch": 0.5738385839350757,
      "grad_norm": 9.221810255793809,
      "learning_rate": 1.1938968778798005e-06,
      "logits/chosen": 0.1875,
      "logits/rejected": 0.69921875,
      "logps/chosen": -432.0,
      "logps/rejected": -460.0,
      "loss": 0.1376,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.875,
      "rewards/margins": 6.53125,
      "rewards/rejected": -21.375,
      "step": 4490
    },
    {
      "epoch": 0.5751166208703431,
      "grad_norm": 9.023979252838528,
      "learning_rate": 1.1925695879998878e-06,
      "logits/chosen": 0.271484375,
      "logits/rejected": 0.88671875,
      "logps/chosen": -442.0,
      "logps/rejected": -476.0,
      "loss": 0.1553,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.1875,
      "rewards/rejected": -23.25,
      "step": 4500
    },
    {
      "epoch": 0.5763946578056106,
      "grad_norm": 10.346925335957318,
      "learning_rate": 1.1912467150602794e-06,
      "logits/chosen": 0.431640625,
      "logits/rejected": 0.80078125,
      "logps/chosen": -406.0,
      "logps/rejected": -454.0,
      "loss": 0.1417,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.8125,
      "rewards/margins": 7.0,
      "rewards/rejected": -21.875,
      "step": 4510
    },
    {
      "epoch": 0.577672694740878,
      "grad_norm": 6.225959715565491,
      "learning_rate": 1.189928234617459e-06,
      "logits/chosen": 0.439453125,
      "logits/rejected": 1.140625,
      "logps/chosen": -422.0,
      "logps/rejected": -438.0,
      "loss": 0.1373,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.875,
      "rewards/margins": 6.46875,
      "rewards/rejected": -22.375,
      "step": 4520
    },
    {
      "epoch": 0.5789507316761454,
      "grad_norm": 7.405387432548825,
      "learning_rate": 1.1886141224168716e-06,
      "logits/chosen": 0.3359375,
      "logits/rejected": 0.8515625,
      "logps/chosen": -434.0,
      "logps/rejected": -460.0,
      "loss": 0.1245,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.625,
      "rewards/margins": 7.0,
      "rewards/rejected": -22.625,
      "step": 4530
    },
    {
      "epoch": 0.5802287686114128,
      "grad_norm": 7.905526742557969,
      "learning_rate": 1.1873043543910495e-06,
      "logits/chosen": 0.302734375,
      "logits/rejected": 0.81640625,
      "logps/chosen": -414.0,
      "logps/rejected": -446.0,
      "loss": 0.1481,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.9375,
      "rewards/margins": 6.8125,
      "rewards/rejected": -20.75,
      "step": 4540
    },
    {
      "epoch": 0.5815068055466803,
      "grad_norm": 9.299350069018523,
      "learning_rate": 1.1859989066577617e-06,
      "logits/chosen": 0.18359375,
      "logits/rejected": 0.80859375,
      "logps/chosen": -404.0,
      "logps/rejected": -438.0,
      "loss": 0.1407,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -12.6875,
      "rewards/margins": 6.96875,
      "rewards/rejected": -19.625,
      "step": 4550
    },
    {
      "epoch": 0.5827848424819477,
      "grad_norm": 8.702796594011065,
      "learning_rate": 1.1846977555181846e-06,
      "logits/chosen": 0.1416015625,
      "logits/rejected": 0.78125,
      "logps/chosen": -418.0,
      "logps/rejected": -446.0,
      "loss": 0.1563,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.5625,
      "rewards/margins": 6.71875,
      "rewards/rejected": -21.25,
      "step": 4560
    },
    {
      "epoch": 0.5840628794172151,
      "grad_norm": 7.585051654648617,
      "learning_rate": 1.1834008774550946e-06,
      "logits/chosen": 0.330078125,
      "logits/rejected": 0.9140625,
      "logps/chosen": -430.0,
      "logps/rejected": -458.0,
      "loss": 0.1562,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -15.5625,
      "rewards/margins": 6.625,
      "rewards/rejected": -22.125,
      "step": 4570
    },
    {
      "epoch": 0.5853409163524825,
      "grad_norm": 6.479732984508555,
      "learning_rate": 1.1821082491310835e-06,
      "logits/chosen": 0.3671875,
      "logits/rejected": 0.953125,
      "logps/chosen": -428.0,
      "logps/rejected": -460.0,
      "loss": 0.1489,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.1875,
      "rewards/rejected": -23.625,
      "step": 4580
    },
    {
      "epoch": 0.58661895328775,
      "grad_norm": 7.112190596505058,
      "learning_rate": 1.1808198473867937e-06,
      "logits/chosen": 0.271484375,
      "logits/rejected": 0.828125,
      "logps/chosen": -432.0,
      "logps/rejected": -444.0,
      "loss": 0.1378,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.4375,
      "rewards/margins": 6.59375,
      "rewards/rejected": -22.0,
      "step": 4590
    },
    {
      "epoch": 0.5878969902230174,
      "grad_norm": 9.725357482510336,
      "learning_rate": 1.179535649239177e-06,
      "logits/chosen": 0.07421875,
      "logits/rejected": 0.640625,
      "logps/chosen": -398.0,
      "logps/rejected": -426.0,
      "loss": 0.1229,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -12.4375,
      "rewards/margins": 7.0625,
      "rewards/rejected": -19.5,
      "step": 4600
    },
    {
      "epoch": 0.5891750271582848,
      "grad_norm": 7.92032761036127,
      "learning_rate": 1.178255631879771e-06,
      "logits/chosen": 0.203125,
      "logits/rejected": 0.8515625,
      "logps/chosen": -400.0,
      "logps/rejected": -414.0,
      "loss": 0.1184,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.25,
      "rewards/margins": 6.375,
      "rewards/rejected": -19.625,
      "step": 4610
    },
    {
      "epoch": 0.5904530640935522,
      "grad_norm": 6.963608416885508,
      "learning_rate": 1.1769797726729992e-06,
      "logits/chosen": 0.04150390625,
      "logits/rejected": 0.67578125,
      "logps/chosen": -404.0,
      "logps/rejected": -428.0,
      "loss": 0.1094,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -12.5,
      "rewards/margins": 6.53125,
      "rewards/rejected": -19.0,
      "step": 4620
    },
    {
      "epoch": 0.5917311010288198,
      "grad_norm": 5.945208329662066,
      "learning_rate": 1.1757080491544881e-06,
      "logits/chosen": 0.1591796875,
      "logits/rejected": 0.6875,
      "logps/chosen": -414.0,
      "logps/rejected": -454.0,
      "loss": 0.1134,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.6875,
      "rewards/margins": 7.0,
      "rewards/rejected": -20.625,
      "step": 4630
    },
    {
      "epoch": 0.5930091379640872,
      "grad_norm": 10.476606026729366,
      "learning_rate": 1.1744404390294068e-06,
      "logits/chosen": 0.232421875,
      "logits/rejected": 0.796875,
      "logps/chosen": -432.0,
      "logps/rejected": -456.0,
      "loss": 0.1381,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.4375,
      "rewards/margins": 6.09375,
      "rewards/rejected": -21.5,
      "step": 4640
    },
    {
      "epoch": 0.5942871748993546,
      "grad_norm": 6.672420481480638,
      "learning_rate": 1.1731769201708264e-06,
      "logits/chosen": 0.32421875,
      "logits/rejected": 0.76953125,
      "logps/chosen": -412.0,
      "logps/rejected": -442.0,
      "loss": 0.1396,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.375,
      "rewards/margins": 6.71875,
      "rewards/rejected": -22.125,
      "step": 4650
    },
    {
      "epoch": 0.595565211834622,
      "grad_norm": 8.452477333471757,
      "learning_rate": 1.1719174706180952e-06,
      "logits/chosen": 0.13671875,
      "logits/rejected": 0.8125,
      "logps/chosen": -440.0,
      "logps/rejected": -456.0,
      "loss": 0.1508,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.5625,
      "rewards/margins": 7.4375,
      "rewards/rejected": -23.0,
      "step": 4660
    },
    {
      "epoch": 0.5968432487698895,
      "grad_norm": 7.997083621003591,
      "learning_rate": 1.1706620685752386e-06,
      "logits/chosen": 0.17578125,
      "logits/rejected": 0.796875,
      "logps/chosen": -416.0,
      "logps/rejected": -446.0,
      "loss": 0.1205,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.25,
      "rewards/margins": 7.25,
      "rewards/rejected": -22.5,
      "step": 4670
    },
    {
      "epoch": 0.5981212857051569,
      "grad_norm": 11.006260140922988,
      "learning_rate": 1.1694106924093723e-06,
      "logits/chosen": 0.251953125,
      "logits/rejected": 1.0078125,
      "logps/chosen": -418.0,
      "logps/rejected": -450.0,
      "loss": 0.1317,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -15.6875,
      "rewards/margins": 8.125,
      "rewards/rejected": -23.75,
      "step": 4680
    },
    {
      "epoch": 0.5993993226404243,
      "grad_norm": 12.745977417727884,
      "learning_rate": 1.1681633206491381e-06,
      "logits/chosen": 0.3515625,
      "logits/rejected": 0.87109375,
      "logps/chosen": -436.0,
      "logps/rejected": -452.0,
      "loss": 0.1389,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.0,
      "rewards/rejected": -23.625,
      "step": 4690
    },
    {
      "epoch": 0.6006773595756917,
      "grad_norm": 10.145154776400462,
      "learning_rate": 1.1669199319831564e-06,
      "logits/chosen": 0.24609375,
      "logits/rejected": 0.94140625,
      "logps/chosen": -422.0,
      "logps/rejected": -444.0,
      "loss": 0.1501,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.25,
      "rewards/margins": 7.4375,
      "rewards/rejected": -22.75,
      "step": 4700
    },
    {
      "epoch": 0.6019553965109592,
      "grad_norm": 8.417870665473714,
      "learning_rate": 1.1656805052584958e-06,
      "logits/chosen": 0.197265625,
      "logits/rejected": 0.97265625,
      "logps/chosen": -424.0,
      "logps/rejected": -448.0,
      "loss": 0.1279,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.75,
      "rewards/margins": 7.5,
      "rewards/rejected": -22.25,
      "step": 4710
    },
    {
      "epoch": 0.6032334334462266,
      "grad_norm": 3.7532823684978625,
      "learning_rate": 1.164445019479164e-06,
      "logits/chosen": 0.1923828125,
      "logits/rejected": 0.87109375,
      "logps/chosen": -410.0,
      "logps/rejected": -430.0,
      "loss": 0.1074,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.0625,
      "rewards/margins": 6.84375,
      "rewards/rejected": -20.875,
      "step": 4720
    },
    {
      "epoch": 0.604511470381494,
      "grad_norm": 8.880884788683034,
      "learning_rate": 1.1632134538046105e-06,
      "logits/chosen": 0.06005859375,
      "logits/rejected": 0.70703125,
      "logps/chosen": -364.0,
      "logps/rejected": -414.0,
      "loss": 0.1367,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -12.3125,
      "rewards/margins": 6.28125,
      "rewards/rejected": -18.625,
      "step": 4730
    },
    {
      "epoch": 0.6057895073167614,
      "grad_norm": 14.35632250165327,
      "learning_rate": 1.1619857875482536e-06,
      "logits/chosen": 0.1611328125,
      "logits/rejected": 0.82421875,
      "logps/chosen": -368.0,
      "logps/rejected": -394.0,
      "loss": 0.119,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -11.5,
      "rewards/margins": 7.3125,
      "rewards/rejected": -18.875,
      "step": 4740
    },
    {
      "epoch": 0.6070675442520289,
      "grad_norm": 7.312385241792215,
      "learning_rate": 1.1607620001760185e-06,
      "logits/chosen": 0.275390625,
      "logits/rejected": 0.82421875,
      "logps/chosen": -382.0,
      "logps/rejected": -412.0,
      "loss": 0.1501,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -13.25,
      "rewards/margins": 6.9375,
      "rewards/rejected": -20.25,
      "step": 4750
    },
    {
      "epoch": 0.6083455811872963,
      "grad_norm": 7.743799973398028,
      "learning_rate": 1.1595420713048968e-06,
      "logits/chosen": 0.1416015625,
      "logits/rejected": 0.71484375,
      "logps/chosen": -410.0,
      "logps/rejected": -420.0,
      "loss": 0.1614,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -13.125,
      "rewards/margins": 7.21875,
      "rewards/rejected": -20.375,
      "step": 4760
    },
    {
      "epoch": 0.6096236181225637,
      "grad_norm": 7.325219280124243,
      "learning_rate": 1.1583259807015182e-06,
      "logits/chosen": 0.1630859375,
      "logits/rejected": 0.81640625,
      "logps/chosen": -392.0,
      "logps/rejected": -420.0,
      "loss": 0.1349,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -12.4375,
      "rewards/margins": 7.3125,
      "rewards/rejected": -19.75,
      "step": 4770
    },
    {
      "epoch": 0.6109016550578311,
      "grad_norm": 7.7535383799107995,
      "learning_rate": 1.1571137082807434e-06,
      "logits/chosen": 0.17578125,
      "logits/rejected": 0.73828125,
      "logps/chosen": -404.0,
      "logps/rejected": -418.0,
      "loss": 0.1257,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -13.1875,
      "rewards/margins": 6.8125,
      "rewards/rejected": -20.0,
      "step": 4780
    },
    {
      "epoch": 0.6121796919930986,
      "grad_norm": 8.54519762117311,
      "learning_rate": 1.155905234104269e-06,
      "logits/chosen": 0.1865234375,
      "logits/rejected": 0.88671875,
      "logps/chosen": -402.0,
      "logps/rejected": -444.0,
      "loss": 0.1497,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -13.3125,
      "rewards/margins": 7.59375,
      "rewards/rejected": -21.0,
      "step": 4790
    },
    {
      "epoch": 0.613457728928366,
      "grad_norm": 11.335967540732407,
      "learning_rate": 1.1547005383792516e-06,
      "logits/chosen": 0.275390625,
      "logits/rejected": 0.796875,
      "logps/chosen": -404.0,
      "logps/rejected": -442.0,
      "loss": 0.1212,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.625,
      "rewards/margins": 7.0,
      "rewards/rejected": -20.625,
      "step": 4800
    },
    {
      "epoch": 0.6147357658636334,
      "grad_norm": 11.487848209372023,
      "learning_rate": 1.1534996014569446e-06,
      "logits/chosen": 0.298828125,
      "logits/rejected": 0.8359375,
      "logps/chosen": -386.0,
      "logps/rejected": -420.0,
      "loss": 0.1603,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -12.9375,
      "rewards/margins": 6.6875,
      "rewards/rejected": -19.625,
      "step": 4810
    },
    {
      "epoch": 0.6160138027989008,
      "grad_norm": 8.00076553294362,
      "learning_rate": 1.1523024038313547e-06,
      "logits/chosen": 0.203125,
      "logits/rejected": 0.77734375,
      "logps/chosen": -386.0,
      "logps/rejected": -414.0,
      "loss": 0.1191,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -11.5625,
      "rewards/margins": 7.8125,
      "rewards/rejected": -19.375,
      "step": 4820
    },
    {
      "epoch": 0.6172918397341683,
      "grad_norm": 8.117573299890937,
      "learning_rate": 1.1511089261379083e-06,
      "logits/chosen": 0.0849609375,
      "logits/rejected": 0.5546875,
      "logps/chosen": -388.0,
      "logps/rejected": -422.0,
      "loss": 0.1405,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -10.8125,
      "rewards/margins": 7.6875,
      "rewards/rejected": -18.5,
      "step": 4830
    },
    {
      "epoch": 0.6185698766694357,
      "grad_norm": 8.233599119558857,
      "learning_rate": 1.149919149152138e-06,
      "logits/chosen": 0.061279296875,
      "logits/rejected": 0.57421875,
      "logps/chosen": -396.0,
      "logps/rejected": -426.0,
      "loss": 0.1536,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -11.3125,
      "rewards/margins": 6.6875,
      "rewards/rejected": -18.0,
      "step": 4840
    },
    {
      "epoch": 0.6198479136047031,
      "grad_norm": 7.041889712379552,
      "learning_rate": 1.148733053788381e-06,
      "logits/chosen": 0.1123046875,
      "logits/rejected": 0.8515625,
      "logps/chosen": -388.0,
      "logps/rejected": -414.0,
      "loss": 0.1126,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -12.0625,
      "rewards/margins": 7.28125,
      "rewards/rejected": -19.375,
      "step": 4850
    },
    {
      "epoch": 0.6211259505399706,
      "grad_norm": 8.119011829091646,
      "learning_rate": 1.1475506210984938e-06,
      "logits/chosen": 0.0791015625,
      "logits/rejected": 0.8828125,
      "logps/chosen": -384.0,
      "logps/rejected": -414.0,
      "loss": 0.1535,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -12.1875,
      "rewards/margins": 7.03125,
      "rewards/rejected": -19.25,
      "step": 4860
    },
    {
      "epoch": 0.6224039874752381,
      "grad_norm": 9.59536215232007,
      "learning_rate": 1.1463718322705807e-06,
      "logits/chosen": 0.076171875,
      "logits/rejected": 0.6953125,
      "logps/chosen": -392.0,
      "logps/rejected": -432.0,
      "loss": 0.1106,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -12.125,
      "rewards/margins": 7.3125,
      "rewards/rejected": -19.375,
      "step": 4870
    },
    {
      "epoch": 0.6236820244105055,
      "grad_norm": 6.979165359349176,
      "learning_rate": 1.1451966686277364e-06,
      "logits/chosen": 0.1845703125,
      "logits/rejected": 0.69140625,
      "logps/chosen": -396.0,
      "logps/rejected": -440.0,
      "loss": 0.1419,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -13.375,
      "rewards/margins": 6.59375,
      "rewards/rejected": -20.0,
      "step": 4880
    },
    {
      "epoch": 0.6249600613457729,
      "grad_norm": 7.798746700420364,
      "learning_rate": 1.1440251116268034e-06,
      "logits/chosen": 0.11865234375,
      "logits/rejected": 0.734375,
      "logps/chosen": -436.0,
      "logps/rejected": -478.0,
      "loss": 0.1513,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.125,
      "rewards/margins": 7.84375,
      "rewards/rejected": -23.0,
      "step": 4890
    },
    {
      "epoch": 0.6262380982810403,
      "grad_norm": 7.697464002435526,
      "learning_rate": 1.1428571428571428e-06,
      "logits/chosen": 0.2001953125,
      "logits/rejected": 0.87109375,
      "logps/chosen": -444.0,
      "logps/rejected": -476.0,
      "loss": 0.1094,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -17.0,
      "rewards/margins": 7.375,
      "rewards/rejected": -24.375,
      "step": 4900
    },
    {
      "epoch": 0.6275161352163078,
      "grad_norm": 5.980262285746097,
      "learning_rate": 1.14169274403942e-06,
      "logits/chosen": 0.2353515625,
      "logits/rejected": 0.8671875,
      "logps/chosen": -402.0,
      "logps/rejected": -448.0,
      "loss": 0.1233,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.75,
      "rewards/margins": 7.15625,
      "rewards/rejected": -21.875,
      "step": 4910
    },
    {
      "epoch": 0.6287941721515752,
      "grad_norm": 8.241062160872398,
      "learning_rate": 1.140531897024402e-06,
      "logits/chosen": 0.1845703125,
      "logits/rejected": 0.75,
      "logps/chosen": -410.0,
      "logps/rejected": -450.0,
      "loss": 0.1494,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.3125,
      "rewards/margins": 7.15625,
      "rewards/rejected": -21.5,
      "step": 4920
    },
    {
      "epoch": 0.6300722090868426,
      "grad_norm": 6.053525549434371,
      "learning_rate": 1.13937458379177e-06,
      "logits/chosen": 0.337890625,
      "logits/rejected": 0.78515625,
      "logps/chosen": -408.0,
      "logps/rejected": -440.0,
      "loss": 0.1424,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.3125,
      "rewards/margins": 7.25,
      "rewards/rejected": -20.5,
      "step": 4930
    },
    {
      "epoch": 0.63135024602211,
      "grad_norm": 8.28289164202277,
      "learning_rate": 1.1382207864489444e-06,
      "logits/chosen": 0.29296875,
      "logits/rejected": 0.83203125,
      "logps/chosen": -420.0,
      "logps/rejected": -430.0,
      "loss": 0.1447,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -13.75,
      "rewards/margins": 6.59375,
      "rewards/rejected": -20.375,
      "step": 4940
    },
    {
      "epoch": 0.6326282829573775,
      "grad_norm": 10.883601747138918,
      "learning_rate": 1.1370704872299223e-06,
      "logits/chosen": 0.271484375,
      "logits/rejected": 0.74609375,
      "logps/chosen": -400.0,
      "logps/rejected": -430.0,
      "loss": 0.145,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -13.25,
      "rewards/margins": 6.3125,
      "rewards/rejected": -19.5,
      "step": 4950
    },
    {
      "epoch": 0.6339063198926449,
      "grad_norm": 6.100474807910865,
      "learning_rate": 1.1359236684941295e-06,
      "logits/chosen": 0.36328125,
      "logits/rejected": 0.83984375,
      "logps/chosen": -422.0,
      "logps/rejected": -444.0,
      "loss": 0.1381,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.75,
      "rewards/margins": 6.5,
      "rewards/rejected": -21.25,
      "step": 4960
    },
    {
      "epoch": 0.6351843568279123,
      "grad_norm": 5.400888119792498,
      "learning_rate": 1.1347803127252839e-06,
      "logits/chosen": 0.2099609375,
      "logits/rejected": 0.71875,
      "logps/chosen": -434.0,
      "logps/rejected": -460.0,
      "loss": 0.1098,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.1875,
      "rewards/margins": 6.84375,
      "rewards/rejected": -22.0,
      "step": 4970
    },
    {
      "epoch": 0.6364623937631797,
      "grad_norm": 12.689682999460608,
      "learning_rate": 1.1336404025302715e-06,
      "logits/chosen": 0.33984375,
      "logits/rejected": 0.82421875,
      "logps/chosen": -422.0,
      "logps/rejected": -456.0,
      "loss": 0.154,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.1875,
      "rewards/margins": 6.9375,
      "rewards/rejected": -22.125,
      "step": 4980
    },
    {
      "epoch": 0.6377404306984472,
      "grad_norm": 11.276981625328313,
      "learning_rate": 1.1325039206380352e-06,
      "logits/chosen": 0.169921875,
      "logits/rejected": 0.7578125,
      "logps/chosen": -430.0,
      "logps/rejected": -458.0,
      "loss": 0.1704,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -15.5625,
      "rewards/margins": 6.5625,
      "rewards/rejected": -22.125,
      "step": 4990
    },
    {
      "epoch": 0.6390184676337146,
      "grad_norm": 8.951759365877086,
      "learning_rate": 1.131370849898476e-06,
      "logits/chosen": 0.166015625,
      "logits/rejected": 0.71484375,
      "logps/chosen": -438.0,
      "logps/rejected": -464.0,
      "loss": 0.1232,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -15.3125,
      "rewards/margins": 7.8125,
      "rewards/rejected": -23.125,
      "step": 5000
    },
    {
      "epoch": 0.6390184676337146,
      "eval_logits/chosen": 0.208984375,
      "eval_logits/rejected": 0.8125,
      "eval_logps/chosen": -416.0,
      "eval_logps/rejected": -438.0,
      "eval_loss": 0.2576223611831665,
      "eval_rewards/accuracies": 0.8927131295204163,
      "eval_rewards/chosen": -15.0,
      "eval_rewards/margins": 5.84375,
      "eval_rewards/rejected": -20.875,
      "eval_runtime": 1502.2843,
      "eval_samples_per_second": 36.948,
      "eval_steps_per_second": 0.578,
      "step": 5000
    },
    {
      "epoch": 0.640296504568982,
      "grad_norm": 9.670104419348979,
      "learning_rate": 1.130241173281366e-06,
      "logits/chosen": 0.091796875,
      "logits/rejected": 0.73828125,
      "logps/chosen": -414.0,
      "logps/rejected": -456.0,
      "loss": 0.1259,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.75,
      "rewards/margins": 7.09375,
      "rewards/rejected": -21.75,
      "step": 5010
    },
    {
      "epoch": 0.6415745415042494,
      "grad_norm": 6.305557043370455,
      "learning_rate": 1.1291148738752732e-06,
      "logits/chosen": 0.310546875,
      "logits/rejected": 0.9140625,
      "logps/chosen": -394.0,
      "logps/rejected": -428.0,
      "loss": 0.1201,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.3125,
      "rewards/margins": 6.46875,
      "rewards/rejected": -19.75,
      "step": 5020
    },
    {
      "epoch": 0.6428525784395169,
      "grad_norm": 6.963874712097572,
      "learning_rate": 1.1279919348864981e-06,
      "logits/chosen": 0.208984375,
      "logits/rejected": 0.6875,
      "logps/chosen": -378.0,
      "logps/rejected": -436.0,
      "loss": 0.1186,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -11.4375,
      "rewards/margins": 7.625,
      "rewards/rejected": -19.125,
      "step": 5030
    },
    {
      "epoch": 0.6441306153747843,
      "grad_norm": 8.126517443710192,
      "learning_rate": 1.126872339638022e-06,
      "logits/chosen": 0.189453125,
      "logits/rejected": 0.796875,
      "logps/chosen": -384.0,
      "logps/rejected": -428.0,
      "loss": 0.1219,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -12.0,
      "rewards/margins": 7.03125,
      "rewards/rejected": -19.0,
      "step": 5040
    },
    {
      "epoch": 0.6454086523100517,
      "grad_norm": 10.255140839393896,
      "learning_rate": 1.1257560715684668e-06,
      "logits/chosen": 0.220703125,
      "logits/rejected": 0.87890625,
      "logps/chosen": -400.0,
      "logps/rejected": -442.0,
      "loss": 0.1228,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.125,
      "rewards/margins": 6.78125,
      "rewards/rejected": -20.875,
      "step": 5050
    },
    {
      "epoch": 0.6466866892453191,
      "grad_norm": 7.8172433143335995,
      "learning_rate": 1.1246431142310665e-06,
      "logits/chosen": 0.146484375,
      "logits/rejected": 0.734375,
      "logps/chosen": -416.0,
      "logps/rejected": -436.0,
      "loss": 0.1581,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -13.4375,
      "rewards/margins": 7.4375,
      "rewards/rejected": -20.875,
      "step": 5060
    },
    {
      "epoch": 0.6479647261805866,
      "grad_norm": 11.941199187831488,
      "learning_rate": 1.1235334512926484e-06,
      "logits/chosen": 0.10302734375,
      "logits/rejected": 0.9296875,
      "logps/chosen": -406.0,
      "logps/rejected": -410.0,
      "loss": 0.1487,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -13.25,
      "rewards/margins": 6.78125,
      "rewards/rejected": -20.125,
      "step": 5070
    },
    {
      "epoch": 0.649242763115854,
      "grad_norm": 7.591146771100241,
      "learning_rate": 1.1224270665326274e-06,
      "logits/chosen": 0.08544921875,
      "logits/rejected": 0.79296875,
      "logps/chosen": -428.0,
      "logps/rejected": -442.0,
      "loss": 0.1299,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.4375,
      "rewards/margins": 7.375,
      "rewards/rejected": -21.875,
      "step": 5080
    },
    {
      "epoch": 0.6505208000511215,
      "grad_norm": 10.231248928125577,
      "learning_rate": 1.12132394384201e-06,
      "logits/chosen": 0.07763671875,
      "logits/rejected": 0.85546875,
      "logps/chosen": -430.0,
      "logps/rejected": -446.0,
      "loss": 0.1425,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -14.3125,
      "rewards/margins": 7.5625,
      "rewards/rejected": -21.875,
      "step": 5090
    },
    {
      "epoch": 0.6517988369863889,
      "grad_norm": 7.354279655256986,
      "learning_rate": 1.1202240672224076e-06,
      "logits/chosen": 0.2060546875,
      "logits/rejected": 0.72265625,
      "logps/chosen": -408.0,
      "logps/rejected": -458.0,
      "loss": 0.1094,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -13.9375,
      "rewards/margins": 7.40625,
      "rewards/rejected": -21.375,
      "step": 5100
    },
    {
      "epoch": 0.6530768739216564,
      "grad_norm": 11.672088225480412,
      "learning_rate": 1.1191274207850654e-06,
      "logits/chosen": 0.375,
      "logits/rejected": 0.8515625,
      "logps/chosen": -402.0,
      "logps/rejected": -440.0,
      "loss": 0.1385,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -13.625,
      "rewards/margins": 6.625,
      "rewards/rejected": -20.25,
      "step": 5110
    },
    {
      "epoch": 0.6543549108569238,
      "grad_norm": 7.566388435092754,
      "learning_rate": 1.1180339887498948e-06,
      "logits/chosen": 0.2001953125,
      "logits/rejected": 0.7890625,
      "logps/chosen": -418.0,
      "logps/rejected": -450.0,
      "loss": 0.1183,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -15.6875,
      "rewards/margins": 6.34375,
      "rewards/rejected": -22.0,
      "step": 5120
    },
    {
      "epoch": 0.6556329477921912,
      "grad_norm": 7.967890433964789,
      "learning_rate": 1.1169437554445213e-06,
      "logits/chosen": 0.1748046875,
      "logits/rejected": 0.78515625,
      "logps/chosen": -410.0,
      "logps/rejected": -452.0,
      "loss": 0.144,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.4375,
      "rewards/margins": 7.15625,
      "rewards/rejected": -20.625,
      "step": 5130
    },
    {
      "epoch": 0.6569109847274586,
      "grad_norm": 11.433657913180994,
      "learning_rate": 1.1158567053033413e-06,
      "logits/chosen": 0.1318359375,
      "logits/rejected": 0.68359375,
      "logps/chosen": -410.0,
      "logps/rejected": -438.0,
      "loss": 0.161,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -13.625,
      "rewards/margins": 6.375,
      "rewards/rejected": -20.0,
      "step": 5140
    },
    {
      "epoch": 0.6581890216627261,
      "grad_norm": 9.810608152652085,
      "learning_rate": 1.1147728228665882e-06,
      "logits/chosen": 0.23046875,
      "logits/rejected": 0.76953125,
      "logps/chosen": -402.0,
      "logps/rejected": -418.0,
      "loss": 0.1325,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.0625,
      "rewards/margins": 6.40625,
      "rewards/rejected": -20.5,
      "step": 5150
    },
    {
      "epoch": 0.6594670585979935,
      "grad_norm": 9.04078316656911,
      "learning_rate": 1.1136920927794092e-06,
      "logits/chosen": 0.2021484375,
      "logits/rejected": 0.796875,
      "logps/chosen": -384.0,
      "logps/rejected": -424.0,
      "loss": 0.1293,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -12.75,
      "rewards/margins": 7.125,
      "rewards/rejected": -19.875,
      "step": 5160
    },
    {
      "epoch": 0.6607450955332609,
      "grad_norm": 9.110126396293472,
      "learning_rate": 1.1126144997909508e-06,
      "logits/chosen": 0.28515625,
      "logits/rejected": 0.61328125,
      "logps/chosen": -394.0,
      "logps/rejected": -418.0,
      "loss": 0.1459,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -12.375,
      "rewards/margins": 6.9375,
      "rewards/rejected": -19.25,
      "step": 5170
    },
    {
      "epoch": 0.6620231324685283,
      "grad_norm": 6.848398854087755,
      "learning_rate": 1.1115400287534568e-06,
      "logits/chosen": 0.248046875,
      "logits/rejected": 0.7734375,
      "logps/chosen": -392.0,
      "logps/rejected": -440.0,
      "loss": 0.1433,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -13.0625,
      "rewards/margins": 7.0625,
      "rewards/rejected": -20.125,
      "step": 5180
    },
    {
      "epoch": 0.6633011694037958,
      "grad_norm": 9.112870826703208,
      "learning_rate": 1.110468664621372e-06,
      "logits/chosen": 0.2216796875,
      "logits/rejected": 0.67578125,
      "logps/chosen": -392.0,
      "logps/rejected": -440.0,
      "loss": 0.1554,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -13.0625,
      "rewards/margins": 7.3125,
      "rewards/rejected": -20.375,
      "step": 5190
    },
    {
      "epoch": 0.6645792063390632,
      "grad_norm": 10.646875581486976,
      "learning_rate": 1.1094003924504583e-06,
      "logits/chosen": 0.072265625,
      "logits/rejected": 0.76171875,
      "logps/chosen": -378.0,
      "logps/rejected": -422.0,
      "loss": 0.1155,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -11.5,
      "rewards/margins": 7.78125,
      "rewards/rejected": -19.25,
      "step": 5200
    },
    {
      "epoch": 0.6658572432743306,
      "grad_norm": 10.454769509738554,
      "learning_rate": 1.1083351973969191e-06,
      "logits/chosen": 0.06396484375,
      "logits/rejected": 0.703125,
      "logps/chosen": -414.0,
      "logps/rejected": -452.0,
      "loss": 0.1795,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -13.625,
      "rewards/margins": 7.375,
      "rewards/rejected": -21.0,
      "step": 5210
    },
    {
      "epoch": 0.667135280209598,
      "grad_norm": 7.8754969344408465,
      "learning_rate": 1.107273064716533e-06,
      "logits/chosen": 0.232421875,
      "logits/rejected": 0.96484375,
      "logps/chosen": -404.0,
      "logps/rejected": -442.0,
      "loss": 0.1291,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.8125,
      "rewards/margins": 7.15625,
      "rewards/rejected": -22.0,
      "step": 5220
    },
    {
      "epoch": 0.6684133171448655,
      "grad_norm": 6.595912709333146,
      "learning_rate": 1.1062139797637962e-06,
      "logits/chosen": 0.197265625,
      "logits/rejected": 0.76953125,
      "logps/chosen": -438.0,
      "logps/rejected": -472.0,
      "loss": 0.1023,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.5,
      "rewards/margins": 7.59375,
      "rewards/rejected": -23.125,
      "step": 5230
    },
    {
      "epoch": 0.6696913540801329,
      "grad_norm": 11.052501039867334,
      "learning_rate": 1.1051579279910751e-06,
      "logits/chosen": 0.376953125,
      "logits/rejected": 1.078125,
      "logps/chosen": -428.0,
      "logps/rejected": -472.0,
      "loss": 0.1632,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -16.75,
      "rewards/margins": 6.71875,
      "rewards/rejected": -23.375,
      "step": 5240
    },
    {
      "epoch": 0.6709693910154003,
      "grad_norm": 7.321427886946239,
      "learning_rate": 1.1041048949477667e-06,
      "logits/chosen": 0.29296875,
      "logits/rejected": 0.98828125,
      "logps/chosen": -414.0,
      "logps/rejected": -460.0,
      "loss": 0.12,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.6875,
      "rewards/margins": 6.71875,
      "rewards/rejected": -22.375,
      "step": 5250
    },
    {
      "epoch": 0.6722474279506677,
      "grad_norm": 3.6976392504744098,
      "learning_rate": 1.1030548662794673e-06,
      "logits/chosen": 0.169921875,
      "logits/rejected": 0.65234375,
      "logps/chosen": -452.0,
      "logps/rejected": -482.0,
      "loss": 0.1075,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.4375,
      "rewards/margins": 7.0625,
      "rewards/rejected": -22.5,
      "step": 5260
    },
    {
      "epoch": 0.6735254648859352,
      "grad_norm": 11.072284365274308,
      "learning_rate": 1.102007827727152e-06,
      "logits/chosen": 0.07373046875,
      "logits/rejected": 0.7578125,
      "logps/chosen": -396.0,
      "logps/rejected": -432.0,
      "loss": 0.1346,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -12.9375,
      "rewards/margins": 6.90625,
      "rewards/rejected": -19.875,
      "step": 5270
    },
    {
      "epoch": 0.6748035018212026,
      "grad_norm": 5.934172613246176,
      "learning_rate": 1.1009637651263607e-06,
      "logits/chosen": 0.1875,
      "logits/rejected": 0.89453125,
      "logps/chosen": -410.0,
      "logps/rejected": -424.0,
      "loss": 0.1838,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -12.9375,
      "rewards/margins": 7.4375,
      "rewards/rejected": -20.375,
      "step": 5280
    },
    {
      "epoch": 0.67608153875647,
      "grad_norm": 7.340788658346921,
      "learning_rate": 1.0999226644063927e-06,
      "logits/chosen": 0.052978515625,
      "logits/rejected": 0.6640625,
      "logps/chosen": -420.0,
      "logps/rejected": -446.0,
      "loss": 0.1165,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.1875,
      "rewards/margins": 7.34375,
      "rewards/rejected": -21.5,
      "step": 5290
    },
    {
      "epoch": 0.6773595756917374,
      "grad_norm": 7.489840443894045,
      "learning_rate": 1.0988845115895123e-06,
      "logits/chosen": 0.26953125,
      "logits/rejected": 0.7890625,
      "logps/chosen": -410.0,
      "logps/rejected": -452.0,
      "loss": 0.1292,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.0625,
      "rewards/margins": 6.84375,
      "rewards/rejected": -21.875,
      "step": 5300
    },
    {
      "epoch": 0.678637612627005,
      "grad_norm": 8.19221082767327,
      "learning_rate": 1.0978492927901574e-06,
      "logits/chosen": 0.115234375,
      "logits/rejected": 0.8984375,
      "logps/chosen": -418.0,
      "logps/rejected": -444.0,
      "loss": 0.1009,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.375,
      "rewards/margins": 7.34375,
      "rewards/rejected": -21.75,
      "step": 5310
    },
    {
      "epoch": 0.6799156495622724,
      "grad_norm": 8.635359474581934,
      "learning_rate": 1.0968169942141634e-06,
      "logits/chosen": 0.1298828125,
      "logits/rejected": 0.8046875,
      "logps/chosen": -418.0,
      "logps/rejected": -448.0,
      "loss": 0.1373,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.1875,
      "rewards/margins": 7.375,
      "rewards/rejected": -21.625,
      "step": 5320
    },
    {
      "epoch": 0.6811936864975398,
      "grad_norm": 10.632158538747518,
      "learning_rate": 1.0957876021579874e-06,
      "logits/chosen": 0.1318359375,
      "logits/rejected": 0.7578125,
      "logps/chosen": -414.0,
      "logps/rejected": -430.0,
      "loss": 0.1314,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.0625,
      "rewards/margins": 7.75,
      "rewards/rejected": -21.875,
      "step": 5330
    },
    {
      "epoch": 0.6824717234328072,
      "grad_norm": 8.01901640232957,
      "learning_rate": 1.0947611030079466e-06,
      "logits/chosen": 0.08740234375,
      "logits/rejected": 0.62109375,
      "logps/chosen": -400.0,
      "logps/rejected": -428.0,
      "loss": 0.1248,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -12.875,
      "rewards/margins": 6.875,
      "rewards/rejected": -19.75,
      "step": 5340
    },
    {
      "epoch": 0.6837497603680747,
      "grad_norm": 7.253188901079145,
      "learning_rate": 1.0937374832394612e-06,
      "logits/chosen": 0.1162109375,
      "logits/rejected": 0.6328125,
      "logps/chosen": -356.0,
      "logps/rejected": -404.0,
      "loss": 0.1187,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -10.5,
      "rewards/margins": 7.0,
      "rewards/rejected": -17.5,
      "step": 5350
    },
    {
      "epoch": 0.6850277973033421,
      "grad_norm": 9.321332499043576,
      "learning_rate": 1.092716729416306e-06,
      "logits/chosen": -0.0888671875,
      "logits/rejected": 0.5625,
      "logps/chosen": -412.0,
      "logps/rejected": -442.0,
      "loss": 0.1447,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -11.5,
      "rewards/margins": 7.5625,
      "rewards/rejected": -19.0,
      "step": 5360
    },
    {
      "epoch": 0.6863058342386095,
      "grad_norm": 8.0113019498129,
      "learning_rate": 1.0916988281898703e-06,
      "logits/chosen": -0.0390625,
      "logits/rejected": 0.53125,
      "logps/chosen": -372.0,
      "logps/rejected": -418.0,
      "loss": 0.1441,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -10.875,
      "rewards/margins": 6.625,
      "rewards/rejected": -17.5,
      "step": 5370
    },
    {
      "epoch": 0.6875838711738769,
      "grad_norm": 6.257113195152115,
      "learning_rate": 1.0906837662984237e-06,
      "logits/chosen": 0.0830078125,
      "logits/rejected": 0.734375,
      "logps/chosen": -376.0,
      "logps/rejected": -418.0,
      "loss": 0.1167,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -11.1875,
      "rewards/margins": 7.0,
      "rewards/rejected": -18.25,
      "step": 5380
    },
    {
      "epoch": 0.6888619081091444,
      "grad_norm": 7.796149264685869,
      "learning_rate": 1.089671530566391e-06,
      "logits/chosen": 0.0301513671875,
      "logits/rejected": 0.5078125,
      "logps/chosen": -400.0,
      "logps/rejected": -448.0,
      "loss": 0.1458,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -12.8125,
      "rewards/margins": 6.78125,
      "rewards/rejected": -19.625,
      "step": 5390
    },
    {
      "epoch": 0.6901399450444118,
      "grad_norm": 7.653986249922465,
      "learning_rate": 1.0886621079036346e-06,
      "logits/chosen": -0.015380859375,
      "logits/rejected": 0.70703125,
      "logps/chosen": -432.0,
      "logps/rejected": -444.0,
      "loss": 0.1096,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -13.875,
      "rewards/margins": 8.125,
      "rewards/rejected": -22.0,
      "step": 5400
    },
    {
      "epoch": 0.6914179819796792,
      "grad_norm": 5.82276196261449,
      "learning_rate": 1.0876554853047417e-06,
      "logits/chosen": 0.142578125,
      "logits/rejected": 0.9140625,
      "logps/chosen": -438.0,
      "logps/rejected": -454.0,
      "loss": 0.112,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.6875,
      "rewards/margins": 7.71875,
      "rewards/rejected": -23.375,
      "step": 5410
    },
    {
      "epoch": 0.6926960189149466,
      "grad_norm": 2.855658966650057,
      "learning_rate": 1.0866516498483225e-06,
      "logits/chosen": 0.1494140625,
      "logits/rejected": 0.80859375,
      "logps/chosen": -434.0,
      "logps/rejected": -454.0,
      "loss": 0.1183,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.03125,
      "rewards/rejected": -23.25,
      "step": 5420
    },
    {
      "epoch": 0.6939740558502141,
      "grad_norm": 6.059547675563375,
      "learning_rate": 1.0856505886963116e-06,
      "logits/chosen": 0.1533203125,
      "logits/rejected": 0.7734375,
      "logps/chosen": -428.0,
      "logps/rejected": -456.0,
      "loss": 0.1268,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -15.75,
      "rewards/margins": 7.625,
      "rewards/rejected": -23.375,
      "step": 5430
    },
    {
      "epoch": 0.6952520927854815,
      "grad_norm": 9.905936251000636,
      "learning_rate": 1.0846522890932808e-06,
      "logits/chosen": 0.043701171875,
      "logits/rejected": 0.5859375,
      "logps/chosen": -452.0,
      "logps/rejected": -496.0,
      "loss": 0.1505,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.25,
      "rewards/margins": 7.9375,
      "rewards/rejected": -25.25,
      "step": 5440
    },
    {
      "epoch": 0.6965301297207489,
      "grad_norm": 5.112335264807164,
      "learning_rate": 1.0836567383657542e-06,
      "logits/chosen": 0.205078125,
      "logits/rejected": 0.91015625,
      "logps/chosen": -416.0,
      "logps/rejected": -468.0,
      "loss": 0.1101,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.875,
      "rewards/margins": 7.71875,
      "rewards/rejected": -23.625,
      "step": 5450
    },
    {
      "epoch": 0.6978081666560163,
      "grad_norm": 5.199925567178265,
      "learning_rate": 1.0826639239215334e-06,
      "logits/chosen": 0.1533203125,
      "logits/rejected": 0.8046875,
      "logps/chosen": -418.0,
      "logps/rejected": -454.0,
      "loss": 0.1196,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.8125,
      "rewards/margins": 8.0,
      "rewards/rejected": -23.75,
      "step": 5460
    },
    {
      "epoch": 0.6990862035912838,
      "grad_norm": 6.103447175076816,
      "learning_rate": 1.0816738332490292e-06,
      "logits/chosen": 0.09765625,
      "logits/rejected": 0.7265625,
      "logps/chosen": -406.0,
      "logps/rejected": -440.0,
      "loss": 0.1191,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.5,
      "rewards/margins": 7.65625,
      "rewards/rejected": -22.125,
      "step": 5470
    },
    {
      "epoch": 0.7003642405265512,
      "grad_norm": 16.197809856905913,
      "learning_rate": 1.0806864539165982e-06,
      "logits/chosen": 0.1943359375,
      "logits/rejected": 0.8828125,
      "logps/chosen": -438.0,
      "logps/rejected": -476.0,
      "loss": 0.1493,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -16.375,
      "rewards/margins": 8.25,
      "rewards/rejected": -24.5,
      "step": 5480
    },
    {
      "epoch": 0.7016422774618186,
      "grad_norm": 10.824037859412686,
      "learning_rate": 1.0797017735718878e-06,
      "logits/chosen": 0.248046875,
      "logits/rejected": 0.8984375,
      "logps/chosen": -430.0,
      "logps/rejected": -454.0,
      "loss": 0.1351,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.21875,
      "rewards/rejected": -23.625,
      "step": 5490
    },
    {
      "epoch": 0.702920314397086,
      "grad_norm": 11.131007075631105,
      "learning_rate": 1.0787197799411874e-06,
      "logits/chosen": 0.1396484375,
      "logits/rejected": 0.72265625,
      "logps/chosen": -430.0,
      "logps/rejected": -464.0,
      "loss": 0.1354,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -23.125,
      "step": 5500
    },
    {
      "epoch": 0.7041983513323535,
      "grad_norm": 5.241194526711526,
      "learning_rate": 1.0777404608287846e-06,
      "logits/chosen": 0.259765625,
      "logits/rejected": 0.9296875,
      "logps/chosen": -406.0,
      "logps/rejected": -452.0,
      "loss": 0.1112,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.15625,
      "rewards/rejected": -23.5,
      "step": 5510
    },
    {
      "epoch": 0.7054763882676209,
      "grad_norm": 9.589649743622418,
      "learning_rate": 1.0767638041163309e-06,
      "logits/chosen": 0.130859375,
      "logits/rejected": 0.84375,
      "logps/chosen": -436.0,
      "logps/rejected": -462.0,
      "loss": 0.1285,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.84375,
      "rewards/rejected": -24.125,
      "step": 5520
    },
    {
      "epoch": 0.7067544252028883,
      "grad_norm": 4.97093628344615,
      "learning_rate": 1.0757897977622107e-06,
      "logits/chosen": 0.08154296875,
      "logits/rejected": 0.55859375,
      "logps/chosen": -416.0,
      "logps/rejected": -468.0,
      "loss": 0.1152,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.125,
      "rewards/margins": 6.84375,
      "rewards/rejected": -21.0,
      "step": 5530
    },
    {
      "epoch": 0.7080324621381557,
      "grad_norm": 6.872453141024929,
      "learning_rate": 1.074818429800918e-06,
      "logits/chosen": 0.0673828125,
      "logits/rejected": 0.51171875,
      "logps/chosen": -376.0,
      "logps/rejected": -420.0,
      "loss": 0.1155,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -11.875,
      "rewards/margins": 6.78125,
      "rewards/rejected": -18.625,
      "step": 5540
    },
    {
      "epoch": 0.7093104990734233,
      "grad_norm": 9.708223024856164,
      "learning_rate": 1.073849688342439e-06,
      "logits/chosen": -0.005767822265625,
      "logits/rejected": 0.77734375,
      "logps/chosen": -390.0,
      "logps/rejected": -408.0,
      "loss": 0.1297,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -12.25,
      "rewards/margins": 7.125,
      "rewards/rejected": -19.375,
      "step": 5550
    },
    {
      "epoch": 0.7105885360086907,
      "grad_norm": 6.23332437969369,
      "learning_rate": 1.07288356157164e-06,
      "logits/chosen": 0.10986328125,
      "logits/rejected": 0.52734375,
      "logps/chosen": -422.0,
      "logps/rejected": -444.0,
      "loss": 0.1058,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.0,
      "rewards/margins": 7.03125,
      "rewards/rejected": -21.0,
      "step": 5560
    },
    {
      "epoch": 0.711866572943958,
      "grad_norm": 6.4560031243369265,
      "learning_rate": 1.0719200377476648e-06,
      "logits/chosen": 0.1259765625,
      "logits/rejected": 0.55859375,
      "logps/chosen": -422.0,
      "logps/rejected": -444.0,
      "loss": 0.147,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.0625,
      "rewards/margins": 7.53125,
      "rewards/rejected": -21.625,
      "step": 5570
    },
    {
      "epoch": 0.7131446098792255,
      "grad_norm": 8.839133829713886,
      "learning_rate": 1.0709591052033317e-06,
      "logits/chosen": 0.09033203125,
      "logits/rejected": 0.55078125,
      "logps/chosen": -388.0,
      "logps/rejected": -458.0,
      "loss": 0.1119,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.75,
      "rewards/margins": 7.0625,
      "rewards/rejected": -20.75,
      "step": 5580
    },
    {
      "epoch": 0.714422646814493,
      "grad_norm": 16.172781655739236,
      "learning_rate": 1.0700007523445435e-06,
      "logits/chosen": 0.09130859375,
      "logits/rejected": 0.78515625,
      "logps/chosen": -446.0,
      "logps/rejected": -466.0,
      "loss": 0.1088,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.0,
      "rewards/margins": 7.65625,
      "rewards/rejected": -23.625,
      "step": 5590
    },
    {
      "epoch": 0.7157006837497604,
      "grad_norm": 9.557336904355944,
      "learning_rate": 1.0690449676496976e-06,
      "logits/chosen": 0.150390625,
      "logits/rejected": 0.75390625,
      "logps/chosen": -436.0,
      "logps/rejected": -476.0,
      "loss": 0.1144,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.25,
      "rewards/margins": 8.25,
      "rewards/rejected": -24.5,
      "step": 5600
    },
    {
      "epoch": 0.7169787206850278,
      "grad_norm": 6.517106173977129,
      "learning_rate": 1.0680917396691054e-06,
      "logits/chosen": 0.05029296875,
      "logits/rejected": 0.515625,
      "logps/chosen": -444.0,
      "logps/rejected": -496.0,
      "loss": 0.1302,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.8125,
      "rewards/margins": 7.875,
      "rewards/rejected": -23.75,
      "step": 5610
    },
    {
      "epoch": 0.7182567576202952,
      "grad_norm": 6.533004292408844,
      "learning_rate": 1.0671410570244164e-06,
      "logits/chosen": 0.052734375,
      "logits/rejected": 0.64453125,
      "logps/chosen": -408.0,
      "logps/rejected": -442.0,
      "loss": 0.127,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -14.0625,
      "rewards/margins": 8.1875,
      "rewards/rejected": -22.25,
      "step": 5620
    },
    {
      "epoch": 0.7195347945555627,
      "grad_norm": 9.522528995269424,
      "learning_rate": 1.0661929084080466e-06,
      "logits/chosen": 0.04541015625,
      "logits/rejected": 0.59765625,
      "logps/chosen": -434.0,
      "logps/rejected": -452.0,
      "loss": 0.1299,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.3125,
      "rewards/margins": 7.46875,
      "rewards/rejected": -21.75,
      "step": 5630
    },
    {
      "epoch": 0.7208128314908301,
      "grad_norm": 6.7940790582108,
      "learning_rate": 1.0652472825826149e-06,
      "logits/chosen": 0.193359375,
      "logits/rejected": 0.703125,
      "logps/chosen": -398.0,
      "logps/rejected": -434.0,
      "loss": 0.1055,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -13.0625,
      "rewards/margins": 7.34375,
      "rewards/rejected": -20.5,
      "step": 5640
    },
    {
      "epoch": 0.7220908684260975,
      "grad_norm": 9.580061001039104,
      "learning_rate": 1.0643041683803828e-06,
      "logits/chosen": -0.03125,
      "logits/rejected": 0.6875,
      "logps/chosen": -432.0,
      "logps/rejected": -454.0,
      "loss": 0.14,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.8125,
      "rewards/margins": 7.625,
      "rewards/rejected": -22.5,
      "step": 5650
    },
    {
      "epoch": 0.7233689053613649,
      "grad_norm": 12.09452552626403,
      "learning_rate": 1.063363554702701e-06,
      "logits/chosen": 0.039306640625,
      "logits/rejected": 0.5546875,
      "logps/chosen": -426.0,
      "logps/rejected": -442.0,
      "loss": 0.1166,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.9375,
      "rewards/margins": 7.21875,
      "rewards/rejected": -22.125,
      "step": 5660
    },
    {
      "epoch": 0.7246469422966324,
      "grad_norm": 9.174481178437128,
      "learning_rate": 1.0624254305194609e-06,
      "logits/chosen": -0.10302734375,
      "logits/rejected": 0.4921875,
      "logps/chosen": -446.0,
      "logps/rejected": -488.0,
      "loss": 0.1129,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.875,
      "rewards/margins": 8.1875,
      "rewards/rejected": -23.0,
      "step": 5670
    },
    {
      "epoch": 0.7259249792318998,
      "grad_norm": 7.063864490727168,
      "learning_rate": 1.0614897848685505e-06,
      "logits/chosen": -0.029052734375,
      "logits/rejected": 0.65234375,
      "logps/chosen": -412.0,
      "logps/rejected": -438.0,
      "loss": 0.0938,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.0,
      "rewards/margins": 7.65625,
      "rewards/rejected": -21.625,
      "step": 5680
    },
    {
      "epoch": 0.7272030161671672,
      "grad_norm": 9.194194123978837,
      "learning_rate": 1.0605566068553173e-06,
      "logits/chosen": 0.0184326171875,
      "logits/rejected": 0.6796875,
      "logps/chosen": -444.0,
      "logps/rejected": -476.0,
      "loss": 0.1326,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.0,
      "rewards/margins": 7.46875,
      "rewards/rejected": -23.5,
      "step": 5690
    },
    {
      "epoch": 0.7284810531024346,
      "grad_norm": 7.005412861571412,
      "learning_rate": 1.0596258856520351e-06,
      "logits/chosen": -0.0020599365234375,
      "logits/rejected": 0.52734375,
      "logps/chosen": -410.0,
      "logps/rejected": -472.0,
      "loss": 0.1278,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.5,
      "rewards/margins": 7.90625,
      "rewards/rejected": -22.375,
      "step": 5700
    },
    {
      "epoch": 0.7297590900377021,
      "grad_norm": 10.321352122196387,
      "learning_rate": 1.0586976104973764e-06,
      "logits/chosen": -0.037841796875,
      "logits/rejected": 0.6484375,
      "logps/chosen": -434.0,
      "logps/rejected": -450.0,
      "loss": 0.1587,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.9375,
      "rewards/margins": 8.0,
      "rewards/rejected": -23.0,
      "step": 5710
    },
    {
      "epoch": 0.7310371269729695,
      "grad_norm": 10.14624600611338,
      "learning_rate": 1.05777177069589e-06,
      "logits/chosen": 0.01092529296875,
      "logits/rejected": 0.62890625,
      "logps/chosen": -424.0,
      "logps/rejected": -442.0,
      "loss": 0.1299,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.0625,
      "rewards/margins": 7.71875,
      "rewards/rejected": -21.75,
      "step": 5720
    },
    {
      "epoch": 0.7323151639082369,
      "grad_norm": 3.7046963478054313,
      "learning_rate": 1.0568483556174834e-06,
      "logits/chosen": 0.0419921875,
      "logits/rejected": 0.65625,
      "logps/chosen": -414.0,
      "logps/rejected": -440.0,
      "loss": 0.1308,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.1875,
      "rewards/margins": 6.90625,
      "rewards/rejected": -22.125,
      "step": 5730
    },
    {
      "epoch": 0.7335932008435043,
      "grad_norm": 5.125540013879883,
      "learning_rate": 1.055927354696909e-06,
      "logits/chosen": 0.0634765625,
      "logits/rejected": 0.72265625,
      "logps/chosen": -416.0,
      "logps/rejected": -446.0,
      "loss": 0.1155,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.5,
      "rewards/margins": 6.78125,
      "rewards/rejected": -22.25,
      "step": 5740
    },
    {
      "epoch": 0.7348712377787718,
      "grad_norm": 9.061196170976402,
      "learning_rate": 1.0550087574332592e-06,
      "logits/chosen": 0.052001953125,
      "logits/rejected": 0.66015625,
      "logps/chosen": -414.0,
      "logps/rejected": -456.0,
      "loss": 0.1276,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -15.625,
      "rewards/margins": 6.90625,
      "rewards/rejected": -22.5,
      "step": 5750
    },
    {
      "epoch": 0.7361492747140392,
      "grad_norm": 11.90173154651184,
      "learning_rate": 1.0540925533894598e-06,
      "logits/chosen": 0.0279541015625,
      "logits/rejected": 0.6875,
      "logps/chosen": -404.0,
      "logps/rejected": -436.0,
      "loss": 0.0997,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -13.25,
      "rewards/margins": 7.3125,
      "rewards/rejected": -20.5,
      "step": 5760
    },
    {
      "epoch": 0.7374273116493066,
      "grad_norm": 11.148761253060105,
      "learning_rate": 1.053178732191775e-06,
      "logits/chosen": 0.1484375,
      "logits/rejected": 0.65625,
      "logps/chosen": -406.0,
      "logps/rejected": -446.0,
      "loss": 0.0924,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.3125,
      "rewards/margins": 7.09375,
      "rewards/rejected": -22.375,
      "step": 5770
    },
    {
      "epoch": 0.738705348584574,
      "grad_norm": 10.920627483756013,
      "learning_rate": 1.0522672835293127e-06,
      "logits/chosen": 0.08203125,
      "logits/rejected": 0.78515625,
      "logps/chosen": -418.0,
      "logps/rejected": -448.0,
      "loss": 0.1276,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.5,
      "rewards/margins": 7.28125,
      "rewards/rejected": -22.75,
      "step": 5780
    },
    {
      "epoch": 0.7399833855198416,
      "grad_norm": 4.289904714275574,
      "learning_rate": 1.0513581971535365e-06,
      "logits/chosen": -0.00909423828125,
      "logits/rejected": 0.57421875,
      "logps/chosen": -408.0,
      "logps/rejected": -452.0,
      "loss": 0.1201,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.6875,
      "rewards/margins": 6.9375,
      "rewards/rejected": -21.625,
      "step": 5790
    },
    {
      "epoch": 0.741261422455109,
      "grad_norm": 10.604202309276873,
      "learning_rate": 1.0504514628777803e-06,
      "logits/chosen": 0.060791015625,
      "logits/rejected": 0.53125,
      "logps/chosen": -420.0,
      "logps/rejected": -468.0,
      "loss": 0.1537,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.6875,
      "rewards/margins": 7.625,
      "rewards/rejected": -22.375,
      "step": 5800
    },
    {
      "epoch": 0.7425394593903764,
      "grad_norm": 6.139945281988492,
      "learning_rate": 1.0495470705767713e-06,
      "logits/chosen": 0.00677490234375,
      "logits/rejected": 0.73046875,
      "logps/chosen": -440.0,
      "logps/rejected": -462.0,
      "loss": 0.1175,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.375,
      "rewards/margins": 7.5625,
      "rewards/rejected": -23.0,
      "step": 5810
    },
    {
      "epoch": 0.7438174963256438,
      "grad_norm": 5.3810826604442745,
      "learning_rate": 1.0486450101861527e-06,
      "logits/chosen": 0.1318359375,
      "logits/rejected": 0.78515625,
      "logps/chosen": -432.0,
      "logps/rejected": -460.0,
      "loss": 0.1249,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.28125,
      "rewards/rejected": -23.375,
      "step": 5820
    },
    {
      "epoch": 0.7450955332609113,
      "grad_norm": 9.961514117979055,
      "learning_rate": 1.0477452717020143e-06,
      "logits/chosen": 0.10107421875,
      "logits/rejected": 0.8515625,
      "logps/chosen": -424.0,
      "logps/rejected": -450.0,
      "loss": 0.138,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.5625,
      "rewards/margins": 7.03125,
      "rewards/rejected": -22.625,
      "step": 5830
    },
    {
      "epoch": 0.7463735701961787,
      "grad_norm": 10.168231388113293,
      "learning_rate": 1.0468478451804272e-06,
      "logits/chosen": 0.0498046875,
      "logits/rejected": 0.7421875,
      "logps/chosen": -442.0,
      "logps/rejected": -470.0,
      "loss": 0.1355,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.46875,
      "rewards/rejected": -23.75,
      "step": 5840
    },
    {
      "epoch": 0.7476516071314461,
      "grad_norm": 7.858946647736502,
      "learning_rate": 1.0459527207369814e-06,
      "logits/chosen": 0.07373046875,
      "logits/rejected": 0.68359375,
      "logps/chosen": -444.0,
      "logps/rejected": -496.0,
      "loss": 0.1261,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.125,
      "rewards/margins": 8.3125,
      "rewards/rejected": -24.5,
      "step": 5850
    },
    {
      "epoch": 0.7489296440667135,
      "grad_norm": 9.142060581382754,
      "learning_rate": 1.0450598885463281e-06,
      "logits/chosen": 0.1533203125,
      "logits/rejected": 0.7109375,
      "logps/chosen": -410.0,
      "logps/rejected": -486.0,
      "loss": 0.1299,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.625,
      "rewards/margins": 8.6875,
      "rewards/rejected": -24.375,
      "step": 5860
    },
    {
      "epoch": 0.750207681001981,
      "grad_norm": 12.377612873515874,
      "learning_rate": 1.0441693388417282e-06,
      "logits/chosen": 0.0390625,
      "logits/rejected": 0.79296875,
      "logps/chosen": -432.0,
      "logps/rejected": -458.0,
      "loss": 0.1343,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.875,
      "rewards/margins": 7.59375,
      "rewards/rejected": -23.5,
      "step": 5870
    },
    {
      "epoch": 0.7514857179372484,
      "grad_norm": 8.837319641597022,
      "learning_rate": 1.0432810619146023e-06,
      "logits/chosen": 0.169921875,
      "logits/rejected": 0.703125,
      "logps/chosen": -426.0,
      "logps/rejected": -472.0,
      "loss": 0.1532,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.875,
      "rewards/margins": 7.5,
      "rewards/rejected": -23.375,
      "step": 5880
    },
    {
      "epoch": 0.7527637548725158,
      "grad_norm": 9.043360333209531,
      "learning_rate": 1.042395048114086e-06,
      "logits/chosen": 0.050537109375,
      "logits/rejected": 0.6328125,
      "logps/chosen": -446.0,
      "logps/rejected": -488.0,
      "loss": 0.1343,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.3125,
      "rewards/margins": 7.09375,
      "rewards/rejected": -22.375,
      "step": 5890
    },
    {
      "epoch": 0.7540417918077832,
      "grad_norm": 7.084994526293647,
      "learning_rate": 1.041511287846591e-06,
      "logits/chosen": 0.0089111328125,
      "logits/rejected": 0.77734375,
      "logps/chosen": -416.0,
      "logps/rejected": -446.0,
      "loss": 0.1233,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.9375,
      "rewards/margins": 7.0,
      "rewards/rejected": -21.875,
      "step": 5900
    },
    {
      "epoch": 0.7553198287430507,
      "grad_norm": 10.571251359615145,
      "learning_rate": 1.0406297715753674e-06,
      "logits/chosen": 0.11181640625,
      "logits/rejected": 0.7890625,
      "logps/chosen": -414.0,
      "logps/rejected": -440.0,
      "loss": 0.156,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.9375,
      "rewards/margins": 6.96875,
      "rewards/rejected": -21.875,
      "step": 5910
    },
    {
      "epoch": 0.7565978656783181,
      "grad_norm": 8.450092764390895,
      "learning_rate": 1.0397504898200726e-06,
      "logits/chosen": 0.06640625,
      "logits/rejected": 0.765625,
      "logps/chosen": -450.0,
      "logps/rejected": -456.0,
      "loss": 0.1248,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.0,
      "rewards/margins": 7.0,
      "rewards/rejected": -23.0,
      "step": 5920
    },
    {
      "epoch": 0.7578759026135855,
      "grad_norm": 10.203399496457635,
      "learning_rate": 1.0388734331563415e-06,
      "logits/chosen": -0.00439453125,
      "logits/rejected": 0.93359375,
      "logps/chosen": -444.0,
      "logps/rejected": -476.0,
      "loss": 0.1453,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.40625,
      "rewards/rejected": -24.0,
      "step": 5930
    },
    {
      "epoch": 0.7591539395488529,
      "grad_norm": 7.372284239429812,
      "learning_rate": 1.037998592215364e-06,
      "logits/chosen": 0.07958984375,
      "logits/rejected": 0.65234375,
      "logps/chosen": -460.0,
      "logps/rejected": -488.0,
      "loss": 0.1325,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.625,
      "rewards/margins": 8.0625,
      "rewards/rejected": -24.75,
      "step": 5940
    },
    {
      "epoch": 0.7604319764841204,
      "grad_norm": 8.006373158663623,
      "learning_rate": 1.037125957683463e-06,
      "logits/chosen": -0.05126953125,
      "logits/rejected": 0.59375,
      "logps/chosen": -460.0,
      "logps/rejected": -486.0,
      "loss": 0.1203,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.4375,
      "rewards/margins": 7.75,
      "rewards/rejected": -23.25,
      "step": 5950
    },
    {
      "epoch": 0.7617100134193878,
      "grad_norm": 10.863475963486735,
      "learning_rate": 1.0362555203016794e-06,
      "logits/chosen": 0.1083984375,
      "logits/rejected": 0.6484375,
      "logps/chosen": -408.0,
      "logps/rejected": -464.0,
      "loss": 0.1239,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -14.875,
      "rewards/margins": 7.125,
      "rewards/rejected": -22.0,
      "step": 5960
    },
    {
      "epoch": 0.7629880503546552,
      "grad_norm": 8.273983979883907,
      "learning_rate": 1.035387270865359e-06,
      "logits/chosen": 0.142578125,
      "logits/rejected": 0.71484375,
      "logps/chosen": -438.0,
      "logps/rejected": -452.0,
      "loss": 0.1392,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -15.625,
      "rewards/margins": 8.0,
      "rewards/rejected": -23.625,
      "step": 5970
    },
    {
      "epoch": 0.7642660872899226,
      "grad_norm": 8.41248843441412,
      "learning_rate": 1.0345212002237434e-06,
      "logits/chosen": 0.0849609375,
      "logits/rejected": 0.7734375,
      "logps/chosen": -452.0,
      "logps/rejected": -452.0,
      "loss": 0.1409,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -16.125,
      "rewards/margins": 6.59375,
      "rewards/rejected": -22.75,
      "step": 5980
    },
    {
      "epoch": 0.7655441242251901,
      "grad_norm": 5.642789839150977,
      "learning_rate": 1.0336572992795644e-06,
      "logits/chosen": 0.1787109375,
      "logits/rejected": 0.7890625,
      "logps/chosen": -434.0,
      "logps/rejected": -462.0,
      "loss": 0.1157,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.8125,
      "rewards/margins": 7.5,
      "rewards/rejected": -23.375,
      "step": 5990
    },
    {
      "epoch": 0.7668221611604575,
      "grad_norm": 6.273309963854991,
      "learning_rate": 1.0327955589886444e-06,
      "logits/chosen": 0.1787109375,
      "logits/rejected": 0.65625,
      "logps/chosen": -434.0,
      "logps/rejected": -472.0,
      "loss": 0.105,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.0,
      "rewards/rejected": -23.75,
      "step": 6000
    },
    {
      "epoch": 0.7681001980957249,
      "grad_norm": 9.636067013024944,
      "learning_rate": 1.0319359703594971e-06,
      "logits/chosen": 0.1044921875,
      "logits/rejected": 0.6328125,
      "logps/chosen": -436.0,
      "logps/rejected": -468.0,
      "loss": 0.1154,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.6875,
      "rewards/margins": 7.84375,
      "rewards/rejected": -23.5,
      "step": 6010
    },
    {
      "epoch": 0.7693782350309923,
      "grad_norm": 7.8920872245331175,
      "learning_rate": 1.0310785244529341e-06,
      "logits/chosen": 0.0654296875,
      "logits/rejected": 0.6484375,
      "logps/chosen": -446.0,
      "logps/rejected": -516.0,
      "loss": 0.0968,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.71875,
      "rewards/rejected": -24.25,
      "step": 6020
    },
    {
      "epoch": 0.7706562719662599,
      "grad_norm": 8.698664438396241,
      "learning_rate": 1.0302232123816746e-06,
      "logits/chosen": 0.11083984375,
      "logits/rejected": 0.82421875,
      "logps/chosen": -418.0,
      "logps/rejected": -444.0,
      "loss": 0.1306,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.375,
      "rewards/margins": 7.71875,
      "rewards/rejected": -22.125,
      "step": 6030
    },
    {
      "epoch": 0.7719343089015273,
      "grad_norm": 7.398592719727823,
      "learning_rate": 1.0293700253099576e-06,
      "logits/chosen": 0.07080078125,
      "logits/rejected": 0.71875,
      "logps/chosen": -406.0,
      "logps/rejected": -442.0,
      "loss": 0.1126,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.6875,
      "rewards/margins": 7.3125,
      "rewards/rejected": -22.0,
      "step": 6040
    },
    {
      "epoch": 0.7732123458367947,
      "grad_norm": 7.244574444100897,
      "learning_rate": 1.02851895445316e-06,
      "logits/chosen": 0.1572265625,
      "logits/rejected": 0.8515625,
      "logps/chosen": -432.0,
      "logps/rejected": -462.0,
      "loss": 0.1145,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.75,
      "rewards/rejected": -24.25,
      "step": 6050
    },
    {
      "epoch": 0.7744903827720621,
      "grad_norm": 9.162267210199335,
      "learning_rate": 1.0276699910774159e-06,
      "logits/chosen": -0.06689453125,
      "logits/rejected": 0.828125,
      "logps/chosen": -434.0,
      "logps/rejected": -464.0,
      "loss": 0.1195,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.5,
      "rewards/margins": 7.84375,
      "rewards/rejected": -23.375,
      "step": 6060
    },
    {
      "epoch": 0.7757684197073296,
      "grad_norm": 11.129975295829345,
      "learning_rate": 1.0268231264992398e-06,
      "logits/chosen": 0.166015625,
      "logits/rejected": 0.765625,
      "logps/chosen": -416.0,
      "logps/rejected": -462.0,
      "loss": 0.1517,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -22.375,
      "step": 6070
    },
    {
      "epoch": 0.777046456642597,
      "grad_norm": 6.1652386419986,
      "learning_rate": 1.0259783520851542e-06,
      "logits/chosen": 0.30859375,
      "logits/rejected": 0.66015625,
      "logps/chosen": -396.0,
      "logps/rejected": -470.0,
      "loss": 0.1149,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.875,
      "rewards/margins": 8.375,
      "rewards/rejected": -23.25,
      "step": 6080
    },
    {
      "epoch": 0.7783244935778644,
      "grad_norm": 6.876890167487323,
      "learning_rate": 1.0251356592513193e-06,
      "logits/chosen": 0.0966796875,
      "logits/rejected": 0.78515625,
      "logps/chosen": -426.0,
      "logps/rejected": -452.0,
      "loss": 0.1111,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.5,
      "rewards/margins": 7.625,
      "rewards/rejected": -23.125,
      "step": 6090
    },
    {
      "epoch": 0.7796025305131318,
      "grad_norm": 5.224332045946296,
      "learning_rate": 1.0242950394631678e-06,
      "logits/chosen": 0.240234375,
      "logits/rejected": 0.69921875,
      "logps/chosen": -424.0,
      "logps/rejected": -466.0,
      "loss": 0.0945,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -15.6875,
      "rewards/margins": 7.21875,
      "rewards/rejected": -22.875,
      "step": 6100
    },
    {
      "epoch": 0.7808805674483993,
      "grad_norm": 12.875496097181719,
      "learning_rate": 1.0234564842350404e-06,
      "logits/chosen": 0.162109375,
      "logits/rejected": 0.74609375,
      "logps/chosen": -404.0,
      "logps/rejected": -440.0,
      "loss": 0.1048,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.5625,
      "rewards/margins": 6.96875,
      "rewards/rejected": -21.5,
      "step": 6110
    },
    {
      "epoch": 0.7821586043836667,
      "grad_norm": 8.588331019237048,
      "learning_rate": 1.0226199851298272e-06,
      "logits/chosen": -0.0311279296875,
      "logits/rejected": 0.57421875,
      "logps/chosen": -420.0,
      "logps/rejected": -460.0,
      "loss": 0.1009,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -15.125,
      "rewards/margins": 7.90625,
      "rewards/rejected": -23.0,
      "step": 6120
    },
    {
      "epoch": 0.7834366413189341,
      "grad_norm": 8.432609301306668,
      "learning_rate": 1.0217855337586106e-06,
      "logits/chosen": 0.0888671875,
      "logits/rejected": 0.609375,
      "logps/chosen": -428.0,
      "logps/rejected": -472.0,
      "loss": 0.1061,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -15.375,
      "rewards/margins": 7.96875,
      "rewards/rejected": -23.375,
      "step": 6130
    },
    {
      "epoch": 0.7847146782542015,
      "grad_norm": 6.555870680389009,
      "learning_rate": 1.0209531217803119e-06,
      "logits/chosen": -0.00341796875,
      "logits/rejected": 0.65625,
      "logps/chosen": -432.0,
      "logps/rejected": -468.0,
      "loss": 0.122,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.25,
      "rewards/margins": 7.65625,
      "rewards/rejected": -22.875,
      "step": 6140
    },
    {
      "epoch": 0.785992715189469,
      "grad_norm": 7.289376441716546,
      "learning_rate": 1.0201227409013412e-06,
      "logits/chosen": 0.041748046875,
      "logits/rejected": 0.5859375,
      "logps/chosen": -426.0,
      "logps/rejected": -490.0,
      "loss": 0.1104,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.625,
      "rewards/margins": 8.8125,
      "rewards/rejected": -24.5,
      "step": 6150
    },
    {
      "epoch": 0.7872707521247364,
      "grad_norm": 5.447905358690935,
      "learning_rate": 1.0192943828752509e-06,
      "logits/chosen": 0.05517578125,
      "logits/rejected": 0.875,
      "logps/chosen": -462.0,
      "logps/rejected": -496.0,
      "loss": 0.1064,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.75,
      "rewards/margins": 8.125,
      "rewards/rejected": -25.875,
      "step": 6160
    },
    {
      "epoch": 0.7885487890600038,
      "grad_norm": 9.05755014858366,
      "learning_rate": 1.0184680395023912e-06,
      "logits/chosen": -0.0830078125,
      "logits/rejected": 0.58203125,
      "logps/chosen": -454.0,
      "logps/rejected": -472.0,
      "loss": 0.1092,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.84375,
      "rewards/rejected": -24.25,
      "step": 6170
    },
    {
      "epoch": 0.7898268259952712,
      "grad_norm": 8.638274424252117,
      "learning_rate": 1.0176437026295688e-06,
      "logits/chosen": 0.11279296875,
      "logits/rejected": 0.6953125,
      "logps/chosen": -432.0,
      "logps/rejected": -502.0,
      "loss": 0.0983,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.625,
      "rewards/margins": 8.75,
      "rewards/rejected": -25.375,
      "step": 6180
    },
    {
      "epoch": 0.7911048629305387,
      "grad_norm": 4.07182743563226,
      "learning_rate": 1.0168213641497094e-06,
      "logits/chosen": 0.00162506103515625,
      "logits/rejected": 0.81640625,
      "logps/chosen": -422.0,
      "logps/rejected": -448.0,
      "loss": 0.1113,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.375,
      "rewards/margins": 8.375,
      "rewards/rejected": -23.75,
      "step": 6190
    },
    {
      "epoch": 0.7923828998658061,
      "grad_norm": 11.479916655727616,
      "learning_rate": 1.016001016001524e-06,
      "logits/chosen": -0.08642578125,
      "logits/rejected": 0.66015625,
      "logps/chosen": -430.0,
      "logps/rejected": -462.0,
      "loss": 0.1312,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.625,
      "rewards/margins": 7.375,
      "rewards/rejected": -23.0,
      "step": 6200
    },
    {
      "epoch": 0.7936609368010735,
      "grad_norm": 9.679791334659527,
      "learning_rate": 1.0151826501691747e-06,
      "logits/chosen": -0.087890625,
      "logits/rejected": 0.66796875,
      "logps/chosen": -446.0,
      "logps/rejected": -460.0,
      "loss": 0.1492,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -17.125,
      "rewards/margins": 6.9375,
      "rewards/rejected": -24.0,
      "step": 6210
    },
    {
      "epoch": 0.7949389737363409,
      "grad_norm": 12.01826314612776,
      "learning_rate": 1.0143662586819475e-06,
      "logits/chosen": 0.09814453125,
      "logits/rejected": 0.7109375,
      "logps/chosen": -438.0,
      "logps/rejected": -472.0,
      "loss": 0.139,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.09375,
      "rewards/rejected": -23.5,
      "step": 6220
    },
    {
      "epoch": 0.7962170106716084,
      "grad_norm": 5.462934030019688,
      "learning_rate": 1.0135518336139257e-06,
      "logits/chosen": 0.13671875,
      "logits/rejected": 0.73046875,
      "logps/chosen": -424.0,
      "logps/rejected": -452.0,
      "loss": 0.1235,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.125,
      "rewards/margins": 6.28125,
      "rewards/rejected": -22.375,
      "step": 6230
    },
    {
      "epoch": 0.7974950476068758,
      "grad_norm": 6.534492785421319,
      "learning_rate": 1.0127393670836666e-06,
      "logits/chosen": 0.1650390625,
      "logits/rejected": 0.65234375,
      "logps/chosen": -422.0,
      "logps/rejected": -456.0,
      "loss": 0.1154,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -15.625,
      "rewards/margins": 6.28125,
      "rewards/rejected": -21.875,
      "step": 6240
    },
    {
      "epoch": 0.7987730845421432,
      "grad_norm": 7.798348333738603,
      "learning_rate": 1.0119288512538813e-06,
      "logits/chosen": 0.076171875,
      "logits/rejected": 0.71484375,
      "logps/chosen": -430.0,
      "logps/rejected": -462.0,
      "loss": 0.1505,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.375,
      "rewards/margins": 6.84375,
      "rewards/rejected": -23.25,
      "step": 6250
    },
    {
      "epoch": 0.8000511214774106,
      "grad_norm": 4.940034029216261,
      "learning_rate": 1.0111202783311173e-06,
      "logits/chosen": -0.04248046875,
      "logits/rejected": 0.5546875,
      "logps/chosen": -432.0,
      "logps/rejected": -460.0,
      "loss": 0.0926,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -15.3125,
      "rewards/margins": 7.1875,
      "rewards/rejected": -22.5,
      "step": 6260
    },
    {
      "epoch": 0.8013291584126782,
      "grad_norm": 7.027886564138196,
      "learning_rate": 1.010313640565443e-06,
      "logits/chosen": 0.119140625,
      "logits/rejected": 0.73828125,
      "logps/chosen": -416.0,
      "logps/rejected": -438.0,
      "loss": 0.1345,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.5,
      "rewards/margins": 6.65625,
      "rewards/rejected": -21.125,
      "step": 6270
    },
    {
      "epoch": 0.8026071953479456,
      "grad_norm": 7.722932177214637,
      "learning_rate": 1.0095089302501373e-06,
      "logits/chosen": 0.0205078125,
      "logits/rejected": 0.59765625,
      "logps/chosen": -442.0,
      "logps/rejected": -476.0,
      "loss": 0.1134,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.375,
      "rewards/margins": 7.4375,
      "rewards/rejected": -22.75,
      "step": 6280
    },
    {
      "epoch": 0.803885232283213,
      "grad_norm": 9.255800385457958,
      "learning_rate": 1.0087061397213787e-06,
      "logits/chosen": 0.20703125,
      "logits/rejected": 0.7265625,
      "logps/chosen": -422.0,
      "logps/rejected": -456.0,
      "loss": 0.1279,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -15.1875,
      "rewards/margins": 7.1875,
      "rewards/rejected": -22.375,
      "step": 6290
    },
    {
      "epoch": 0.8051632692184804,
      "grad_norm": 7.990972797673917,
      "learning_rate": 1.007905261357939e-06,
      "logits/chosen": 0.1748046875,
      "logits/rejected": 0.77734375,
      "logps/chosen": -394.0,
      "logps/rejected": -450.0,
      "loss": 0.1115,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.8125,
      "rewards/margins": 7.96875,
      "rewards/rejected": -22.75,
      "step": 6300
    },
    {
      "epoch": 0.8064413061537479,
      "grad_norm": 9.998938029238875,
      "learning_rate": 1.0071062875808811e-06,
      "logits/chosen": 0.1318359375,
      "logits/rejected": 0.703125,
      "logps/chosen": -408.0,
      "logps/rejected": -454.0,
      "loss": 0.1124,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.125,
      "rewards/margins": 7.6875,
      "rewards/rejected": -21.875,
      "step": 6310
    },
    {
      "epoch": 0.8077193430890153,
      "grad_norm": 4.817103554266857,
      "learning_rate": 1.0063092108532552e-06,
      "logits/chosen": 0.1728515625,
      "logits/rejected": 0.71875,
      "logps/chosen": -438.0,
      "logps/rejected": -472.0,
      "loss": 0.1264,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.21875,
      "rewards/rejected": -23.5,
      "step": 6320
    },
    {
      "epoch": 0.8089973800242827,
      "grad_norm": 10.825221591467866,
      "learning_rate": 1.005514023679802e-06,
      "logits/chosen": 0.22265625,
      "logits/rejected": 0.76171875,
      "logps/chosen": -440.0,
      "logps/rejected": -480.0,
      "loss": 0.1257,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.0,
      "rewards/margins": 7.375,
      "rewards/rejected": -24.375,
      "step": 6330
    },
    {
      "epoch": 0.8102754169595501,
      "grad_norm": 6.043797260418159,
      "learning_rate": 1.0047207186066567e-06,
      "logits/chosen": 0.1298828125,
      "logits/rejected": 0.77734375,
      "logps/chosen": -452.0,
      "logps/rejected": -480.0,
      "loss": 0.1072,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.75,
      "rewards/margins": 7.53125,
      "rewards/rejected": -25.25,
      "step": 6340
    },
    {
      "epoch": 0.8115534538948176,
      "grad_norm": 9.076771709672522,
      "learning_rate": 1.0039292882210538e-06,
      "logits/chosen": 0.263671875,
      "logits/rejected": 0.69140625,
      "logps/chosen": -450.0,
      "logps/rejected": -482.0,
      "loss": 0.1242,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -17.125,
      "rewards/margins": 6.8125,
      "rewards/rejected": -23.875,
      "step": 6350
    },
    {
      "epoch": 0.812831490830085,
      "grad_norm": 13.107435254561711,
      "learning_rate": 1.0031397251510382e-06,
      "logits/chosen": 0.12060546875,
      "logits/rejected": 0.66015625,
      "logps/chosen": -434.0,
      "logps/rejected": -484.0,
      "loss": 0.1179,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.5625,
      "rewards/rejected": -23.875,
      "step": 6360
    },
    {
      "epoch": 0.8141095277653524,
      "grad_norm": 6.142700712296344,
      "learning_rate": 1.0023520220651762e-06,
      "logits/chosen": -0.00732421875,
      "logits/rejected": 0.73046875,
      "logps/chosen": -412.0,
      "logps/rejected": -444.0,
      "loss": 0.0949,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.0625,
      "rewards/margins": 7.46875,
      "rewards/rejected": -21.625,
      "step": 6370
    },
    {
      "epoch": 0.8153875647006198,
      "grad_norm": 6.083143082562113,
      "learning_rate": 1.0015661716722687e-06,
      "logits/chosen": 0.181640625,
      "logits/rejected": 0.72265625,
      "logps/chosen": -406.0,
      "logps/rejected": -476.0,
      "loss": 0.1196,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -15.0625,
      "rewards/margins": 8.125,
      "rewards/rejected": -23.125,
      "step": 6380
    },
    {
      "epoch": 0.8166656016358873,
      "grad_norm": 3.7536260513657997,
      "learning_rate": 1.0007821667210687e-06,
      "logits/chosen": 0.058349609375,
      "logits/rejected": 0.71484375,
      "logps/chosen": -450.0,
      "logps/rejected": -474.0,
      "loss": 0.1142,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.8125,
      "rewards/rejected": -24.25,
      "step": 6390
    },
    {
      "epoch": 0.8179436385711547,
      "grad_norm": 6.958047775231169,
      "learning_rate": 1e-06,
      "logits/chosen": 0.298828125,
      "logits/rejected": 0.68359375,
      "logps/chosen": -428.0,
      "logps/rejected": -486.0,
      "loss": 0.1249,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -17.5,
      "rewards/margins": 6.84375,
      "rewards/rejected": -24.375,
      "step": 6400
    },
    {
      "epoch": 0.8192216755064221,
      "grad_norm": 8.379376955591015,
      "learning_rate": 9.992196643368784e-07,
      "logits/chosen": 0.2255859375,
      "logits/rejected": 0.86328125,
      "logps/chosen": -426.0,
      "logps/rejected": -464.0,
      "loss": 0.1365,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.875,
      "rewards/margins": 7.375,
      "rewards/rejected": -24.25,
      "step": 6410
    },
    {
      "epoch": 0.8204997124416895,
      "grad_norm": 4.76296683559907,
      "learning_rate": 9.984411525986355e-07,
      "logits/chosen": 0.1943359375,
      "logits/rejected": 0.7265625,
      "logps/chosen": -456.0,
      "logps/rejected": -510.0,
      "loss": 0.0999,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.125,
      "rewards/margins": 8.125,
      "rewards/rejected": -26.25,
      "step": 6420
    },
    {
      "epoch": 0.821777749376957,
      "grad_norm": 8.902873672303011,
      "learning_rate": 9.97664457691046e-07,
      "logits/chosen": 0.2412109375,
      "logits/rejected": 0.83203125,
      "logps/chosen": -450.0,
      "logps/rejected": -512.0,
      "loss": 0.1052,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.625,
      "rewards/margins": 8.1875,
      "rewards/rejected": -26.75,
      "step": 6430
    },
    {
      "epoch": 0.8230557863122244,
      "grad_norm": 7.177069325357264,
      "learning_rate": 9.968895725584535e-07,
      "logits/chosen": 0.1220703125,
      "logits/rejected": 0.87890625,
      "logps/chosen": -446.0,
      "logps/rejected": -472.0,
      "loss": 0.1124,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.125,
      "rewards/margins": 8.125,
      "rewards/rejected": -25.375,
      "step": 6440
    },
    {
      "epoch": 0.8243338232474918,
      "grad_norm": 3.810295308769632,
      "learning_rate": 9.961164901835046e-07,
      "logits/chosen": 0.2138671875,
      "logits/rejected": 0.91796875,
      "logps/chosen": -444.0,
      "logps/rejected": -496.0,
      "loss": 0.1029,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -18.25,
      "rewards/margins": 7.25,
      "rewards/rejected": -25.5,
      "step": 6450
    },
    {
      "epoch": 0.8256118601827592,
      "grad_norm": 5.550414582734014,
      "learning_rate": 9.95345203586879e-07,
      "logits/chosen": 0.291015625,
      "logits/rejected": 0.9375,
      "logps/chosen": -420.0,
      "logps/rejected": -472.0,
      "loss": 0.1182,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -17.625,
      "rewards/margins": 7.6875,
      "rewards/rejected": -25.25,
      "step": 6460
    },
    {
      "epoch": 0.8268898971180267,
      "grad_norm": 8.761988088207138,
      "learning_rate": 9.94575705827027e-07,
      "logits/chosen": -0.07861328125,
      "logits/rejected": 0.7109375,
      "logps/chosen": -452.0,
      "logps/rejected": -506.0,
      "loss": 0.1234,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.875,
      "rewards/margins": 8.4375,
      "rewards/rejected": -25.25,
      "step": 6470
    },
    {
      "epoch": 0.8281679340532941,
      "grad_norm": 9.098531129377148,
      "learning_rate": 9.938079899999065e-07,
      "logits/chosen": 0.1279296875,
      "logits/rejected": 0.71484375,
      "logps/chosen": -438.0,
      "logps/rejected": -452.0,
      "loss": 0.145,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.0625,
      "rewards/rejected": -23.25,
      "step": 6480
    },
    {
      "epoch": 0.8294459709885615,
      "grad_norm": 3.3741716029544193,
      "learning_rate": 9.930420492387219e-07,
      "logits/chosen": 0.0595703125,
      "logits/rejected": 0.48828125,
      "logps/chosen": -422.0,
      "logps/rejected": -468.0,
      "loss": 0.1263,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -15.1875,
      "rewards/margins": 7.84375,
      "rewards/rejected": -23.0,
      "step": 6490
    },
    {
      "epoch": 0.8307240079238289,
      "grad_norm": 7.706675813847085,
      "learning_rate": 9.922778767136676e-07,
      "logits/chosen": -0.06103515625,
      "logits/rejected": 0.58984375,
      "logps/chosen": -440.0,
      "logps/rejected": -502.0,
      "loss": 0.1118,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -15.9375,
      "rewards/margins": 7.78125,
      "rewards/rejected": -23.75,
      "step": 6500
    },
    {
      "epoch": 0.8320020448590965,
      "grad_norm": 8.198130401144248,
      "learning_rate": 9.915154656316713e-07,
      "logits/chosen": 0.1025390625,
      "logits/rejected": 0.77734375,
      "logps/chosen": -444.0,
      "logps/rejected": -474.0,
      "loss": 0.1266,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -17.125,
      "rewards/margins": 7.15625,
      "rewards/rejected": -24.375,
      "step": 6510
    },
    {
      "epoch": 0.8332800817943639,
      "grad_norm": 7.959135013076853,
      "learning_rate": 9.907548092361398e-07,
      "logits/chosen": -0.134765625,
      "logits/rejected": 0.765625,
      "logps/chosen": -460.0,
      "logps/rejected": -482.0,
      "loss": 0.106,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.25,
      "rewards/margins": 7.40625,
      "rewards/rejected": -24.625,
      "step": 6520
    },
    {
      "epoch": 0.8345581187296313,
      "grad_norm": 10.52639436840607,
      "learning_rate": 9.899959008067097e-07,
      "logits/chosen": 0.00164794921875,
      "logits/rejected": 0.49609375,
      "logps/chosen": -448.0,
      "logps/rejected": -520.0,
      "loss": 0.1575,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -18.0,
      "rewards/margins": 7.8125,
      "rewards/rejected": -25.75,
      "step": 6530
    },
    {
      "epoch": 0.8358361556648987,
      "grad_norm": 4.660479209706208,
      "learning_rate": 9.892387336589959e-07,
      "logits/chosen": 0.09375,
      "logits/rejected": 0.79296875,
      "logps/chosen": -430.0,
      "logps/rejected": -460.0,
      "loss": 0.1126,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.875,
      "rewards/margins": 7.34375,
      "rewards/rejected": -23.125,
      "step": 6540
    },
    {
      "epoch": 0.8371141926001662,
      "grad_norm": 9.475573912751981,
      "learning_rate": 9.884833011443446e-07,
      "logits/chosen": 0.06640625,
      "logits/rejected": 0.6953125,
      "logps/chosen": -412.0,
      "logps/rejected": -474.0,
      "loss": 0.1257,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -15.875,
      "rewards/margins": 7.5,
      "rewards/rejected": -23.375,
      "step": 6550
    },
    {
      "epoch": 0.8383922295354336,
      "grad_norm": 12.00458087739251,
      "learning_rate": 9.877295966495897e-07,
      "logits/chosen": -0.001953125,
      "logits/rejected": 0.6484375,
      "logps/chosen": -388.0,
      "logps/rejected": -426.0,
      "loss": 0.1165,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -13.3125,
      "rewards/margins": 7.40625,
      "rewards/rejected": -20.75,
      "step": 6560
    },
    {
      "epoch": 0.839670266470701,
      "grad_norm": 8.339258109673613,
      "learning_rate": 9.86977613596807e-07,
      "logits/chosen": 0.037841796875,
      "logits/rejected": 0.72265625,
      "logps/chosen": -400.0,
      "logps/rejected": -436.0,
      "loss": 0.1213,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.0625,
      "rewards/margins": 7.25,
      "rewards/rejected": -22.25,
      "step": 6570
    },
    {
      "epoch": 0.8409483034059684,
      "grad_norm": 8.938110516870447,
      "learning_rate": 9.862273454430757e-07,
      "logits/chosen": -0.0211181640625,
      "logits/rejected": 0.72265625,
      "logps/chosen": -444.0,
      "logps/rejected": -460.0,
      "loss": 0.1197,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.6875,
      "rewards/rejected": -23.875,
      "step": 6580
    },
    {
      "epoch": 0.8422263403412359,
      "grad_norm": 6.670693537381478,
      "learning_rate": 9.85478785680238e-07,
      "logits/chosen": 0.1875,
      "logits/rejected": 0.72265625,
      "logps/chosen": -468.0,
      "logps/rejected": -496.0,
      "loss": 0.1313,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -18.625,
      "rewards/margins": 7.125,
      "rewards/rejected": -25.75,
      "step": 6590
    },
    {
      "epoch": 0.8435043772765033,
      "grad_norm": 5.933275632103338,
      "learning_rate": 9.847319278346618e-07,
      "logits/chosen": 0.123046875,
      "logits/rejected": 0.73828125,
      "logps/chosen": -446.0,
      "logps/rejected": -488.0,
      "loss": 0.1097,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.875,
      "rewards/margins": 8.5,
      "rewards/rejected": -25.25,
      "step": 6600
    },
    {
      "epoch": 0.8447824142117707,
      "grad_norm": 10.70971453280206,
      "learning_rate": 9.839867654670063e-07,
      "logits/chosen": 0.12109375,
      "logits/rejected": 0.89453125,
      "logps/chosen": -450.0,
      "logps/rejected": -490.0,
      "loss": 0.0874,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.375,
      "rewards/margins": 7.25,
      "rewards/rejected": -25.625,
      "step": 6610
    },
    {
      "epoch": 0.8460604511470381,
      "grad_norm": 4.802796304209624,
      "learning_rate": 9.832432921719876e-07,
      "logits/chosen": 0.1650390625,
      "logits/rejected": 0.96875,
      "logps/chosen": -432.0,
      "logps/rejected": -476.0,
      "loss": 0.1092,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -18.0,
      "rewards/margins": 8.3125,
      "rewards/rejected": -26.25,
      "step": 6620
    },
    {
      "epoch": 0.8473384880823056,
      "grad_norm": 4.0316277691024,
      "learning_rate": 9.825015015781493e-07,
      "logits/chosen": 0.0341796875,
      "logits/rejected": 0.6875,
      "logps/chosen": -460.0,
      "logps/rejected": -524.0,
      "loss": 0.0924,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.5,
      "rewards/margins": 8.1875,
      "rewards/rejected": -26.625,
      "step": 6630
    },
    {
      "epoch": 0.848616525017573,
      "grad_norm": 6.806200494992519,
      "learning_rate": 9.81761387347632e-07,
      "logits/chosen": 0.1650390625,
      "logits/rejected": 0.8046875,
      "logps/chosen": -444.0,
      "logps/rejected": -482.0,
      "loss": 0.1201,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.0,
      "rewards/margins": 8.125,
      "rewards/rejected": -26.125,
      "step": 6640
    },
    {
      "epoch": 0.8498945619528404,
      "grad_norm": 5.457237439229392,
      "learning_rate": 9.810229431759452e-07,
      "logits/chosen": 0.0830078125,
      "logits/rejected": 0.7109375,
      "logps/chosen": -432.0,
      "logps/rejected": -476.0,
      "loss": 0.1076,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.125,
      "rewards/margins": 8.5,
      "rewards/rejected": -24.625,
      "step": 6650
    },
    {
      "epoch": 0.8511725988881078,
      "grad_norm": 10.895327472815062,
      "learning_rate": 9.802861627917437e-07,
      "logits/chosen": 0.287109375,
      "logits/rejected": 0.8203125,
      "logps/chosen": -446.0,
      "logps/rejected": -494.0,
      "loss": 0.1048,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.25,
      "rewards/margins": 7.53125,
      "rewards/rejected": -25.75,
      "step": 6660
    },
    {
      "epoch": 0.8524506358233753,
      "grad_norm": 7.099643805988576,
      "learning_rate": 9.795510399566016e-07,
      "logits/chosen": 0.189453125,
      "logits/rejected": 0.78515625,
      "logps/chosen": -458.0,
      "logps/rejected": -494.0,
      "loss": 0.1095,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.75,
      "rewards/margins": 7.375,
      "rewards/rejected": -26.125,
      "step": 6670
    },
    {
      "epoch": 0.8537286727586427,
      "grad_norm": 8.623743758617154,
      "learning_rate": 9.788175684647926e-07,
      "logits/chosen": 0.107421875,
      "logits/rejected": 0.78125,
      "logps/chosen": -452.0,
      "logps/rejected": -502.0,
      "loss": 0.1206,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -18.125,
      "rewards/margins": 8.1875,
      "rewards/rejected": -26.375,
      "step": 6680
    },
    {
      "epoch": 0.8550067096939101,
      "grad_norm": 2.9479908008537703,
      "learning_rate": 9.780857421430687e-07,
      "logits/chosen": 0.2392578125,
      "logits/rejected": 0.91015625,
      "logps/chosen": -440.0,
      "logps/rejected": -468.0,
      "loss": 0.1097,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.875,
      "rewards/margins": 7.65625,
      "rewards/rejected": -25.5,
      "step": 6690
    },
    {
      "epoch": 0.8562847466291775,
      "grad_norm": 9.95088408797989,
      "learning_rate": 9.773555548504417e-07,
      "logits/chosen": -0.012451171875,
      "logits/rejected": 0.6015625,
      "logps/chosen": -442.0,
      "logps/rejected": -456.0,
      "loss": 0.1112,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.21875,
      "rewards/rejected": -23.625,
      "step": 6700
    },
    {
      "epoch": 0.857562783564445,
      "grad_norm": 6.414553994580373,
      "learning_rate": 9.76627000477968e-07,
      "logits/chosen": 0.051513671875,
      "logits/rejected": 0.68359375,
      "logps/chosen": -420.0,
      "logps/rejected": -458.0,
      "loss": 0.1088,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.25,
      "rewards/rejected": -23.625,
      "step": 6710
    },
    {
      "epoch": 0.8588408204997124,
      "grad_norm": 11.285209538146022,
      "learning_rate": 9.75900072948533e-07,
      "logits/chosen": 0.041259765625,
      "logits/rejected": 0.64453125,
      "logps/chosen": -440.0,
      "logps/rejected": -476.0,
      "loss": 0.1287,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -17.375,
      "rewards/margins": 7.59375,
      "rewards/rejected": -24.875,
      "step": 6720
    },
    {
      "epoch": 0.8601188574349798,
      "grad_norm": 5.20729880547949,
      "learning_rate": 9.751747662166388e-07,
      "logits/chosen": 0.19140625,
      "logits/rejected": 0.61328125,
      "logps/chosen": -438.0,
      "logps/rejected": -498.0,
      "loss": 0.0957,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.125,
      "rewards/margins": 8.125,
      "rewards/rejected": -25.25,
      "step": 6730
    },
    {
      "epoch": 0.8613968943702472,
      "grad_norm": 4.184508787136277,
      "learning_rate": 9.744510742681917e-07,
      "logits/chosen": 0.07861328125,
      "logits/rejected": 0.60546875,
      "logps/chosen": -460.0,
      "logps/rejected": -504.0,
      "loss": 0.0975,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.75,
      "rewards/margins": 8.4375,
      "rewards/rejected": -27.25,
      "step": 6740
    },
    {
      "epoch": 0.8626749313055148,
      "grad_norm": 8.216170853266908,
      "learning_rate": 9.737289911202953e-07,
      "logits/chosen": 0.1474609375,
      "logits/rejected": 0.7578125,
      "logps/chosen": -432.0,
      "logps/rejected": -472.0,
      "loss": 0.1201,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.125,
      "rewards/margins": 7.90625,
      "rewards/rejected": -26.0,
      "step": 6750
    },
    {
      "epoch": 0.8639529682407822,
      "grad_norm": 10.075243151113453,
      "learning_rate": 9.730085108210398e-07,
      "logits/chosen": -0.0084228515625,
      "logits/rejected": 0.67578125,
      "logps/chosen": -450.0,
      "logps/rejected": -474.0,
      "loss": 0.0829,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.90625,
      "rewards/rejected": -24.625,
      "step": 6760
    },
    {
      "epoch": 0.8652310051760496,
      "grad_norm": 9.798913767948358,
      "learning_rate": 9.72289627449298e-07,
      "logits/chosen": 0.054931640625,
      "logits/rejected": 0.60546875,
      "logps/chosen": -450.0,
      "logps/rejected": -488.0,
      "loss": 0.1199,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -17.5,
      "rewards/margins": 7.09375,
      "rewards/rejected": -24.625,
      "step": 6770
    },
    {
      "epoch": 0.866509042111317,
      "grad_norm": 6.864277996546673,
      "learning_rate": 9.715723351145206e-07,
      "logits/chosen": 0.0400390625,
      "logits/rejected": 0.7734375,
      "logps/chosen": -420.0,
      "logps/rejected": -460.0,
      "loss": 0.1188,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -15.75,
      "rewards/margins": 8.0,
      "rewards/rejected": -23.75,
      "step": 6780
    },
    {
      "epoch": 0.8677870790465845,
      "grad_norm": 6.225476890625224,
      "learning_rate": 9.70856627956532e-07,
      "logits/chosen": -0.00927734375,
      "logits/rejected": 0.70703125,
      "logps/chosen": -418.0,
      "logps/rejected": -436.0,
      "loss": 0.1292,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -14.8125,
      "rewards/margins": 7.5625,
      "rewards/rejected": -22.375,
      "step": 6790
    },
    {
      "epoch": 0.8690651159818519,
      "grad_norm": 8.372812635653286,
      "learning_rate": 9.701425001453318e-07,
      "logits/chosen": 0.15625,
      "logits/rejected": 0.76953125,
      "logps/chosen": -408.0,
      "logps/rejected": -446.0,
      "loss": 0.1143,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -15.375,
      "rewards/margins": 7.34375,
      "rewards/rejected": -22.75,
      "step": 6800
    },
    {
      "epoch": 0.8703431529171193,
      "grad_norm": 9.634214493558588,
      "learning_rate": 9.694299458808932e-07,
      "logits/chosen": 0.037841796875,
      "logits/rejected": 0.6640625,
      "logps/chosen": -436.0,
      "logps/rejected": -484.0,
      "loss": 0.1008,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.625,
      "rewards/margins": 7.90625,
      "rewards/rejected": -24.5,
      "step": 6810
    },
    {
      "epoch": 0.8716211898523867,
      "grad_norm": 4.42261402353327,
      "learning_rate": 9.687189593929655e-07,
      "logits/chosen": 0.09130859375,
      "logits/rejected": 0.74609375,
      "logps/chosen": -416.0,
      "logps/rejected": -456.0,
      "loss": 0.0911,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.5625,
      "rewards/rejected": -23.625,
      "step": 6820
    },
    {
      "epoch": 0.8728992267876542,
      "grad_norm": 6.541017366120674,
      "learning_rate": 9.680095349408789e-07,
      "logits/chosen": -0.18359375,
      "logits/rejected": 0.5625,
      "logps/chosen": -444.0,
      "logps/rejected": -490.0,
      "loss": 0.1013,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.75,
      "rewards/margins": 8.5,
      "rewards/rejected": -25.25,
      "step": 6830
    },
    {
      "epoch": 0.8741772637229216,
      "grad_norm": 6.706190912019408,
      "learning_rate": 9.673016668133487e-07,
      "logits/chosen": -0.060546875,
      "logits/rejected": 0.65625,
      "logps/chosen": -468.0,
      "logps/rejected": -498.0,
      "loss": 0.1212,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -18.625,
      "rewards/margins": 7.375,
      "rewards/rejected": -26.0,
      "step": 6840
    },
    {
      "epoch": 0.875455300658189,
      "grad_norm": 10.97471275381447,
      "learning_rate": 9.66595349328283e-07,
      "logits/chosen": 0.01708984375,
      "logits/rejected": 0.71484375,
      "logps/chosen": -462.0,
      "logps/rejected": -504.0,
      "loss": 0.0957,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.875,
      "rewards/margins": 8.4375,
      "rewards/rejected": -27.25,
      "step": 6850
    },
    {
      "epoch": 0.8767333375934564,
      "grad_norm": 4.966281697302952,
      "learning_rate": 9.658905768325902e-07,
      "logits/chosen": 0.05810546875,
      "logits/rejected": 0.69921875,
      "logps/chosen": -446.0,
      "logps/rejected": -488.0,
      "loss": 0.1189,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -18.25,
      "rewards/margins": 6.96875,
      "rewards/rejected": -25.25,
      "step": 6860
    },
    {
      "epoch": 0.8780113745287239,
      "grad_norm": 7.836194570098642,
      "learning_rate": 9.651873437019902e-07,
      "logits/chosen": 0.036376953125,
      "logits/rejected": 0.61328125,
      "logps/chosen": -454.0,
      "logps/rejected": -492.0,
      "loss": 0.0931,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.5,
      "rewards/margins": 7.59375,
      "rewards/rejected": -25.0,
      "step": 6870
    },
    {
      "epoch": 0.8792894114639913,
      "grad_norm": 6.555647627901188,
      "learning_rate": 9.644856443408243e-07,
      "logits/chosen": 0.0625,
      "logits/rejected": 0.60546875,
      "logps/chosen": -402.0,
      "logps/rejected": -470.0,
      "loss": 0.0953,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -14.875,
      "rewards/margins": 7.75,
      "rewards/rejected": -22.625,
      "step": 6880
    },
    {
      "epoch": 0.8805674483992587,
      "grad_norm": 10.872330644982556,
      "learning_rate": 9.637854731818697e-07,
      "logits/chosen": -0.0120849609375,
      "logits/rejected": 0.5625,
      "logps/chosen": -422.0,
      "logps/rejected": -470.0,
      "loss": 0.0984,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.25,
      "rewards/margins": 7.875,
      "rewards/rejected": -23.125,
      "step": 6890
    },
    {
      "epoch": 0.8818454853345261,
      "grad_norm": 8.61972028460661,
      "learning_rate": 9.630868246861536e-07,
      "logits/chosen": -0.2158203125,
      "logits/rejected": 0.361328125,
      "logps/chosen": -430.0,
      "logps/rejected": -482.0,
      "loss": 0.1224,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.0,
      "rewards/margins": 8.5625,
      "rewards/rejected": -23.5,
      "step": 6900
    },
    {
      "epoch": 0.8831235222697936,
      "grad_norm": 10.90773699253078,
      "learning_rate": 9.623896933427685e-07,
      "logits/chosen": -0.31640625,
      "logits/rejected": 0.318359375,
      "logps/chosen": -416.0,
      "logps/rejected": -470.0,
      "loss": 0.1036,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -13.5625,
      "rewards/margins": 8.125,
      "rewards/rejected": -21.625,
      "step": 6910
    },
    {
      "epoch": 0.884401559205061,
      "grad_norm": 10.164457700039367,
      "learning_rate": 9.6169407366869e-07,
      "logits/chosen": -0.061767578125,
      "logits/rejected": 0.6328125,
      "logps/chosen": -418.0,
      "logps/rejected": -450.0,
      "loss": 0.1568,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -15.4375,
      "rewards/margins": 7.40625,
      "rewards/rejected": -22.875,
      "step": 6920
    },
    {
      "epoch": 0.8856795961403284,
      "grad_norm": 8.305903967105959,
      "learning_rate": 9.609999602085963e-07,
      "logits/chosen": -0.042236328125,
      "logits/rejected": 0.416015625,
      "logps/chosen": -416.0,
      "logps/rejected": -454.0,
      "loss": 0.1287,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.1875,
      "rewards/margins": 7.4375,
      "rewards/rejected": -22.625,
      "step": 6930
    },
    {
      "epoch": 0.8869576330755958,
      "grad_norm": 9.82289264164415,
      "learning_rate": 9.603073475346872e-07,
      "logits/chosen": -0.11328125,
      "logits/rejected": 0.4921875,
      "logps/chosen": -436.0,
      "logps/rejected": -468.0,
      "loss": 0.1119,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.5625,
      "rewards/margins": 7.5,
      "rewards/rejected": -23.0,
      "step": 6940
    },
    {
      "epoch": 0.8882356700108633,
      "grad_norm": 9.960268874098912,
      "learning_rate": 9.596162302465074e-07,
      "logits/chosen": -0.1416015625,
      "logits/rejected": 0.53125,
      "logps/chosen": -424.0,
      "logps/rejected": -476.0,
      "loss": 0.1013,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.6875,
      "rewards/margins": 7.5625,
      "rewards/rejected": -22.25,
      "step": 6950
    },
    {
      "epoch": 0.8895137069461307,
      "grad_norm": 4.879985540359983,
      "learning_rate": 9.589266029707683e-07,
      "logits/chosen": -0.09130859375,
      "logits/rejected": 0.6953125,
      "logps/chosen": -414.0,
      "logps/rejected": -440.0,
      "loss": 0.1022,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -14.125,
      "rewards/margins": 8.0625,
      "rewards/rejected": -22.25,
      "step": 6960
    },
    {
      "epoch": 0.8907917438813981,
      "grad_norm": 8.118259245442013,
      "learning_rate": 9.582384603611731e-07,
      "logits/chosen": -0.0732421875,
      "logits/rejected": 0.51171875,
      "logps/chosen": -414.0,
      "logps/rejected": -458.0,
      "loss": 0.1131,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -15.4375,
      "rewards/margins": 7.1875,
      "rewards/rejected": -22.625,
      "step": 6970
    },
    {
      "epoch": 0.8920697808166655,
      "grad_norm": 8.492182993068848,
      "learning_rate": 9.575517970982428e-07,
      "logits/chosen": -0.06884765625,
      "logits/rejected": 0.5546875,
      "logps/chosen": -444.0,
      "logps/rejected": -456.0,
      "loss": 0.1081,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.1875,
      "rewards/margins": 7.625,
      "rewards/rejected": -22.75,
      "step": 6980
    },
    {
      "epoch": 0.8933478177519331,
      "grad_norm": 10.95848429533346,
      "learning_rate": 9.568666078891436e-07,
      "logits/chosen": 0.04638671875,
      "logits/rejected": 0.58984375,
      "logps/chosen": -436.0,
      "logps/rejected": -488.0,
      "loss": 0.1468,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.71875,
      "rewards/rejected": -23.875,
      "step": 6990
    },
    {
      "epoch": 0.8946258546872005,
      "grad_norm": 6.546959428719152,
      "learning_rate": 9.561828874675149e-07,
      "logits/chosen": -0.0771484375,
      "logits/rejected": 0.59765625,
      "logps/chosen": -422.0,
      "logps/rejected": -448.0,
      "loss": 0.0907,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.125,
      "rewards/margins": 7.53125,
      "rewards/rejected": -22.75,
      "step": 7000
    },
    {
      "epoch": 0.8959038916224679,
      "grad_norm": 7.456597930852643,
      "learning_rate": 9.555006305933e-07,
      "logits/chosen": -0.058837890625,
      "logits/rejected": 0.60546875,
      "logps/chosen": -418.0,
      "logps/rejected": -456.0,
      "loss": 0.1105,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.75,
      "rewards/margins": 7.5,
      "rewards/rejected": -22.25,
      "step": 7010
    },
    {
      "epoch": 0.8971819285577353,
      "grad_norm": 9.570278891382616,
      "learning_rate": 9.548198320525771e-07,
      "logits/chosen": -0.1162109375,
      "logits/rejected": 0.5390625,
      "logps/chosen": -426.0,
      "logps/rejected": -466.0,
      "loss": 0.1309,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.625,
      "rewards/margins": 7.78125,
      "rewards/rejected": -24.375,
      "step": 7020
    },
    {
      "epoch": 0.8984599654930028,
      "grad_norm": 8.918544196108895,
      "learning_rate": 9.54140486657392e-07,
      "logits/chosen": -0.25390625,
      "logits/rejected": 0.365234375,
      "logps/chosen": -464.0,
      "logps/rejected": -506.0,
      "loss": 0.1142,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.5,
      "rewards/margins": 8.6875,
      "rewards/rejected": -25.125,
      "step": 7030
    },
    {
      "epoch": 0.8997380024282702,
      "grad_norm": 6.002151857216528,
      "learning_rate": 9.534625892455922e-07,
      "logits/chosen": -0.05517578125,
      "logits/rejected": 0.5625,
      "logps/chosen": -422.0,
      "logps/rejected": -468.0,
      "loss": 0.0916,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.5625,
      "rewards/rejected": -23.875,
      "step": 7040
    },
    {
      "epoch": 0.9010160393635376,
      "grad_norm": 8.528358737267954,
      "learning_rate": 9.527861346806618e-07,
      "logits/chosen": -0.14453125,
      "logits/rejected": 0.494140625,
      "logps/chosen": -434.0,
      "logps/rejected": -466.0,
      "loss": 0.1002,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.625,
      "rewards/margins": 8.3125,
      "rewards/rejected": -23.875,
      "step": 7050
    },
    {
      "epoch": 0.902294076298805,
      "grad_norm": 6.497813792775296,
      "learning_rate": 9.521111178515582e-07,
      "logits/chosen": -0.1826171875,
      "logits/rejected": 0.310546875,
      "logps/chosen": -444.0,
      "logps/rejected": -482.0,
      "loss": 0.1188,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.625,
      "rewards/margins": 7.3125,
      "rewards/rejected": -23.875,
      "step": 7060
    },
    {
      "epoch": 0.9035721132340725,
      "grad_norm": 5.881606176308462,
      "learning_rate": 9.514375336725502e-07,
      "logits/chosen": -0.038330078125,
      "logits/rejected": 0.59375,
      "logps/chosen": -454.0,
      "logps/rejected": -492.0,
      "loss": 0.1012,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -18.25,
      "rewards/margins": 6.90625,
      "rewards/rejected": -25.125,
      "step": 7070
    },
    {
      "epoch": 0.9048501501693399,
      "grad_norm": 8.280046843461106,
      "learning_rate": 9.507653770830566e-07,
      "logits/chosen": -0.0732421875,
      "logits/rejected": 0.55078125,
      "logps/chosen": -432.0,
      "logps/rejected": -462.0,
      "loss": 0.1096,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.0,
      "rewards/margins": 7.28125,
      "rewards/rejected": -24.25,
      "step": 7080
    },
    {
      "epoch": 0.9061281871046073,
      "grad_norm": 9.002190778300143,
      "learning_rate": 9.500946430474869e-07,
      "logits/chosen": 0.01904296875,
      "logits/rejected": 0.58203125,
      "logps/chosen": -424.0,
      "logps/rejected": -464.0,
      "loss": 0.1156,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.03125,
      "rewards/rejected": -23.75,
      "step": 7090
    },
    {
      "epoch": 0.9074062240398747,
      "grad_norm": 5.130225293268683,
      "learning_rate": 9.494253265550825e-07,
      "logits/chosen": -0.1943359375,
      "logits/rejected": 0.48046875,
      "logps/chosen": -424.0,
      "logps/rejected": -474.0,
      "loss": 0.1165,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.71875,
      "rewards/rejected": -23.75,
      "step": 7100
    },
    {
      "epoch": 0.9086842609751422,
      "grad_norm": 5.986914076493917,
      "learning_rate": 9.4875742261976e-07,
      "logits/chosen": -0.0020904541015625,
      "logits/rejected": 0.58984375,
      "logps/chosen": -434.0,
      "logps/rejected": -458.0,
      "loss": 0.1179,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.875,
      "rewards/margins": 6.8125,
      "rewards/rejected": -23.75,
      "step": 7110
    },
    {
      "epoch": 0.9099622979104096,
      "grad_norm": 6.121997195117546,
      "learning_rate": 9.480909262799544e-07,
      "logits/chosen": -0.0113525390625,
      "logits/rejected": 0.59375,
      "logps/chosen": -426.0,
      "logps/rejected": -460.0,
      "loss": 0.1187,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.8125,
      "rewards/margins": 7.0,
      "rewards/rejected": -22.875,
      "step": 7120
    },
    {
      "epoch": 0.911240334845677,
      "grad_norm": 10.80883635019568,
      "learning_rate": 9.47425832598465e-07,
      "logits/chosen": 0.000701904296875,
      "logits/rejected": 0.58984375,
      "logps/chosen": -412.0,
      "logps/rejected": -438.0,
      "loss": 0.1126,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.75,
      "rewards/margins": 6.375,
      "rewards/rejected": -21.125,
      "step": 7130
    },
    {
      "epoch": 0.9125183717809444,
      "grad_norm": 6.158059240561595,
      "learning_rate": 9.467621366623017e-07,
      "logits/chosen": -0.11279296875,
      "logits/rejected": 0.5546875,
      "logps/chosen": -428.0,
      "logps/rejected": -484.0,
      "loss": 0.0868,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -15.0625,
      "rewards/margins": 7.6875,
      "rewards/rejected": -22.75,
      "step": 7140
    },
    {
      "epoch": 0.9137964087162119,
      "grad_norm": 10.269742995411553,
      "learning_rate": 9.46099833582532e-07,
      "logits/chosen": -0.1416015625,
      "logits/rejected": 0.51171875,
      "logps/chosen": -424.0,
      "logps/rejected": -482.0,
      "loss": 0.0996,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.75,
      "rewards/margins": 7.875,
      "rewards/rejected": -23.625,
      "step": 7150
    },
    {
      "epoch": 0.9150744456514793,
      "grad_norm": 8.823279834609293,
      "learning_rate": 9.45438918494131e-07,
      "logits/chosen": -0.045654296875,
      "logits/rejected": 0.50390625,
      "logps/chosen": -426.0,
      "logps/rejected": -476.0,
      "loss": 0.1068,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.8125,
      "rewards/margins": 7.5,
      "rewards/rejected": -23.375,
      "step": 7160
    },
    {
      "epoch": 0.9163524825867467,
      "grad_norm": 12.026641298318923,
      "learning_rate": 9.447793865558291e-07,
      "logits/chosen": -0.049560546875,
      "logits/rejected": 0.6796875,
      "logps/chosen": -410.0,
      "logps/rejected": -434.0,
      "loss": 0.1282,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.6875,
      "rewards/margins": 7.4375,
      "rewards/rejected": -22.125,
      "step": 7170
    },
    {
      "epoch": 0.9176305195220141,
      "grad_norm": 8.81495807267748,
      "learning_rate": 9.441212329499659e-07,
      "logits/chosen": 0.0634765625,
      "logits/rejected": 0.64453125,
      "logps/chosen": -408.0,
      "logps/rejected": -434.0,
      "loss": 0.1125,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.6875,
      "rewards/margins": 7.375,
      "rewards/rejected": -22.0,
      "step": 7180
    },
    {
      "epoch": 0.9189085564572816,
      "grad_norm": 8.948321360494242,
      "learning_rate": 9.434644528823399e-07,
      "logits/chosen": -0.0220947265625,
      "logits/rejected": 0.71484375,
      "logps/chosen": -428.0,
      "logps/rejected": -448.0,
      "loss": 0.0913,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.3125,
      "rewards/margins": 7.3125,
      "rewards/rejected": -22.625,
      "step": 7190
    },
    {
      "epoch": 0.920186593392549,
      "grad_norm": 8.952771349522957,
      "learning_rate": 9.428090415820634e-07,
      "logits/chosen": 0.020263671875,
      "logits/rejected": 0.75390625,
      "logps/chosen": -440.0,
      "logps/rejected": -490.0,
      "loss": 0.1244,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.125,
      "rewards/margins": 7.875,
      "rewards/rejected": -25.0,
      "step": 7200
    },
    {
      "epoch": 0.9214646303278164,
      "grad_norm": 8.941615959975683,
      "learning_rate": 9.42154994301416e-07,
      "logits/chosen": -0.1005859375,
      "logits/rejected": 0.73828125,
      "logps/chosen": -442.0,
      "logps/rejected": -460.0,
      "loss": 0.1101,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.90625,
      "rewards/rejected": -24.25,
      "step": 7210
    },
    {
      "epoch": 0.9227426672630838,
      "grad_norm": 9.393355508333574,
      "learning_rate": 9.415023063157008e-07,
      "logits/chosen": -0.06787109375,
      "logits/rejected": 0.640625,
      "logps/chosen": -462.0,
      "logps/rejected": -472.0,
      "loss": 0.1068,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.125,
      "rewards/margins": 8.375,
      "rewards/rejected": -25.375,
      "step": 7220
    },
    {
      "epoch": 0.9240207041983514,
      "grad_norm": 6.448107181156964,
      "learning_rate": 9.408509729231009e-07,
      "logits/chosen": -0.009521484375,
      "logits/rejected": 0.63671875,
      "logps/chosen": -420.0,
      "logps/rejected": -470.0,
      "loss": 0.1014,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.5625,
      "rewards/rejected": -23.875,
      "step": 7230
    },
    {
      "epoch": 0.9252987411336188,
      "grad_norm": 5.408833852990839,
      "learning_rate": 9.402009894445369e-07,
      "logits/chosen": -0.048095703125,
      "logits/rejected": 0.65625,
      "logps/chosen": -428.0,
      "logps/rejected": -450.0,
      "loss": 0.1222,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.0,
      "rewards/margins": 7.78125,
      "rewards/rejected": -23.75,
      "step": 7240
    },
    {
      "epoch": 0.9265767780688862,
      "grad_norm": 11.667824304037062,
      "learning_rate": 9.395523512235255e-07,
      "logits/chosen": -0.01544189453125,
      "logits/rejected": 0.56640625,
      "logps/chosen": -416.0,
      "logps/rejected": -448.0,
      "loss": 0.1304,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -14.9375,
      "rewards/margins": 7.15625,
      "rewards/rejected": -22.125,
      "step": 7250
    },
    {
      "epoch": 0.9278548150041536,
      "grad_norm": 7.805467935783571,
      "learning_rate": 9.389050536260404e-07,
      "logits/chosen": -0.01708984375,
      "logits/rejected": 0.6171875,
      "logps/chosen": -430.0,
      "logps/rejected": -480.0,
      "loss": 0.1222,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.5,
      "rewards/margins": 7.6875,
      "rewards/rejected": -23.25,
      "step": 7260
    },
    {
      "epoch": 0.9291328519394211,
      "grad_norm": 7.720476118312215,
      "learning_rate": 9.382590920403722e-07,
      "logits/chosen": 0.000598907470703125,
      "logits/rejected": 0.6484375,
      "logps/chosen": -416.0,
      "logps/rejected": -442.0,
      "loss": 0.1114,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.6875,
      "rewards/margins": 7.0,
      "rewards/rejected": -21.625,
      "step": 7270
    },
    {
      "epoch": 0.9304108888746885,
      "grad_norm": 11.368906929391327,
      "learning_rate": 9.376144618769908e-07,
      "logits/chosen": -0.0107421875,
      "logits/rejected": 0.71484375,
      "logps/chosen": -442.0,
      "logps/rejected": -484.0,
      "loss": 0.0912,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.375,
      "rewards/margins": 7.96875,
      "rewards/rejected": -25.25,
      "step": 7280
    },
    {
      "epoch": 0.9316889258099559,
      "grad_norm": 9.068458189915283,
      "learning_rate": 9.369711585684086e-07,
      "logits/chosen": 0.08447265625,
      "logits/rejected": 0.84375,
      "logps/chosen": -454.0,
      "logps/rejected": -494.0,
      "loss": 0.0856,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.75,
      "rewards/margins": 8.3125,
      "rewards/rejected": -27.125,
      "step": 7290
    },
    {
      "epoch": 0.9329669627452233,
      "grad_norm": 6.323891471243699,
      "learning_rate": 9.363291775690445e-07,
      "logits/chosen": 0.0272216796875,
      "logits/rejected": 0.65234375,
      "logps/chosen": -448.0,
      "logps/rejected": -504.0,
      "loss": 0.1071,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -16.625,
      "rewards/margins": 8.75,
      "rewards/rejected": -25.375,
      "step": 7300
    },
    {
      "epoch": 0.9342449996804908,
      "grad_norm": 7.001549314135997,
      "learning_rate": 9.356885143550886e-07,
      "logits/chosen": -0.0517578125,
      "logits/rejected": 0.61328125,
      "logps/chosen": -412.0,
      "logps/rejected": -458.0,
      "loss": 0.1145,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.875,
      "rewards/margins": 8.0,
      "rewards/rejected": -22.875,
      "step": 7310
    },
    {
      "epoch": 0.9355230366157582,
      "grad_norm": 7.46309663804639,
      "learning_rate": 9.350491644243688e-07,
      "logits/chosen": 0.048583984375,
      "logits/rejected": 0.61328125,
      "logps/chosen": -400.0,
      "logps/rejected": -440.0,
      "loss": 0.0997,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.0,
      "rewards/margins": 7.53125,
      "rewards/rejected": -22.5,
      "step": 7320
    },
    {
      "epoch": 0.9368010735510256,
      "grad_norm": 8.470826337498329,
      "learning_rate": 9.344111232962179e-07,
      "logits/chosen": 0.057373046875,
      "logits/rejected": 0.6484375,
      "logps/chosen": -422.0,
      "logps/rejected": -476.0,
      "loss": 0.1057,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -17.25,
      "rewards/margins": 7.90625,
      "rewards/rejected": -25.125,
      "step": 7330
    },
    {
      "epoch": 0.938079110486293,
      "grad_norm": 7.339616388581498,
      "learning_rate": 9.337743865113415e-07,
      "logits/chosen": -0.01104736328125,
      "logits/rejected": 0.5546875,
      "logps/chosen": -460.0,
      "logps/rejected": -512.0,
      "loss": 0.1089,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.75,
      "rewards/margins": 7.6875,
      "rewards/rejected": -25.5,
      "step": 7340
    },
    {
      "epoch": 0.9393571474215605,
      "grad_norm": 9.282714790703364,
      "learning_rate": 9.331389496316868e-07,
      "logits/chosen": -0.099609375,
      "logits/rejected": 0.57421875,
      "logps/chosen": -436.0,
      "logps/rejected": -484.0,
      "loss": 0.1076,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.375,
      "rewards/margins": 8.5625,
      "rewards/rejected": -23.875,
      "step": 7350
    },
    {
      "epoch": 0.9406351843568279,
      "grad_norm": 10.196961717961075,
      "learning_rate": 9.325048082403138e-07,
      "logits/chosen": -0.1875,
      "logits/rejected": 0.55078125,
      "logps/chosen": -460.0,
      "logps/rejected": -488.0,
      "loss": 0.1257,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.625,
      "rewards/margins": 8.625,
      "rewards/rejected": -25.25,
      "step": 7360
    },
    {
      "epoch": 0.9419132212920953,
      "grad_norm": 8.951697878424126,
      "learning_rate": 9.318719579412648e-07,
      "logits/chosen": -0.1923828125,
      "logits/rejected": 0.478515625,
      "logps/chosen": -464.0,
      "logps/rejected": -484.0,
      "loss": 0.1154,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.125,
      "rewards/margins": 7.8125,
      "rewards/rejected": -24.875,
      "step": 7370
    },
    {
      "epoch": 0.9431912582273627,
      "grad_norm": 2.2186441410732516,
      "learning_rate": 9.312403943594374e-07,
      "logits/chosen": 0.0859375,
      "logits/rejected": 0.515625,
      "logps/chosen": -400.0,
      "logps/rejected": -448.0,
      "loss": 0.1117,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -14.9375,
      "rewards/margins": 7.25,
      "rewards/rejected": -22.125,
      "step": 7380
    },
    {
      "epoch": 0.9444692951626302,
      "grad_norm": 6.2139953624815725,
      "learning_rate": 9.306101131404582e-07,
      "logits/chosen": -0.10595703125,
      "logits/rejected": 0.41015625,
      "logps/chosen": -402.0,
      "logps/rejected": -458.0,
      "loss": 0.1044,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.375,
      "rewards/margins": 7.59375,
      "rewards/rejected": -22.0,
      "step": 7390
    },
    {
      "epoch": 0.9457473320978976,
      "grad_norm": 8.510120537820876,
      "learning_rate": 9.299811099505542e-07,
      "logits/chosen": -0.2255859375,
      "logits/rejected": 0.384765625,
      "logps/chosen": -432.0,
      "logps/rejected": -468.0,
      "loss": 0.0964,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.875,
      "rewards/margins": 7.6875,
      "rewards/rejected": -22.5,
      "step": 7400
    },
    {
      "epoch": 0.947025369033165,
      "grad_norm": 9.352736564058162,
      "learning_rate": 9.293533804764305e-07,
      "logits/chosen": -0.158203125,
      "logits/rejected": 0.44140625,
      "logps/chosen": -446.0,
      "logps/rejected": -498.0,
      "loss": 0.0975,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -16.875,
      "rewards/margins": 8.5625,
      "rewards/rejected": -25.5,
      "step": 7410
    },
    {
      "epoch": 0.9483034059684324,
      "grad_norm": 5.8082436663849135,
      "learning_rate": 9.28726920425144e-07,
      "logits/chosen": -0.035400390625,
      "logits/rejected": 0.5078125,
      "logps/chosen": -446.0,
      "logps/rejected": -496.0,
      "loss": 0.0971,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -18.0,
      "rewards/margins": 7.34375,
      "rewards/rejected": -25.25,
      "step": 7420
    },
    {
      "epoch": 0.9495814429037,
      "grad_norm": 9.106968867449307,
      "learning_rate": 9.281017255239815e-07,
      "logits/chosen": -0.12890625,
      "logits/rejected": 0.43359375,
      "logps/chosen": -420.0,
      "logps/rejected": -484.0,
      "loss": 0.108,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.3125,
      "rewards/margins": 8.0,
      "rewards/rejected": -23.375,
      "step": 7430
    },
    {
      "epoch": 0.9508594798389673,
      "grad_norm": 6.485678347994726,
      "learning_rate": 9.274777915203365e-07,
      "logits/chosen": -0.16015625,
      "logits/rejected": 0.3671875,
      "logps/chosen": -404.0,
      "logps/rejected": -478.0,
      "loss": 0.1007,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -13.9375,
      "rewards/margins": 7.25,
      "rewards/rejected": -21.125,
      "step": 7440
    },
    {
      "epoch": 0.9521375167742347,
      "grad_norm": 9.010199691171296,
      "learning_rate": 9.268551141815875e-07,
      "logits/chosen": -0.25390625,
      "logits/rejected": 0.24609375,
      "logps/chosen": -408.0,
      "logps/rejected": -448.0,
      "loss": 0.1149,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -12.875,
      "rewards/margins": 6.84375,
      "rewards/rejected": -19.75,
      "step": 7450
    },
    {
      "epoch": 0.9534155537095022,
      "grad_norm": 7.434664605229494,
      "learning_rate": 9.262336892949784e-07,
      "logits/chosen": -0.1484375,
      "logits/rejected": 0.43359375,
      "logps/chosen": -392.0,
      "logps/rejected": -422.0,
      "loss": 0.1175,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -12.25,
      "rewards/margins": 7.375,
      "rewards/rejected": -19.625,
      "step": 7460
    },
    {
      "epoch": 0.9546935906447697,
      "grad_norm": 7.464758866504516,
      "learning_rate": 9.256135126674977e-07,
      "logits/chosen": 0.00506591796875,
      "logits/rejected": 0.60546875,
      "logps/chosen": -388.0,
      "logps/rejected": -434.0,
      "loss": 0.1126,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.4375,
      "rewards/margins": 7.15625,
      "rewards/rejected": -21.625,
      "step": 7470
    },
    {
      "epoch": 0.9559716275800371,
      "grad_norm": 8.62332833553537,
      "learning_rate": 9.249945801257605e-07,
      "logits/chosen": -0.022216796875,
      "logits/rejected": 0.453125,
      "logps/chosen": -418.0,
      "logps/rejected": -464.0,
      "loss": 0.102,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.125,
      "rewards/margins": 7.84375,
      "rewards/rejected": -23.0,
      "step": 7480
    },
    {
      "epoch": 0.9572496645153045,
      "grad_norm": 6.493891708417804,
      "learning_rate": 9.243768875158902e-07,
      "logits/chosen": -0.1630859375,
      "logits/rejected": 0.59765625,
      "logps/chosen": -426.0,
      "logps/rejected": -450.0,
      "loss": 0.1018,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.25,
      "rewards/margins": 7.8125,
      "rewards/rejected": -23.0,
      "step": 7490
    },
    {
      "epoch": 0.9585277014505719,
      "grad_norm": 6.8899899324535205,
      "learning_rate": 9.23760430703401e-07,
      "logits/chosen": -0.08203125,
      "logits/rejected": 0.57421875,
      "logps/chosen": -422.0,
      "logps/rejected": -450.0,
      "loss": 0.1132,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -15.5,
      "rewards/margins": 7.875,
      "rewards/rejected": -23.375,
      "step": 7500
    },
    {
      "epoch": 0.9598057383858394,
      "grad_norm": 18.979022793688685,
      "learning_rate": 9.231452055730832e-07,
      "logits/chosen": -0.03564453125,
      "logits/rejected": 0.66015625,
      "logps/chosen": -444.0,
      "logps/rejected": -462.0,
      "loss": 0.13,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.25,
      "rewards/margins": 8.0625,
      "rewards/rejected": -24.25,
      "step": 7510
    },
    {
      "epoch": 0.9610837753211068,
      "grad_norm": 5.948173312932866,
      "learning_rate": 9.225312080288851e-07,
      "logits/chosen": -0.181640625,
      "logits/rejected": 0.49609375,
      "logps/chosen": -396.0,
      "logps/rejected": -450.0,
      "loss": 0.1023,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.0,
      "rewards/margins": 7.65625,
      "rewards/rejected": -21.625,
      "step": 7520
    },
    {
      "epoch": 0.9623618122563742,
      "grad_norm": 7.368059830034939,
      "learning_rate": 9.219184339938013e-07,
      "logits/chosen": -0.07763671875,
      "logits/rejected": 0.419921875,
      "logps/chosen": -432.0,
      "logps/rejected": -484.0,
      "loss": 0.1223,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.875,
      "rewards/margins": 7.75,
      "rewards/rejected": -23.625,
      "step": 7530
    },
    {
      "epoch": 0.9636398491916416,
      "grad_norm": 4.214521877223137,
      "learning_rate": 9.213068794097574e-07,
      "logits/chosen": -0.1982421875,
      "logits/rejected": 0.50390625,
      "logps/chosen": -430.0,
      "logps/rejected": -478.0,
      "loss": 0.0889,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -15.125,
      "rewards/margins": 8.75,
      "rewards/rejected": -23.875,
      "step": 7540
    },
    {
      "epoch": 0.9649178861269091,
      "grad_norm": 10.013697118243432,
      "learning_rate": 9.206965402374975e-07,
      "logits/chosen": -0.1630859375,
      "logits/rejected": 0.490234375,
      "logps/chosen": -418.0,
      "logps/rejected": -468.0,
      "loss": 0.1009,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.8125,
      "rewards/margins": 7.9375,
      "rewards/rejected": -22.75,
      "step": 7550
    },
    {
      "epoch": 0.9661959230621765,
      "grad_norm": 8.52651211310212,
      "learning_rate": 9.200874124564723e-07,
      "logits/chosen": -0.279296875,
      "logits/rejected": 0.490234375,
      "logps/chosen": -456.0,
      "logps/rejected": -470.0,
      "loss": 0.1156,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.375,
      "rewards/margins": 8.5,
      "rewards/rejected": -24.875,
      "step": 7560
    },
    {
      "epoch": 0.9674739599974439,
      "grad_norm": 7.435778602209648,
      "learning_rate": 9.194794920647274e-07,
      "logits/chosen": -0.232421875,
      "logits/rejected": 0.494140625,
      "logps/chosen": -442.0,
      "logps/rejected": -472.0,
      "loss": 0.0757,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -15.3125,
      "rewards/margins": 8.5,
      "rewards/rejected": -23.75,
      "step": 7570
    },
    {
      "epoch": 0.9687519969327113,
      "grad_norm": 7.478995059927268,
      "learning_rate": 9.188727750787932e-07,
      "logits/chosen": -0.11328125,
      "logits/rejected": 0.55078125,
      "logps/chosen": -438.0,
      "logps/rejected": -468.0,
      "loss": 0.0899,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.6875,
      "rewards/margins": 7.84375,
      "rewards/rejected": -23.5,
      "step": 7580
    },
    {
      "epoch": 0.9700300338679788,
      "grad_norm": 9.539750417393941,
      "learning_rate": 9.182672575335757e-07,
      "logits/chosen": -0.302734375,
      "logits/rejected": 0.39453125,
      "logps/chosen": -416.0,
      "logps/rejected": -456.0,
      "loss": 0.0981,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -13.8125,
      "rewards/margins": 8.125,
      "rewards/rejected": -22.0,
      "step": 7590
    },
    {
      "epoch": 0.9713080708032462,
      "grad_norm": 10.327618302673422,
      "learning_rate": 9.176629354822469e-07,
      "logits/chosen": -0.251953125,
      "logits/rejected": 0.3125,
      "logps/chosen": -400.0,
      "logps/rejected": -438.0,
      "loss": 0.1231,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -13.25,
      "rewards/margins": 7.59375,
      "rewards/rejected": -20.875,
      "step": 7600
    },
    {
      "epoch": 0.9725861077385136,
      "grad_norm": 11.380098851101266,
      "learning_rate": 9.170598049961371e-07,
      "logits/chosen": -0.283203125,
      "logits/rejected": 0.326171875,
      "logps/chosen": -434.0,
      "logps/rejected": -472.0,
      "loss": 0.1156,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.75,
      "rewards/margins": 7.46875,
      "rewards/rejected": -22.25,
      "step": 7610
    },
    {
      "epoch": 0.973864144673781,
      "grad_norm": 4.917401744334269,
      "learning_rate": 9.164578621646276e-07,
      "logits/chosen": -0.23828125,
      "logits/rejected": 0.38671875,
      "logps/chosen": -412.0,
      "logps/rejected": -448.0,
      "loss": 0.097,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -14.0,
      "rewards/margins": 7.9375,
      "rewards/rejected": -21.875,
      "step": 7620
    },
    {
      "epoch": 0.9751421816090485,
      "grad_norm": 4.785038343686194,
      "learning_rate": 9.15857103095044e-07,
      "logits/chosen": -0.18359375,
      "logits/rejected": 0.490234375,
      "logps/chosen": -432.0,
      "logps/rejected": -420.0,
      "loss": 0.1228,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -13.1875,
      "rewards/margins": 7.3125,
      "rewards/rejected": -20.5,
      "step": 7630
    },
    {
      "epoch": 0.9764202185443159,
      "grad_norm": 8.511042671529292,
      "learning_rate": 9.15257523912551e-07,
      "logits/chosen": -0.388671875,
      "logits/rejected": 0.369140625,
      "logps/chosen": -416.0,
      "logps/rejected": -460.0,
      "loss": 0.1519,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.875,
      "rewards/margins": 6.5,
      "rewards/rejected": -21.375,
      "step": 7640
    },
    {
      "epoch": 0.9776982554795833,
      "grad_norm": 13.271399291296076,
      "learning_rate": 9.146591207600472e-07,
      "logits/chosen": -0.11474609375,
      "logits/rejected": 0.54296875,
      "logps/chosen": -418.0,
      "logps/rejected": -458.0,
      "loss": 0.1169,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.4375,
      "rewards/margins": 7.28125,
      "rewards/rejected": -22.75,
      "step": 7650
    },
    {
      "epoch": 0.9789762924148507,
      "grad_norm": 6.70049793111946,
      "learning_rate": 9.140618897980601e-07,
      "logits/chosen": -0.318359375,
      "logits/rejected": 0.474609375,
      "logps/chosen": -422.0,
      "logps/rejected": -460.0,
      "loss": 0.1061,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.75,
      "rewards/margins": 8.375,
      "rewards/rejected": -23.125,
      "step": 7660
    },
    {
      "epoch": 0.9802543293501182,
      "grad_norm": 10.554381804960974,
      "learning_rate": 9.134658272046442e-07,
      "logits/chosen": -0.1806640625,
      "logits/rejected": 0.45703125,
      "logps/chosen": -442.0,
      "logps/rejected": -488.0,
      "loss": 0.0978,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.90625,
      "rewards/rejected": -24.25,
      "step": 7670
    },
    {
      "epoch": 0.9815323662853856,
      "grad_norm": 10.91244889561177,
      "learning_rate": 9.128709291752768e-07,
      "logits/chosen": -0.2119140625,
      "logits/rejected": 0.345703125,
      "logps/chosen": -448.0,
      "logps/rejected": -486.0,
      "loss": 0.0975,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.96875,
      "rewards/rejected": -24.25,
      "step": 7680
    },
    {
      "epoch": 0.982810403220653,
      "grad_norm": 7.077021673543942,
      "learning_rate": 9.122771919227568e-07,
      "logits/chosen": -0.08935546875,
      "logits/rejected": 0.5546875,
      "logps/chosen": -436.0,
      "logps/rejected": -488.0,
      "loss": 0.0954,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.25,
      "rewards/margins": 7.96875,
      "rewards/rejected": -25.25,
      "step": 7690
    },
    {
      "epoch": 0.9840884401559205,
      "grad_norm": 4.9783653855832775,
      "learning_rate": 9.116846116771035e-07,
      "logits/chosen": -0.298828125,
      "logits/rejected": 0.5859375,
      "logps/chosen": -456.0,
      "logps/rejected": -492.0,
      "loss": 0.0808,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.875,
      "rewards/margins": 9.3125,
      "rewards/rejected": -26.125,
      "step": 7700
    },
    {
      "epoch": 0.985366477091188,
      "grad_norm": 10.271107139540888,
      "learning_rate": 9.110931846854553e-07,
      "logits/chosen": -0.337890625,
      "logits/rejected": 0.28125,
      "logps/chosen": -448.0,
      "logps/rejected": -486.0,
      "loss": 0.103,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.0,
      "rewards/margins": 8.3125,
      "rewards/rejected": -24.375,
      "step": 7710
    },
    {
      "epoch": 0.9866445140264554,
      "grad_norm": 6.661860237655354,
      "learning_rate": 9.105029072119708e-07,
      "logits/chosen": -0.23828125,
      "logits/rejected": 0.39453125,
      "logps/chosen": -428.0,
      "logps/rejected": -460.0,
      "loss": 0.11,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.4375,
      "rewards/margins": 7.4375,
      "rewards/rejected": -22.875,
      "step": 7720
    },
    {
      "epoch": 0.9879225509617228,
      "grad_norm": 7.896883146500513,
      "learning_rate": 9.099137755377291e-07,
      "logits/chosen": -0.29296875,
      "logits/rejected": 0.298828125,
      "logps/chosen": -406.0,
      "logps/rejected": -448.0,
      "loss": 0.1118,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.375,
      "rewards/margins": 7.5625,
      "rewards/rejected": -22.0,
      "step": 7730
    },
    {
      "epoch": 0.9892005878969902,
      "grad_norm": 8.885348335106164,
      "learning_rate": 9.093257859606311e-07,
      "logits/chosen": -0.2099609375,
      "logits/rejected": 0.38671875,
      "logps/chosen": -426.0,
      "logps/rejected": -454.0,
      "loss": 0.0981,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.375,
      "rewards/margins": 7.34375,
      "rewards/rejected": -22.625,
      "step": 7740
    },
    {
      "epoch": 0.9904786248322577,
      "grad_norm": 6.353666514764862,
      "learning_rate": 9.087389347953037e-07,
      "logits/chosen": -0.265625,
      "logits/rejected": 0.55078125,
      "logps/chosen": -432.0,
      "logps/rejected": -454.0,
      "loss": 0.0935,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.125,
      "rewards/margins": 8.25,
      "rewards/rejected": -23.375,
      "step": 7750
    },
    {
      "epoch": 0.9917566617675251,
      "grad_norm": 6.528331775751688,
      "learning_rate": 9.081532183729995e-07,
      "logits/chosen": -0.1826171875,
      "logits/rejected": 0.439453125,
      "logps/chosen": -418.0,
      "logps/rejected": -470.0,
      "loss": 0.1147,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -15.3125,
      "rewards/margins": 7.96875,
      "rewards/rejected": -23.25,
      "step": 7760
    },
    {
      "epoch": 0.9930346987027925,
      "grad_norm": 4.399089352382179,
      "learning_rate": 9.075686330415037e-07,
      "logits/chosen": -0.2158203125,
      "logits/rejected": 0.39453125,
      "logps/chosen": -424.0,
      "logps/rejected": -452.0,
      "loss": 0.1014,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.8125,
      "rewards/margins": 7.65625,
      "rewards/rejected": -22.5,
      "step": 7770
    },
    {
      "epoch": 0.9943127356380599,
      "grad_norm": 10.240218876324196,
      "learning_rate": 9.069851751650364e-07,
      "logits/chosen": -0.1826171875,
      "logits/rejected": 0.2578125,
      "logps/chosen": -418.0,
      "logps/rejected": -470.0,
      "loss": 0.1181,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.8125,
      "rewards/margins": 7.40625,
      "rewards/rejected": -22.25,
      "step": 7780
    },
    {
      "epoch": 0.9955907725733274,
      "grad_norm": 11.195942104539192,
      "learning_rate": 9.064028411241582e-07,
      "logits/chosen": -0.236328125,
      "logits/rejected": 0.486328125,
      "logps/chosen": -398.0,
      "logps/rejected": -434.0,
      "loss": 0.0889,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.375,
      "rewards/margins": 7.78125,
      "rewards/rejected": -22.125,
      "step": 7790
    },
    {
      "epoch": 0.9968688095085948,
      "grad_norm": 9.161096336694266,
      "learning_rate": 9.058216273156764e-07,
      "logits/chosen": -0.19921875,
      "logits/rejected": 0.4375,
      "logps/chosen": -416.0,
      "logps/rejected": -466.0,
      "loss": 0.0839,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.6875,
      "rewards/margins": 7.71875,
      "rewards/rejected": -22.375,
      "step": 7800
    },
    {
      "epoch": 0.9981468464438622,
      "grad_norm": 12.968283830059248,
      "learning_rate": 9.052415301525511e-07,
      "logits/chosen": -0.2265625,
      "logits/rejected": 0.322265625,
      "logps/chosen": -454.0,
      "logps/rejected": -512.0,
      "loss": 0.0976,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.9375,
      "rewards/margins": 8.5625,
      "rewards/rejected": -24.5,
      "step": 7810
    },
    {
      "epoch": 0.9994248833791296,
      "grad_norm": 8.189437859663597,
      "learning_rate": 9.046625460638012e-07,
      "logits/chosen": -0.0211181640625,
      "logits/rejected": 0.64453125,
      "logps/chosen": -426.0,
      "logps/rejected": -470.0,
      "loss": 0.0786,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -16.5,
      "rewards/margins": 9.1875,
      "rewards/rejected": -25.75,
      "step": 7820
    },
    {
      "epoch": 1.0007029203143971,
      "grad_norm": 6.861234102872688,
      "learning_rate": 9.040846714944138e-07,
      "logits/chosen": -0.051025390625,
      "logits/rejected": 0.482421875,
      "logps/chosen": -452.0,
      "logps/rejected": -496.0,
      "loss": 0.0947,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.75,
      "rewards/margins": 8.625,
      "rewards/rejected": -26.25,
      "step": 7830
    },
    {
      "epoch": 1.0019809572496645,
      "grad_norm": 5.414544333230439,
      "learning_rate": 9.035079029052513e-07,
      "logits/chosen": -0.1875,
      "logits/rejected": 0.50390625,
      "logps/chosen": -428.0,
      "logps/rejected": -492.0,
      "loss": 0.0735,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -17.25,
      "rewards/margins": 8.8125,
      "rewards/rejected": -26.0,
      "step": 7840
    },
    {
      "epoch": 1.003258994184932,
      "grad_norm": 5.789353786179957,
      "learning_rate": 9.029322367729605e-07,
      "logits/chosen": -0.1611328125,
      "logits/rejected": 0.4765625,
      "logps/chosen": -430.0,
      "logps/rejected": -476.0,
      "loss": 0.0595,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.125,
      "rewards/margins": 8.0625,
      "rewards/rejected": -25.125,
      "step": 7850
    },
    {
      "epoch": 1.0045370311201993,
      "grad_norm": 7.5927307643665305,
      "learning_rate": 9.02357669589883e-07,
      "logits/chosen": -0.2158203125,
      "logits/rejected": 0.37890625,
      "logps/chosen": -458.0,
      "logps/rejected": -512.0,
      "loss": 0.0437,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.25,
      "rewards/margins": 8.125,
      "rewards/rejected": -26.375,
      "step": 7860
    },
    {
      "epoch": 1.0058150680554667,
      "grad_norm": 8.551241588580146,
      "learning_rate": 9.017841978639643e-07,
      "logits/chosen": -0.1708984375,
      "logits/rejected": 0.4140625,
      "logps/chosen": -478.0,
      "logps/rejected": -520.0,
      "loss": 0.0508,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -17.75,
      "rewards/margins": 9.3125,
      "rewards/rejected": -27.0,
      "step": 7870
    },
    {
      "epoch": 1.0070931049907341,
      "grad_norm": 2.7122696670965403,
      "learning_rate": 9.012118181186658e-07,
      "logits/chosen": -0.251953125,
      "logits/rejected": 0.41796875,
      "logps/chosen": -490.0,
      "logps/rejected": -532.0,
      "loss": 0.0503,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.375,
      "rewards/margins": 9.375,
      "rewards/rejected": -28.75,
      "step": 7880
    },
    {
      "epoch": 1.0083711419260017,
      "grad_norm": 5.889627257747703,
      "learning_rate": 9.00640526892875e-07,
      "logits/chosen": -0.1826171875,
      "logits/rejected": 0.5078125,
      "logps/chosen": -444.0,
      "logps/rejected": -498.0,
      "loss": 0.0652,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.625,
      "rewards/margins": 9.5,
      "rewards/rejected": -27.0,
      "step": 7890
    },
    {
      "epoch": 1.0096491788612691,
      "grad_norm": 5.853619729193451,
      "learning_rate": 9.000703207408191e-07,
      "logits/chosen": -0.1533203125,
      "logits/rejected": 0.57421875,
      "logps/chosen": -438.0,
      "logps/rejected": -510.0,
      "loss": 0.0697,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.125,
      "rewards/margins": 9.5625,
      "rewards/rejected": -27.75,
      "step": 7900
    },
    {
      "epoch": 1.0109272157965365,
      "grad_norm": 6.127208589030695,
      "learning_rate": 8.995011962319761e-07,
      "logits/chosen": -0.208984375,
      "logits/rejected": 0.3828125,
      "logps/chosen": -410.0,
      "logps/rejected": -470.0,
      "loss": 0.0629,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.6875,
      "rewards/margins": 8.3125,
      "rewards/rejected": -24.0,
      "step": 7910
    },
    {
      "epoch": 1.012205252731804,
      "grad_norm": 4.77500975511013,
      "learning_rate": 8.989331499509894e-07,
      "logits/chosen": -0.2578125,
      "logits/rejected": 0.45703125,
      "logps/chosen": -472.0,
      "logps/rejected": -494.0,
      "loss": 0.0629,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.125,
      "rewards/margins": 8.0625,
      "rewards/rejected": -26.125,
      "step": 7920
    },
    {
      "epoch": 1.0134832896670714,
      "grad_norm": 5.2753862955908275,
      "learning_rate": 8.983661784975812e-07,
      "logits/chosen": -0.1689453125,
      "logits/rejected": 0.62890625,
      "logps/chosen": -460.0,
      "logps/rejected": -488.0,
      "loss": 0.0491,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.25,
      "rewards/margins": 8.5625,
      "rewards/rejected": -26.875,
      "step": 7930
    },
    {
      "epoch": 1.0147613266023388,
      "grad_norm": 4.297949520953858,
      "learning_rate": 8.97800278486467e-07,
      "logits/chosen": -0.353515625,
      "logits/rejected": 0.58203125,
      "logps/chosen": -466.0,
      "logps/rejected": -502.0,
      "loss": 0.0482,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.5,
      "rewards/margins": 9.3125,
      "rewards/rejected": -27.75,
      "step": 7940
    },
    {
      "epoch": 1.0160393635376062,
      "grad_norm": 3.6394742901662447,
      "learning_rate": 8.972354465472708e-07,
      "logits/chosen": -0.1796875,
      "logits/rejected": 0.4921875,
      "logps/chosen": -440.0,
      "logps/rejected": -496.0,
      "loss": 0.0555,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -17.5,
      "rewards/margins": 9.125,
      "rewards/rejected": -26.625,
      "step": 7950
    },
    {
      "epoch": 1.0173174004728738,
      "grad_norm": 1.685613799733221,
      "learning_rate": 8.966716793244405e-07,
      "logits/chosen": -0.34765625,
      "logits/rejected": 0.294921875,
      "logps/chosen": -444.0,
      "logps/rejected": -494.0,
      "loss": 0.059,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.75,
      "rewards/margins": 8.25,
      "rewards/rejected": -26.0,
      "step": 7960
    },
    {
      "epoch": 1.0185954374081412,
      "grad_norm": 3.5095688940948757,
      "learning_rate": 8.96108973477165e-07,
      "logits/chosen": -0.404296875,
      "logits/rejected": 0.357421875,
      "logps/chosen": -424.0,
      "logps/rejected": -494.0,
      "loss": 0.0535,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -15.375,
      "rewards/margins": 9.625,
      "rewards/rejected": -25.0,
      "step": 7970
    },
    {
      "epoch": 1.0198734743434086,
      "grad_norm": 5.506422767092577,
      "learning_rate": 8.955473256792899e-07,
      "logits/chosen": -0.337890625,
      "logits/rejected": 0.349609375,
      "logps/chosen": -422.0,
      "logps/rejected": -496.0,
      "loss": 0.0473,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -16.125,
      "rewards/margins": 9.1875,
      "rewards/rejected": -25.375,
      "step": 7980
    },
    {
      "epoch": 1.021151511278676,
      "grad_norm": 3.9322429122355054,
      "learning_rate": 8.949867326192358e-07,
      "logits/chosen": -0.333984375,
      "logits/rejected": 0.46484375,
      "logps/chosen": -422.0,
      "logps/rejected": -472.0,
      "loss": 0.0595,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -14.8125,
      "rewards/margins": 9.6875,
      "rewards/rejected": -24.5,
      "step": 7990
    },
    {
      "epoch": 1.0224295482139434,
      "grad_norm": 1.3403415185523129,
      "learning_rate": 8.944271909999158e-07,
      "logits/chosen": -0.330078125,
      "logits/rejected": 0.333984375,
      "logps/chosen": -454.0,
      "logps/rejected": -510.0,
      "loss": 0.0538,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -15.9375,
      "rewards/margins": 10.625,
      "rewards/rejected": -26.625,
      "step": 8000
    },
    {
      "epoch": 1.0237075851492108,
      "grad_norm": 5.530165596230844,
      "learning_rate": 8.938686975386545e-07,
      "logits/chosen": -0.3515625,
      "logits/rejected": 0.26953125,
      "logps/chosen": -418.0,
      "logps/rejected": -484.0,
      "loss": 0.0683,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -14.375,
      "rewards/margins": 9.1875,
      "rewards/rejected": -23.5,
      "step": 8010
    },
    {
      "epoch": 1.0249856220844782,
      "grad_norm": 7.188194153454651,
      "learning_rate": 8.933112489671067e-07,
      "logits/chosen": -0.314453125,
      "logits/rejected": 0.3515625,
      "logps/chosen": -422.0,
      "logps/rejected": -456.0,
      "loss": 0.0518,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -13.5625,
      "rewards/margins": 8.9375,
      "rewards/rejected": -22.5,
      "step": 8020
    },
    {
      "epoch": 1.0262636590197456,
      "grad_norm": 6.920105584691587,
      "learning_rate": 8.927548420311771e-07,
      "logits/chosen": -0.416015625,
      "logits/rejected": 0.36328125,
      "logps/chosen": -430.0,
      "logps/rejected": -496.0,
      "loss": 0.0554,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -15.4375,
      "rewards/margins": 9.9375,
      "rewards/rejected": -25.375,
      "step": 8030
    },
    {
      "epoch": 1.027541695955013,
      "grad_norm": 8.708887094267606,
      "learning_rate": 8.921994734909409e-07,
      "logits/chosen": -0.3125,
      "logits/rejected": 0.216796875,
      "logps/chosen": -416.0,
      "logps/rejected": -488.0,
      "loss": 0.0652,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -15.9375,
      "rewards/margins": 9.0,
      "rewards/rejected": -25.0,
      "step": 8040
    },
    {
      "epoch": 1.0288197328902806,
      "grad_norm": 4.211108348720398,
      "learning_rate": 8.916451401205645e-07,
      "logits/chosen": -0.4609375,
      "logits/rejected": 0.322265625,
      "logps/chosen": -428.0,
      "logps/rejected": -508.0,
      "loss": 0.0507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -16.375,
      "rewards/margins": 10.5,
      "rewards/rejected": -26.875,
      "step": 8050
    },
    {
      "epoch": 1.030097769825548,
      "grad_norm": 5.915217186647217,
      "learning_rate": 8.91091838708226e-07,
      "logits/chosen": -0.423828125,
      "logits/rejected": 0.431640625,
      "logps/chosen": -444.0,
      "logps/rejected": -490.0,
      "loss": 0.059,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.25,
      "rewards/margins": 9.5625,
      "rewards/rejected": -26.75,
      "step": 8060
    },
    {
      "epoch": 1.0313758067608154,
      "grad_norm": 5.001597889520234,
      "learning_rate": 8.905395660560378e-07,
      "logits/chosen": -0.48828125,
      "logits/rejected": 0.2421875,
      "logps/chosen": -454.0,
      "logps/rejected": -536.0,
      "loss": 0.0533,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -18.75,
      "rewards/margins": 9.875,
      "rewards/rejected": -28.625,
      "step": 8070
    },
    {
      "epoch": 1.0326538436960828,
      "grad_norm": 5.536112032456919,
      "learning_rate": 8.899883189799695e-07,
      "logits/chosen": -0.36328125,
      "logits/rejected": 0.27734375,
      "logps/chosen": -436.0,
      "logps/rejected": -516.0,
      "loss": 0.0462,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.625,
      "rewards/margins": 9.6875,
      "rewards/rejected": -27.25,
      "step": 8080
    },
    {
      "epoch": 1.0339318806313502,
      "grad_norm": 5.267616983461831,
      "learning_rate": 8.894380943097694e-07,
      "logits/chosen": -0.50390625,
      "logits/rejected": 0.353515625,
      "logps/chosen": -480.0,
      "logps/rejected": -532.0,
      "loss": 0.0499,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -19.875,
      "rewards/margins": 10.0625,
      "rewards/rejected": -29.875,
      "step": 8090
    },
    {
      "epoch": 1.0352099175666176,
      "grad_norm": 8.674904752838941,
      "learning_rate": 8.888888888888888e-07,
      "logits/chosen": -0.470703125,
      "logits/rejected": 0.12890625,
      "logps/chosen": -458.0,
      "logps/rejected": -524.0,
      "loss": 0.0477,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.75,
      "rewards/margins": 10.3125,
      "rewards/rejected": -30.0,
      "step": 8100
    },
    {
      "epoch": 1.036487954501885,
      "grad_norm": 5.051597267171127,
      "learning_rate": 8.883406995744061e-07,
      "logits/chosen": -0.58203125,
      "logits/rejected": 0.283203125,
      "logps/chosen": -436.0,
      "logps/rejected": -498.0,
      "loss": 0.0591,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.375,
      "rewards/margins": 10.375,
      "rewards/rejected": -26.75,
      "step": 8110
    },
    {
      "epoch": 1.0377659914371526,
      "grad_norm": 4.708748468204312,
      "learning_rate": 8.877935232369506e-07,
      "logits/chosen": -0.5234375,
      "logits/rejected": 0.19921875,
      "logps/chosen": -434.0,
      "logps/rejected": -492.0,
      "loss": 0.0612,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -16.5,
      "rewards/margins": 9.5625,
      "rewards/rejected": -26.125,
      "step": 8120
    },
    {
      "epoch": 1.03904402837242,
      "grad_norm": 3.328374234011167,
      "learning_rate": 8.872473567606276e-07,
      "logits/chosen": -0.546875,
      "logits/rejected": 0.291015625,
      "logps/chosen": -444.0,
      "logps/rejected": -490.0,
      "loss": 0.0501,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -16.875,
      "rewards/margins": 9.5625,
      "rewards/rejected": -26.375,
      "step": 8130
    },
    {
      "epoch": 1.0403220653076874,
      "grad_norm": 10.662643478048613,
      "learning_rate": 8.867021970429453e-07,
      "logits/chosen": -0.451171875,
      "logits/rejected": 0.29296875,
      "logps/chosen": -424.0,
      "logps/rejected": -476.0,
      "loss": 0.0635,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -16.75,
      "rewards/margins": 8.75,
      "rewards/rejected": -25.5,
      "step": 8140
    },
    {
      "epoch": 1.0416001022429549,
      "grad_norm": 6.4922487276262535,
      "learning_rate": 8.86158040994738e-07,
      "logits/chosen": -0.451171875,
      "logits/rejected": 0.291015625,
      "logps/chosen": -420.0,
      "logps/rejected": -482.0,
      "loss": 0.057,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.875,
      "rewards/margins": 9.875,
      "rewards/rejected": -26.75,
      "step": 8150
    },
    {
      "epoch": 1.0428781391782223,
      "grad_norm": 4.232286424128722,
      "learning_rate": 8.856148855400954e-07,
      "logits/chosen": -0.51953125,
      "logits/rejected": 0.2275390625,
      "logps/chosen": -414.0,
      "logps/rejected": -482.0,
      "loss": 0.0832,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -15.875,
      "rewards/margins": 9.875,
      "rewards/rejected": -25.75,
      "step": 8160
    },
    {
      "epoch": 1.0441561761134897,
      "grad_norm": 3.5675165400907387,
      "learning_rate": 8.850727276162873e-07,
      "logits/chosen": -0.6484375,
      "logits/rejected": 0.035400390625,
      "logps/chosen": -418.0,
      "logps/rejected": -490.0,
      "loss": 0.056,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.5,
      "rewards/margins": 9.4375,
      "rewards/rejected": -25.0,
      "step": 8170
    },
    {
      "epoch": 1.045434213048757,
      "grad_norm": 5.554514126740501,
      "learning_rate": 8.845315641736929e-07,
      "logits/chosen": -0.61328125,
      "logits/rejected": 0.283203125,
      "logps/chosen": -440.0,
      "logps/rejected": -480.0,
      "loss": 0.0592,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.25,
      "rewards/margins": 9.625,
      "rewards/rejected": -25.875,
      "step": 8180
    },
    {
      "epoch": 1.0467122499840245,
      "grad_norm": 6.777212434326151,
      "learning_rate": 8.839913921757278e-07,
      "logits/chosen": -0.54296875,
      "logits/rejected": 0.16796875,
      "logps/chosen": -438.0,
      "logps/rejected": -512.0,
      "loss": 0.0882,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -16.25,
      "rewards/margins": 9.5625,
      "rewards/rejected": -25.75,
      "step": 8190
    },
    {
      "epoch": 1.0479902869192919,
      "grad_norm": 6.49277803778793,
      "learning_rate": 8.834522085987722e-07,
      "logits/chosen": -0.609375,
      "logits/rejected": 0.26953125,
      "logps/chosen": -444.0,
      "logps/rejected": -494.0,
      "loss": 0.0758,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -15.0625,
      "rewards/margins": 9.5625,
      "rewards/rejected": -24.625,
      "step": 8200
    },
    {
      "epoch": 1.0492683238545595,
      "grad_norm": 4.216092887607009,
      "learning_rate": 8.829140104321008e-07,
      "logits/chosen": -0.5390625,
      "logits/rejected": 0.2353515625,
      "logps/chosen": -426.0,
      "logps/rejected": -456.0,
      "loss": 0.0574,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -14.875,
      "rewards/margins": 8.8125,
      "rewards/rejected": -23.75,
      "step": 8210
    },
    {
      "epoch": 1.0505463607898269,
      "grad_norm": 10.449750138061844,
      "learning_rate": 8.82376794677811e-07,
      "logits/chosen": -0.59765625,
      "logits/rejected": 0.1953125,
      "logps/chosen": -448.0,
      "logps/rejected": -484.0,
      "loss": 0.0547,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.0625,
      "rewards/margins": 8.5625,
      "rewards/rejected": -23.625,
      "step": 8220
    },
    {
      "epoch": 1.0518243977250943,
      "grad_norm": 3.5141283017437828,
      "learning_rate": 8.818405583507537e-07,
      "logits/chosen": -0.55078125,
      "logits/rejected": 0.1767578125,
      "logps/chosen": -418.0,
      "logps/rejected": -478.0,
      "loss": 0.0658,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -15.0625,
      "rewards/margins": 9.125,
      "rewards/rejected": -24.25,
      "step": 8230
    },
    {
      "epoch": 1.0531024346603617,
      "grad_norm": 4.465520791405367,
      "learning_rate": 8.813052984784634e-07,
      "logits/chosen": -0.47265625,
      "logits/rejected": 0.296875,
      "logps/chosen": -434.0,
      "logps/rejected": -492.0,
      "loss": 0.0506,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.375,
      "rewards/margins": 9.125,
      "rewards/rejected": -26.5,
      "step": 8240
    },
    {
      "epoch": 1.054380471595629,
      "grad_norm": 3.8654000243291957,
      "learning_rate": 8.807710121010885e-07,
      "logits/chosen": -0.4609375,
      "logits/rejected": 0.1708984375,
      "logps/chosen": -428.0,
      "logps/rejected": -512.0,
      "loss": 0.0639,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.75,
      "rewards/margins": 9.5,
      "rewards/rejected": -27.25,
      "step": 8250
    },
    {
      "epoch": 1.0556585085308965,
      "grad_norm": 1.998792033587222,
      "learning_rate": 8.802376962713231e-07,
      "logits/chosen": -0.578125,
      "logits/rejected": 0.333984375,
      "logps/chosen": -446.0,
      "logps/rejected": -504.0,
      "loss": 0.0548,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.875,
      "rewards/margins": 10.375,
      "rewards/rejected": -28.25,
      "step": 8260
    },
    {
      "epoch": 1.056936545466164,
      "grad_norm": 9.702295670474273,
      "learning_rate": 8.797053480543386e-07,
      "logits/chosen": -0.51171875,
      "logits/rejected": 0.0859375,
      "logps/chosen": -460.0,
      "logps/rejected": -536.0,
      "loss": 0.0776,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.375,
      "rewards/margins": 10.6875,
      "rewards/rejected": -29.125,
      "step": 8270
    },
    {
      "epoch": 1.0582145824014315,
      "grad_norm": 7.094741294826157,
      "learning_rate": 8.79173964527716e-07,
      "logits/chosen": -0.4921875,
      "logits/rejected": 0.1103515625,
      "logps/chosen": -474.0,
      "logps/rejected": -540.0,
      "loss": 0.0534,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.875,
      "rewards/margins": 10.0,
      "rewards/rejected": -28.875,
      "step": 8280
    },
    {
      "epoch": 1.059492619336699,
      "grad_norm": 6.684531831886535,
      "learning_rate": 8.78643542781378e-07,
      "logits/chosen": -0.5703125,
      "logits/rejected": 0.32421875,
      "logps/chosen": -432.0,
      "logps/rejected": -486.0,
      "loss": 0.0445,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.125,
      "rewards/margins": 9.6875,
      "rewards/rejected": -26.875,
      "step": 8290
    },
    {
      "epoch": 1.0607706562719663,
      "grad_norm": 9.629515668483066,
      "learning_rate": 8.781140799175228e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": 0.2890625,
      "logps/chosen": -434.0,
      "logps/rejected": -496.0,
      "loss": 0.0579,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.75,
      "rewards/margins": 9.4375,
      "rewards/rejected": -27.25,
      "step": 8300
    },
    {
      "epoch": 1.0620486932072337,
      "grad_norm": 6.542331492451529,
      "learning_rate": 8.775855730505568e-07,
      "logits/chosen": -0.52734375,
      "logits/rejected": 0.1845703125,
      "logps/chosen": -436.0,
      "logps/rejected": -512.0,
      "loss": 0.0568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -16.75,
      "rewards/margins": 9.8125,
      "rewards/rejected": -26.625,
      "step": 8310
    },
    {
      "epoch": 1.0633267301425011,
      "grad_norm": 6.909903072288922,
      "learning_rate": 8.770580193070291e-07,
      "logits/chosen": -0.59375,
      "logits/rejected": 0.244140625,
      "logps/chosen": -466.0,
      "logps/rejected": -528.0,
      "loss": 0.0568,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.25,
      "rewards/margins": 9.5625,
      "rewards/rejected": -27.875,
      "step": 8320
    },
    {
      "epoch": 1.0646047670777685,
      "grad_norm": 8.32297694124657,
      "learning_rate": 8.765314158255661e-07,
      "logits/chosen": -0.55078125,
      "logits/rejected": 0.08837890625,
      "logps/chosen": -440.0,
      "logps/rejected": -520.0,
      "loss": 0.0815,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.0,
      "rewards/margins": 9.5625,
      "rewards/rejected": -26.5,
      "step": 8330
    },
    {
      "epoch": 1.065882804013036,
      "grad_norm": 7.374909009184176,
      "learning_rate": 8.760057597568057e-07,
      "logits/chosen": -0.5859375,
      "logits/rejected": 0.2294921875,
      "logps/chosen": -416.0,
      "logps/rejected": -470.0,
      "loss": 0.0569,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -16.25,
      "rewards/margins": 8.9375,
      "rewards/rejected": -25.25,
      "step": 8340
    },
    {
      "epoch": 1.0671608409483033,
      "grad_norm": 2.7295626952123335,
      "learning_rate": 8.754810482633324e-07,
      "logits/chosen": -0.4765625,
      "logits/rejected": 0.302734375,
      "logps/chosen": -432.0,
      "logps/rejected": -480.0,
      "loss": 0.0464,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -17.375,
      "rewards/margins": 9.3125,
      "rewards/rejected": -26.75,
      "step": 8350
    },
    {
      "epoch": 1.0684388778835707,
      "grad_norm": 6.462376865414659,
      "learning_rate": 8.749572785196142e-07,
      "logits/chosen": -0.5546875,
      "logits/rejected": 0.1484375,
      "logps/chosen": -430.0,
      "logps/rejected": -510.0,
      "loss": 0.0497,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -17.0,
      "rewards/margins": 9.9375,
      "rewards/rejected": -26.875,
      "step": 8360
    },
    {
      "epoch": 1.0697169148188383,
      "grad_norm": 5.3496492811375305,
      "learning_rate": 8.744344477119373e-07,
      "logits/chosen": -0.55859375,
      "logits/rejected": 0.0556640625,
      "logps/chosen": -422.0,
      "logps/rejected": -492.0,
      "loss": 0.052,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -16.375,
      "rewards/margins": 9.1875,
      "rewards/rejected": -25.5,
      "step": 8370
    },
    {
      "epoch": 1.0709949517541058,
      "grad_norm": 7.096898066921041,
      "learning_rate": 8.739125530383433e-07,
      "logits/chosen": -0.609375,
      "logits/rejected": 0.279296875,
      "logps/chosen": -456.0,
      "logps/rejected": -492.0,
      "loss": 0.0568,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.375,
      "rewards/margins": 9.75,
      "rewards/rejected": -27.125,
      "step": 8380
    },
    {
      "epoch": 1.0722729886893732,
      "grad_norm": 6.870760703747319,
      "learning_rate": 8.733915917085661e-07,
      "logits/chosen": -0.55859375,
      "logits/rejected": 0.150390625,
      "logps/chosen": -444.0,
      "logps/rejected": -498.0,
      "loss": 0.0669,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.875,
      "rewards/margins": 9.4375,
      "rewards/rejected": -26.25,
      "step": 8390
    },
    {
      "epoch": 1.0735510256246406,
      "grad_norm": 8.215250190308998,
      "learning_rate": 8.728715609439695e-07,
      "logits/chosen": -0.57421875,
      "logits/rejected": 0.294921875,
      "logps/chosen": -428.0,
      "logps/rejected": -482.0,
      "loss": 0.0538,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.5,
      "rewards/margins": 9.0625,
      "rewards/rejected": -26.625,
      "step": 8400
    },
    {
      "epoch": 1.074829062559908,
      "grad_norm": 7.949010369416841,
      "learning_rate": 8.72352457977484e-07,
      "logits/chosen": -0.5859375,
      "logits/rejected": 0.45703125,
      "logps/chosen": -440.0,
      "logps/rejected": -490.0,
      "loss": 0.0729,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.875,
      "rewards/margins": 9.375,
      "rewards/rejected": -27.25,
      "step": 8410
    },
    {
      "epoch": 1.0761070994951754,
      "grad_norm": 3.995487370245279,
      "learning_rate": 8.718342800535456e-07,
      "logits/chosen": -0.58984375,
      "logits/rejected": -0.0211181640625,
      "logps/chosen": -446.0,
      "logps/rejected": -512.0,
      "loss": 0.0545,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.0,
      "rewards/margins": 8.9375,
      "rewards/rejected": -25.875,
      "step": 8420
    },
    {
      "epoch": 1.0773851364304428,
      "grad_norm": 2.783098360884097,
      "learning_rate": 8.713170244280353e-07,
      "logits/chosen": -0.55078125,
      "logits/rejected": 0.171875,
      "logps/chosen": -458.0,
      "logps/rejected": -496.0,
      "loss": 0.0427,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.25,
      "rewards/margins": 9.75,
      "rewards/rejected": -27.0,
      "step": 8430
    },
    {
      "epoch": 1.0786631733657104,
      "grad_norm": 5.1585586165108905,
      "learning_rate": 8.708006883682162e-07,
      "logits/chosen": -0.443359375,
      "logits/rejected": 0.376953125,
      "logps/chosen": -446.0,
      "logps/rejected": -500.0,
      "loss": 0.0648,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.125,
      "rewards/margins": 9.5625,
      "rewards/rejected": -27.75,
      "step": 8440
    },
    {
      "epoch": 1.0799412103009778,
      "grad_norm": 6.78502556984906,
      "learning_rate": 8.702852691526739e-07,
      "logits/chosen": -0.4765625,
      "logits/rejected": 0.453125,
      "logps/chosen": -450.0,
      "logps/rejected": -512.0,
      "loss": 0.07,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.125,
      "rewards/margins": 9.8125,
      "rewards/rejected": -27.875,
      "step": 8450
    },
    {
      "epoch": 1.0812192472362452,
      "grad_norm": 3.4550843274608507,
      "learning_rate": 8.697707640712562e-07,
      "logits/chosen": -0.400390625,
      "logits/rejected": 0.50390625,
      "logps/chosen": -456.0,
      "logps/rejected": -496.0,
      "loss": 0.0394,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.0,
      "rewards/margins": 9.5625,
      "rewards/rejected": -27.5,
      "step": 8460
    },
    {
      "epoch": 1.0824972841715126,
      "grad_norm": 5.1646145378556865,
      "learning_rate": 8.692571704250135e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": 0.498046875,
      "logps/chosen": -480.0,
      "logps/rejected": -516.0,
      "loss": 0.0537,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -19.75,
      "rewards/margins": 9.8125,
      "rewards/rejected": -29.625,
      "step": 8470
    },
    {
      "epoch": 1.08377532110678,
      "grad_norm": 5.862974440953242,
      "learning_rate": 8.687444855261388e-07,
      "logits/chosen": -0.4296875,
      "logits/rejected": 0.18359375,
      "logps/chosen": -488.0,
      "logps/rejected": -556.0,
      "loss": 0.0769,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -20.125,
      "rewards/margins": 9.3125,
      "rewards/rejected": -29.5,
      "step": 8480
    },
    {
      "epoch": 1.0850533580420474,
      "grad_norm": 2.21959420262151,
      "learning_rate": 8.682327066979084e-07,
      "logits/chosen": -0.3203125,
      "logits/rejected": 0.4765625,
      "logps/chosen": -440.0,
      "logps/rejected": -494.0,
      "loss": 0.0619,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.75,
      "rewards/margins": 9.0625,
      "rewards/rejected": -26.75,
      "step": 8490
    },
    {
      "epoch": 1.0863313949773148,
      "grad_norm": 2.5160249135879584,
      "learning_rate": 8.677218312746247e-07,
      "logits/chosen": -0.3671875,
      "logits/rejected": 0.404296875,
      "logps/chosen": -438.0,
      "logps/rejected": -476.0,
      "loss": 0.0628,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.625,
      "rewards/margins": 8.875,
      "rewards/rejected": -26.625,
      "step": 8500
    },
    {
      "epoch": 1.0876094319125822,
      "grad_norm": 4.849697814062802,
      "learning_rate": 8.672118566015558e-07,
      "logits/chosen": -0.3515625,
      "logits/rejected": 0.2578125,
      "logps/chosen": -456.0,
      "logps/rejected": -512.0,
      "loss": 0.0563,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.625,
      "rewards/margins": 8.875,
      "rewards/rejected": -26.625,
      "step": 8510
    },
    {
      "epoch": 1.0888874688478496,
      "grad_norm": 4.031504317534064,
      "learning_rate": 8.667027800348789e-07,
      "logits/chosen": -0.486328125,
      "logits/rejected": 0.244140625,
      "logps/chosen": -470.0,
      "logps/rejected": -516.0,
      "loss": 0.0635,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.75,
      "rewards/margins": 9.5625,
      "rewards/rejected": -27.25,
      "step": 8520
    },
    {
      "epoch": 1.0901655057831172,
      "grad_norm": 9.977156194588154,
      "learning_rate": 8.661945989416229e-07,
      "logits/chosen": -0.31640625,
      "logits/rejected": 0.36328125,
      "logps/chosen": -420.0,
      "logps/rejected": -468.0,
      "loss": 0.0821,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.75,
      "rewards/margins": 8.4375,
      "rewards/rejected": -25.125,
      "step": 8530
    },
    {
      "epoch": 1.0914435427183846,
      "grad_norm": 4.930727339688369,
      "learning_rate": 8.6568731069961e-07,
      "logits/chosen": -0.421875,
      "logits/rejected": 0.185546875,
      "logps/chosen": -464.0,
      "logps/rejected": -528.0,
      "loss": 0.0447,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.25,
      "rewards/margins": 9.3125,
      "rewards/rejected": -26.5,
      "step": 8540
    },
    {
      "epoch": 1.092721579653652,
      "grad_norm": 1.6766817756672765,
      "learning_rate": 8.651809126974002e-07,
      "logits/chosen": -0.2109375,
      "logits/rejected": 0.314453125,
      "logps/chosen": -418.0,
      "logps/rejected": -502.0,
      "loss": 0.0365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -17.0,
      "rewards/margins": 9.375,
      "rewards/rejected": -26.5,
      "step": 8550
    },
    {
      "epoch": 1.0939996165889194,
      "grad_norm": 5.342922713375712,
      "learning_rate": 8.646754023342339e-07,
      "logits/chosen": -0.484375,
      "logits/rejected": 0.2265625,
      "logps/chosen": -478.0,
      "logps/rejected": -516.0,
      "loss": 0.0496,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.625,
      "rewards/margins": 9.875,
      "rewards/rejected": -27.625,
      "step": 8560
    },
    {
      "epoch": 1.0952776535241868,
      "grad_norm": 6.411210156798341,
      "learning_rate": 8.64170777019976e-07,
      "logits/chosen": -0.439453125,
      "logits/rejected": 0.34765625,
      "logps/chosen": -442.0,
      "logps/rejected": -498.0,
      "loss": 0.0786,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.75,
      "rewards/margins": 9.625,
      "rewards/rejected": -27.375,
      "step": 8570
    },
    {
      "epoch": 1.0965556904594542,
      "grad_norm": 6.856843274878725,
      "learning_rate": 8.636670341750609e-07,
      "logits/chosen": -0.330078125,
      "logits/rejected": 0.326171875,
      "logps/chosen": -430.0,
      "logps/rejected": -488.0,
      "loss": 0.0658,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.75,
      "rewards/margins": 9.875,
      "rewards/rejected": -26.625,
      "step": 8580
    },
    {
      "epoch": 1.0978337273947216,
      "grad_norm": 4.59081833949407,
      "learning_rate": 8.631641712304359e-07,
      "logits/chosen": -0.38671875,
      "logits/rejected": 0.5859375,
      "logps/chosen": -446.0,
      "logps/rejected": -498.0,
      "loss": 0.0628,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.0,
      "rewards/margins": 9.5,
      "rewards/rejected": -27.5,
      "step": 8590
    },
    {
      "epoch": 1.0991117643299892,
      "grad_norm": 1.901613268410469,
      "learning_rate": 8.626621856275073e-07,
      "logits/chosen": -0.42578125,
      "logits/rejected": 0.1328125,
      "logps/chosen": -444.0,
      "logps/rejected": -510.0,
      "loss": 0.0726,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -15.9375,
      "rewards/margins": 9.6875,
      "rewards/rejected": -25.625,
      "step": 8600
    },
    {
      "epoch": 1.1003898012652567,
      "grad_norm": 3.2397096952061633,
      "learning_rate": 8.621610748180847e-07,
      "logits/chosen": -0.40625,
      "logits/rejected": 0.515625,
      "logps/chosen": -424.0,
      "logps/rejected": -478.0,
      "loss": 0.054,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -16.375,
      "rewards/margins": 9.375,
      "rewards/rejected": -25.75,
      "step": 8610
    },
    {
      "epoch": 1.101667838200524,
      "grad_norm": 6.859515859937961,
      "learning_rate": 8.616608362643274e-07,
      "logits/chosen": -0.333984375,
      "logits/rejected": 0.404296875,
      "logps/chosen": -456.0,
      "logps/rejected": -508.0,
      "loss": 0.0512,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.0,
      "rewards/margins": 9.6875,
      "rewards/rejected": -27.75,
      "step": 8620
    },
    {
      "epoch": 1.1029458751357915,
      "grad_norm": 8.1997905066464,
      "learning_rate": 8.611614674386904e-07,
      "logits/chosen": -0.46484375,
      "logits/rejected": 0.2041015625,
      "logps/chosen": -408.0,
      "logps/rejected": -478.0,
      "loss": 0.0598,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.1875,
      "rewards/margins": 9.3125,
      "rewards/rejected": -24.5,
      "step": 8630
    },
    {
      "epoch": 1.1042239120710589,
      "grad_norm": 5.201482768259036,
      "learning_rate": 8.606629658238703e-07,
      "logits/chosen": -0.54296875,
      "logits/rejected": 0.3515625,
      "logps/chosen": -432.0,
      "logps/rejected": -484.0,
      "loss": 0.0687,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.25,
      "rewards/margins": 10.25,
      "rewards/rejected": -26.5,
      "step": 8640
    },
    {
      "epoch": 1.1055019490063263,
      "grad_norm": 5.50425150943279,
      "learning_rate": 8.601653289127525e-07,
      "logits/chosen": -0.447265625,
      "logits/rejected": 0.2119140625,
      "logps/chosen": -452.0,
      "logps/rejected": -528.0,
      "loss": 0.0499,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.375,
      "rewards/margins": 9.5,
      "rewards/rejected": -25.75,
      "step": 8650
    },
    {
      "epoch": 1.1067799859415937,
      "grad_norm": 4.131236182884247,
      "learning_rate": 8.596685542083577e-07,
      "logits/chosen": -0.201171875,
      "logits/rejected": 0.50390625,
      "logps/chosen": -454.0,
      "logps/rejected": -492.0,
      "loss": 0.04,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.25,
      "rewards/margins": 9.4375,
      "rewards/rejected": -27.75,
      "step": 8660
    },
    {
      "epoch": 1.108058022876861,
      "grad_norm": 2.5402584378298285,
      "learning_rate": 8.591726392237899e-07,
      "logits/chosen": -0.3828125,
      "logits/rejected": 0.41796875,
      "logps/chosen": -446.0,
      "logps/rejected": -492.0,
      "loss": 0.0307,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.5,
      "rewards/margins": 9.875,
      "rewards/rejected": -28.375,
      "step": 8670
    },
    {
      "epoch": 1.1093360598121285,
      "grad_norm": 5.4679685878279045,
      "learning_rate": 8.586775814821837e-07,
      "logits/chosen": -0.4921875,
      "logits/rejected": 0.11962890625,
      "logps/chosen": -464.0,
      "logps/rejected": -524.0,
      "loss": 0.0661,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.125,
      "rewards/margins": 9.8125,
      "rewards/rejected": -28.875,
      "step": 8680
    },
    {
      "epoch": 1.110614096747396,
      "grad_norm": 3.6782026089102233,
      "learning_rate": 8.58183378516652e-07,
      "logits/chosen": -0.5,
      "logits/rejected": 0.08203125,
      "logps/chosen": -484.0,
      "logps/rejected": -540.0,
      "loss": 0.0563,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.375,
      "rewards/margins": 10.4375,
      "rewards/rejected": -28.875,
      "step": 8690
    },
    {
      "epoch": 1.1118921336826635,
      "grad_norm": 6.085040121551929,
      "learning_rate": 8.576900278702358e-07,
      "logits/chosen": -0.41796875,
      "logits/rejected": 0.34375,
      "logps/chosen": -460.0,
      "logps/rejected": -512.0,
      "loss": 0.067,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.375,
      "rewards/margins": 9.8125,
      "rewards/rejected": -29.125,
      "step": 8700
    },
    {
      "epoch": 1.1131701706179309,
      "grad_norm": 4.812067741670891,
      "learning_rate": 8.57197527095851e-07,
      "logits/chosen": -0.45703125,
      "logits/rejected": 0.37109375,
      "logps/chosen": -442.0,
      "logps/rejected": -468.0,
      "loss": 0.0637,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.0,
      "rewards/margins": 8.9375,
      "rewards/rejected": -26.0,
      "step": 8710
    },
    {
      "epoch": 1.1144482075531983,
      "grad_norm": 3.0853887544833603,
      "learning_rate": 8.567058737562385e-07,
      "logits/chosen": -0.380859375,
      "logits/rejected": 0.34375,
      "logps/chosen": -420.0,
      "logps/rejected": -482.0,
      "loss": 0.0697,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.125,
      "rewards/margins": 8.875,
      "rewards/rejected": -25.125,
      "step": 8720
    },
    {
      "epoch": 1.1157262444884657,
      "grad_norm": 6.834328054145813,
      "learning_rate": 8.562150654239141e-07,
      "logits/chosen": -0.60546875,
      "logits/rejected": 0.125,
      "logps/chosen": -432.0,
      "logps/rejected": -498.0,
      "loss": 0.0733,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.3125,
      "rewards/margins": 9.1875,
      "rewards/rejected": -24.5,
      "step": 8730
    },
    {
      "epoch": 1.117004281423733,
      "grad_norm": 7.52366553922908,
      "learning_rate": 8.55725099681116e-07,
      "logits/chosen": -0.416015625,
      "logits/rejected": 0.061767578125,
      "logps/chosen": -394.0,
      "logps/rejected": -486.0,
      "loss": 0.0527,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -14.125,
      "rewards/margins": 9.25,
      "rewards/rejected": -23.375,
      "step": 8740
    },
    {
      "epoch": 1.1182823183590005,
      "grad_norm": 6.723203685946788,
      "learning_rate": 8.552359741197579e-07,
      "logits/chosen": -0.4609375,
      "logits/rejected": 0.1982421875,
      "logps/chosen": -408.0,
      "logps/rejected": -462.0,
      "loss": 0.0671,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -14.4375,
      "rewards/margins": 8.75,
      "rewards/rejected": -23.125,
      "step": 8750
    },
    {
      "epoch": 1.1195603552942681,
      "grad_norm": 6.036997227064993,
      "learning_rate": 8.547476863413765e-07,
      "logits/chosen": -0.4453125,
      "logits/rejected": 0.16796875,
      "logps/chosen": -426.0,
      "logps/rejected": -468.0,
      "loss": 0.0561,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -14.0625,
      "rewards/margins": 8.3125,
      "rewards/rejected": -22.375,
      "step": 8760
    },
    {
      "epoch": 1.1208383922295355,
      "grad_norm": 9.134968785984467,
      "learning_rate": 8.54260233957083e-07,
      "logits/chosen": -0.3671875,
      "logits/rejected": 0.41015625,
      "logps/chosen": -406.0,
      "logps/rejected": -462.0,
      "loss": 0.0733,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -14.875,
      "rewards/margins": 9.0,
      "rewards/rejected": -23.875,
      "step": 8770
    },
    {
      "epoch": 1.122116429164803,
      "grad_norm": 3.5224677477096558,
      "learning_rate": 8.537736145875154e-07,
      "logits/chosen": -0.388671875,
      "logits/rejected": 0.259765625,
      "logps/chosen": -434.0,
      "logps/rejected": -508.0,
      "loss": 0.0414,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.0,
      "rewards/margins": 9.0,
      "rewards/rejected": -25.0,
      "step": 8780
    },
    {
      "epoch": 1.1233944661000703,
      "grad_norm": 5.843725597935624,
      "learning_rate": 8.532878258627874e-07,
      "logits/chosen": -0.40234375,
      "logits/rejected": 0.357421875,
      "logps/chosen": -424.0,
      "logps/rejected": -486.0,
      "loss": 0.0517,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.6875,
      "rewards/margins": 9.375,
      "rewards/rejected": -25.125,
      "step": 8790
    },
    {
      "epoch": 1.1246725030353377,
      "grad_norm": 6.902718760938501,
      "learning_rate": 8.528028654224416e-07,
      "logits/chosen": -0.349609375,
      "logits/rejected": 0.51953125,
      "logps/chosen": -440.0,
      "logps/rejected": -490.0,
      "loss": 0.068,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -18.125,
      "rewards/margins": 9.25,
      "rewards/rejected": -27.375,
      "step": 8800
    },
    {
      "epoch": 1.1259505399706051,
      "grad_norm": 6.289081724797012,
      "learning_rate": 8.523187309154008e-07,
      "logits/chosen": -0.40625,
      "logits/rejected": 0.2177734375,
      "logps/chosen": -424.0,
      "logps/rejected": -480.0,
      "loss": 0.0761,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.0,
      "rewards/margins": 8.4375,
      "rewards/rejected": -24.5,
      "step": 8810
    },
    {
      "epoch": 1.1272285769058725,
      "grad_norm": 7.472539600954268,
      "learning_rate": 8.518354199999198e-07,
      "logits/chosen": -0.48046875,
      "logits/rejected": 0.12158203125,
      "logps/chosen": -408.0,
      "logps/rejected": -470.0,
      "loss": 0.0513,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -13.375,
      "rewards/margins": 9.625,
      "rewards/rejected": -23.0,
      "step": 8820
    },
    {
      "epoch": 1.12850661384114,
      "grad_norm": 2.4316497976658034,
      "learning_rate": 8.513529303435386e-07,
      "logits/chosen": -0.435546875,
      "logits/rejected": 0.2158203125,
      "logps/chosen": -474.0,
      "logps/rejected": -524.0,
      "loss": 0.0649,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -16.375,
      "rewards/margins": 9.125,
      "rewards/rejected": -25.5,
      "step": 8830
    },
    {
      "epoch": 1.1297846507764073,
      "grad_norm": 9.148847772560503,
      "learning_rate": 8.50871259623034e-07,
      "logits/chosen": -0.33984375,
      "logits/rejected": 0.3203125,
      "logps/chosen": -412.0,
      "logps/rejected": -504.0,
      "loss": 0.0714,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -16.0,
      "rewards/margins": 9.75,
      "rewards/rejected": -25.75,
      "step": 8840
    },
    {
      "epoch": 1.131062687711675,
      "grad_norm": 1.521003441425215,
      "learning_rate": 8.503904055243742e-07,
      "logits/chosen": -0.57421875,
      "logits/rejected": 0.1416015625,
      "logps/chosen": -432.0,
      "logps/rejected": -506.0,
      "loss": 0.0498,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.0625,
      "rewards/margins": 10.0625,
      "rewards/rejected": -25.125,
      "step": 8850
    },
    {
      "epoch": 1.1323407246469424,
      "grad_norm": 4.406173798249354,
      "learning_rate": 8.499103657426704e-07,
      "logits/chosen": -0.58984375,
      "logits/rejected": 0.3515625,
      "logps/chosen": -438.0,
      "logps/rejected": -496.0,
      "loss": 0.0521,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -16.5,
      "rewards/margins": 9.9375,
      "rewards/rejected": -26.5,
      "step": 8860
    },
    {
      "epoch": 1.1336187615822098,
      "grad_norm": 9.81888294454049,
      "learning_rate": 8.494311379821314e-07,
      "logits/chosen": -0.392578125,
      "logits/rejected": 0.390625,
      "logps/chosen": -446.0,
      "logps/rejected": -512.0,
      "loss": 0.0646,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.875,
      "rewards/margins": 9.75,
      "rewards/rejected": -27.625,
      "step": 8870
    },
    {
      "epoch": 1.1348967985174772,
      "grad_norm": 8.710372059740628,
      "learning_rate": 8.489527199560178e-07,
      "logits/chosen": -0.482421875,
      "logits/rejected": 0.2734375,
      "logps/chosen": -448.0,
      "logps/rejected": -510.0,
      "loss": 0.0597,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.875,
      "rewards/margins": 9.8125,
      "rewards/rejected": -27.75,
      "step": 8880
    },
    {
      "epoch": 1.1361748354527446,
      "grad_norm": 6.944418945838836,
      "learning_rate": 8.484751093865948e-07,
      "logits/chosen": -0.294921875,
      "logits/rejected": 0.41796875,
      "logps/chosen": -450.0,
      "logps/rejected": -498.0,
      "loss": 0.0671,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.625,
      "rewards/margins": 9.75,
      "rewards/rejected": -28.375,
      "step": 8890
    },
    {
      "epoch": 1.137452872388012,
      "grad_norm": 3.8937149305018863,
      "learning_rate": 8.47998304005088e-07,
      "logits/chosen": -0.279296875,
      "logits/rejected": 0.3671875,
      "logps/chosen": -440.0,
      "logps/rejected": -504.0,
      "loss": 0.0799,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.5,
      "rewards/margins": 9.5625,
      "rewards/rejected": -28.125,
      "step": 8900
    },
    {
      "epoch": 1.1387309093232794,
      "grad_norm": 3.762552560273072,
      "learning_rate": 8.475223015516377e-07,
      "logits/chosen": -0.33984375,
      "logits/rejected": 0.337890625,
      "logps/chosen": -452.0,
      "logps/rejected": -498.0,
      "loss": 0.0606,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.75,
      "rewards/margins": 9.5625,
      "rewards/rejected": -28.375,
      "step": 8910
    },
    {
      "epoch": 1.140008946258547,
      "grad_norm": 7.987241572488861,
      "learning_rate": 8.470470997752534e-07,
      "logits/chosen": -0.29296875,
      "logits/rejected": 0.294921875,
      "logps/chosen": -484.0,
      "logps/rejected": -520.0,
      "loss": 0.0574,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.875,
      "rewards/margins": 8.75,
      "rewards/rejected": -28.625,
      "step": 8920
    },
    {
      "epoch": 1.1412869831938144,
      "grad_norm": 6.871438557390857,
      "learning_rate": 8.465726964337702e-07,
      "logits/chosen": -0.384765625,
      "logits/rejected": 0.3125,
      "logps/chosen": -480.0,
      "logps/rejected": -528.0,
      "loss": 0.0765,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.375,
      "rewards/margins": 9.8125,
      "rewards/rejected": -29.25,
      "step": 8930
    },
    {
      "epoch": 1.1425650201290818,
      "grad_norm": 12.436017329246217,
      "learning_rate": 8.460990892938031e-07,
      "logits/chosen": -0.318359375,
      "logits/rejected": 0.35546875,
      "logps/chosen": -466.0,
      "logps/rejected": -520.0,
      "loss": 0.0614,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.25,
      "rewards/margins": 9.125,
      "rewards/rejected": -29.375,
      "step": 8940
    },
    {
      "epoch": 1.1438430570643492,
      "grad_norm": 4.546368313294303,
      "learning_rate": 8.456262761307038e-07,
      "logits/chosen": -0.34375,
      "logits/rejected": 0.390625,
      "logps/chosen": -472.0,
      "logps/rejected": -524.0,
      "loss": 0.071,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.75,
      "rewards/margins": 8.9375,
      "rewards/rejected": -28.75,
      "step": 8950
    },
    {
      "epoch": 1.1451210939996166,
      "grad_norm": 6.4918176349658205,
      "learning_rate": 8.451542547285166e-07,
      "logits/chosen": -0.224609375,
      "logits/rejected": 0.416015625,
      "logps/chosen": -456.0,
      "logps/rejected": -516.0,
      "loss": 0.0527,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.5,
      "rewards/margins": 9.75,
      "rewards/rejected": -29.125,
      "step": 8960
    },
    {
      "epoch": 1.146399130934884,
      "grad_norm": 4.7153343582407485,
      "learning_rate": 8.44683022879934e-07,
      "logits/chosen": -0.294921875,
      "logits/rejected": 0.47265625,
      "logps/chosen": -458.0,
      "logps/rejected": -512.0,
      "loss": 0.0707,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.875,
      "rewards/margins": 10.25,
      "rewards/rejected": -29.125,
      "step": 8970
    },
    {
      "epoch": 1.1476771678701514,
      "grad_norm": 5.637901865282421,
      "learning_rate": 8.442125783862544e-07,
      "logits/chosen": -0.357421875,
      "logits/rejected": 0.294921875,
      "logps/chosen": -438.0,
      "logps/rejected": -516.0,
      "loss": 0.058,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.0,
      "rewards/margins": 10.0625,
      "rewards/rejected": -27.0,
      "step": 8980
    },
    {
      "epoch": 1.1489552048054188,
      "grad_norm": 8.369450554409553,
      "learning_rate": 8.437429190573388e-07,
      "logits/chosen": -0.2314453125,
      "logits/rejected": 0.41015625,
      "logps/chosen": -436.0,
      "logps/rejected": -478.0,
      "loss": 0.0584,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -16.375,
      "rewards/margins": 9.3125,
      "rewards/rejected": -25.625,
      "step": 8990
    },
    {
      "epoch": 1.1502332417406862,
      "grad_norm": 2.523143360761709,
      "learning_rate": 8.432740427115678e-07,
      "logits/chosen": -0.333984375,
      "logits/rejected": 0.294921875,
      "logps/chosen": -454.0,
      "logps/rejected": -506.0,
      "loss": 0.0556,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.5,
      "rewards/margins": 8.9375,
      "rewards/rejected": -26.5,
      "step": 9000
    },
    {
      "epoch": 1.1515112786759538,
      "grad_norm": 7.013230215059176,
      "learning_rate": 8.428059471757984e-07,
      "logits/chosen": -0.396484375,
      "logits/rejected": 0.361328125,
      "logps/chosen": -452.0,
      "logps/rejected": -524.0,
      "loss": 0.0466,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.875,
      "rewards/margins": 10.3125,
      "rewards/rejected": -28.25,
      "step": 9010
    },
    {
      "epoch": 1.1527893156112212,
      "grad_norm": 6.106980308731573,
      "learning_rate": 8.423386302853226e-07,
      "logits/chosen": -0.3125,
      "logits/rejected": 0.26953125,
      "logps/chosen": -454.0,
      "logps/rejected": -524.0,
      "loss": 0.0533,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.25,
      "rewards/margins": 10.0,
      "rewards/rejected": -27.25,
      "step": 9020
    },
    {
      "epoch": 1.1540673525464886,
      "grad_norm": 5.391146077498199,
      "learning_rate": 8.418720898838254e-07,
      "logits/chosen": -0.421875,
      "logits/rejected": 0.251953125,
      "logps/chosen": -440.0,
      "logps/rejected": -502.0,
      "loss": 0.0768,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.25,
      "rewards/margins": 9.0,
      "rewards/rejected": -26.25,
      "step": 9030
    },
    {
      "epoch": 1.155345389481756,
      "grad_norm": 7.91785266967722,
      "learning_rate": 8.414063238233425e-07,
      "logits/chosen": -0.306640625,
      "logits/rejected": 0.36328125,
      "logps/chosen": -440.0,
      "logps/rejected": -494.0,
      "loss": 0.0631,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.5,
      "rewards/margins": 9.625,
      "rewards/rejected": -27.125,
      "step": 9040
    },
    {
      "epoch": 1.1566234264170234,
      "grad_norm": 9.067568569631874,
      "learning_rate": 8.409413299642188e-07,
      "logits/chosen": -0.3203125,
      "logits/rejected": 0.466796875,
      "logps/chosen": -442.0,
      "logps/rejected": -502.0,
      "loss": 0.044,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.375,
      "rewards/margins": 9.25,
      "rewards/rejected": -27.625,
      "step": 9050
    },
    {
      "epoch": 1.1579014633522908,
      "grad_norm": 6.074878473151594,
      "learning_rate": 8.404771061750672e-07,
      "logits/chosen": -0.451171875,
      "logits/rejected": 0.251953125,
      "logps/chosen": -480.0,
      "logps/rejected": -536.0,
      "loss": 0.0639,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.25,
      "rewards/margins": 10.3125,
      "rewards/rejected": -28.5,
      "step": 9060
    },
    {
      "epoch": 1.1591795002875582,
      "grad_norm": 5.380232862806035,
      "learning_rate": 8.400136503327277e-07,
      "logits/chosen": -0.353515625,
      "logits/rejected": 0.44140625,
      "logps/chosen": -452.0,
      "logps/rejected": -498.0,
      "loss": 0.0499,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.25,
      "rewards/margins": 9.75,
      "rewards/rejected": -28.0,
      "step": 9070
    },
    {
      "epoch": 1.1604575372228259,
      "grad_norm": 6.752500957937942,
      "learning_rate": 8.395509603222271e-07,
      "logits/chosen": -0.4609375,
      "logits/rejected": 0.224609375,
      "logps/chosen": -448.0,
      "logps/rejected": -512.0,
      "loss": 0.0523,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.125,
      "rewards/margins": 9.3125,
      "rewards/rejected": -27.5,
      "step": 9080
    },
    {
      "epoch": 1.1617355741580933,
      "grad_norm": 6.655805328798849,
      "learning_rate": 8.390890340367368e-07,
      "logits/chosen": -0.427734375,
      "logits/rejected": 0.33203125,
      "logps/chosen": -440.0,
      "logps/rejected": -476.0,
      "loss": 0.0934,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.375,
      "rewards/margins": 9.375,
      "rewards/rejected": -26.75,
      "step": 9090
    },
    {
      "epoch": 1.1630136110933607,
      "grad_norm": 4.492148764353409,
      "learning_rate": 8.386278693775346e-07,
      "logits/chosen": -0.42578125,
      "logits/rejected": 0.345703125,
      "logps/chosen": -482.0,
      "logps/rejected": -532.0,
      "loss": 0.0537,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.875,
      "rewards/margins": 9.9375,
      "rewards/rejected": -29.75,
      "step": 9100
    },
    {
      "epoch": 1.164291648028628,
      "grad_norm": 7.807282130995772,
      "learning_rate": 8.381674642539632e-07,
      "logits/chosen": -0.470703125,
      "logits/rejected": 0.2734375,
      "logps/chosen": -474.0,
      "logps/rejected": -524.0,
      "loss": 0.0721,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.625,
      "rewards/margins": 9.3125,
      "rewards/rejected": -28.0,
      "step": 9110
    },
    {
      "epoch": 1.1655696849638955,
      "grad_norm": 2.868148009751236,
      "learning_rate": 8.37707816583391e-07,
      "logits/chosen": -0.4296875,
      "logits/rejected": 0.337890625,
      "logps/chosen": -468.0,
      "logps/rejected": -520.0,
      "loss": 0.0817,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.375,
      "rewards/margins": 9.0,
      "rewards/rejected": -29.5,
      "step": 9120
    },
    {
      "epoch": 1.1668477218991629,
      "grad_norm": 5.772744813493889,
      "learning_rate": 8.372489242911724e-07,
      "logits/chosen": -0.35546875,
      "logits/rejected": 0.416015625,
      "logps/chosen": -472.0,
      "logps/rejected": -544.0,
      "loss": 0.0629,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -21.25,
      "rewards/margins": 9.3125,
      "rewards/rejected": -30.5,
      "step": 9130
    },
    {
      "epoch": 1.1681257588344303,
      "grad_norm": 3.6717592585934513,
      "learning_rate": 8.367907853106078e-07,
      "logits/chosen": -0.314453125,
      "logits/rejected": 0.33984375,
      "logps/chosen": -478.0,
      "logps/rejected": -560.0,
      "loss": 0.0459,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.5,
      "rewards/margins": 9.3125,
      "rewards/rejected": -30.75,
      "step": 9140
    },
    {
      "epoch": 1.1694037957696977,
      "grad_norm": 6.253289651991332,
      "learning_rate": 8.363333975829066e-07,
      "logits/chosen": -0.2734375,
      "logits/rejected": 0.400390625,
      "logps/chosen": -500.0,
      "logps/rejected": -560.0,
      "loss": 0.0554,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -23.375,
      "rewards/margins": 9.5,
      "rewards/rejected": -32.75,
      "step": 9150
    },
    {
      "epoch": 1.170681832704965,
      "grad_norm": 7.1202587625868645,
      "learning_rate": 8.358767590571457e-07,
      "logits/chosen": -0.482421875,
      "logits/rejected": 0.33984375,
      "logps/chosen": -498.0,
      "logps/rejected": -556.0,
      "loss": 0.0635,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.625,
      "rewards/margins": 10.8125,
      "rewards/rejected": -32.5,
      "step": 9160
    },
    {
      "epoch": 1.1719598696402327,
      "grad_norm": 5.76217154255802,
      "learning_rate": 8.354208676902326e-07,
      "logits/chosen": -0.32421875,
      "logits/rejected": 0.2734375,
      "logps/chosen": -476.0,
      "logps/rejected": -552.0,
      "loss": 0.0807,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.75,
      "rewards/margins": 10.1875,
      "rewards/rejected": -30.875,
      "step": 9170
    },
    {
      "epoch": 1.1732379065755,
      "grad_norm": 4.240648277433848,
      "learning_rate": 8.349657214468659e-07,
      "logits/chosen": -0.345703125,
      "logits/rejected": 0.412109375,
      "logps/chosen": -462.0,
      "logps/rejected": -520.0,
      "loss": 0.0681,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -20.125,
      "rewards/margins": 9.6875,
      "rewards/rejected": -29.75,
      "step": 9180
    },
    {
      "epoch": 1.1745159435107675,
      "grad_norm": 7.68096312477842,
      "learning_rate": 8.345113182994988e-07,
      "logits/chosen": -0.263671875,
      "logits/rejected": 0.2431640625,
      "logps/chosen": -466.0,
      "logps/rejected": -506.0,
      "loss": 0.1046,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.125,
      "rewards/margins": 8.1875,
      "rewards/rejected": -28.25,
      "step": 9190
    },
    {
      "epoch": 1.175793980446035,
      "grad_norm": 6.127687081399572,
      "learning_rate": 8.34057656228299e-07,
      "logits/chosen": -0.515625,
      "logits/rejected": 0.181640625,
      "logps/chosen": -456.0,
      "logps/rejected": -506.0,
      "loss": 0.0609,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.625,
      "rewards/margins": 9.5625,
      "rewards/rejected": -27.25,
      "step": 9200
    },
    {
      "epoch": 1.1770720173813023,
      "grad_norm": 5.991803462822291,
      "learning_rate": 8.336047332211128e-07,
      "logits/chosen": -0.51171875,
      "logits/rejected": 0.1630859375,
      "logps/chosen": -456.0,
      "logps/rejected": -508.0,
      "loss": 0.0546,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.0,
      "rewards/margins": 9.4375,
      "rewards/rejected": -26.375,
      "step": 9210
    },
    {
      "epoch": 1.1783500543165697,
      "grad_norm": 5.598555132528687,
      "learning_rate": 8.331525472734267e-07,
      "logits/chosen": -0.57421875,
      "logits/rejected": 0.2041015625,
      "logps/chosen": -436.0,
      "logps/rejected": -480.0,
      "loss": 0.0626,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.25,
      "rewards/margins": 9.4375,
      "rewards/rejected": -26.625,
      "step": 9220
    },
    {
      "epoch": 1.179628091251837,
      "grad_norm": 3.9243565535082157,
      "learning_rate": 8.327010963883302e-07,
      "logits/chosen": -0.53125,
      "logits/rejected": 0.1708984375,
      "logps/chosen": -474.0,
      "logps/rejected": -524.0,
      "loss": 0.0733,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.625,
      "rewards/margins": 9.0625,
      "rewards/rejected": -28.625,
      "step": 9230
    },
    {
      "epoch": 1.1809061281871047,
      "grad_norm": 8.995172065600979,
      "learning_rate": 8.322503785764789e-07,
      "logits/chosen": -0.6015625,
      "logits/rejected": 0.29296875,
      "logps/chosen": -448.0,
      "logps/rejected": -476.0,
      "loss": 0.062,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.375,
      "rewards/margins": 9.0,
      "rewards/rejected": -26.375,
      "step": 9240
    },
    {
      "epoch": 1.1821841651223721,
      "grad_norm": 6.334559204967448,
      "learning_rate": 8.318003918560583e-07,
      "logits/chosen": -0.58984375,
      "logits/rejected": 0.1123046875,
      "logps/chosen": -452.0,
      "logps/rejected": -490.0,
      "loss": 0.0676,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -17.125,
      "rewards/margins": 9.625,
      "rewards/rejected": -26.75,
      "step": 9250
    },
    {
      "epoch": 1.1834622020576395,
      "grad_norm": 12.851562357360837,
      "learning_rate": 8.313511342527453e-07,
      "logits/chosen": -0.384765625,
      "logits/rejected": 0.28515625,
      "logps/chosen": -424.0,
      "logps/rejected": -488.0,
      "loss": 0.0569,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.0,
      "rewards/margins": 8.75,
      "rewards/rejected": -26.75,
      "step": 9260
    },
    {
      "epoch": 1.184740238992907,
      "grad_norm": 1.4992315865956207,
      "learning_rate": 8.309026037996745e-07,
      "logits/chosen": -0.41796875,
      "logits/rejected": 0.248046875,
      "logps/chosen": -464.0,
      "logps/rejected": -524.0,
      "loss": 0.0539,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -19.75,
      "rewards/margins": 9.5,
      "rewards/rejected": -29.25,
      "step": 9270
    },
    {
      "epoch": 1.1860182759281743,
      "grad_norm": 4.424842034289086,
      "learning_rate": 8.304547985373996e-07,
      "logits/chosen": -0.5,
      "logits/rejected": 0.169921875,
      "logps/chosen": -486.0,
      "logps/rejected": -528.0,
      "loss": 0.0525,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.125,
      "rewards/margins": 9.6875,
      "rewards/rejected": -29.75,
      "step": 9280
    },
    {
      "epoch": 1.1872963128634417,
      "grad_norm": 1.7774031365809333,
      "learning_rate": 8.300077165138592e-07,
      "logits/chosen": -0.61328125,
      "logits/rejected": 0.3046875,
      "logps/chosen": -478.0,
      "logps/rejected": -524.0,
      "loss": 0.0594,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.0,
      "rewards/margins": 10.125,
      "rewards/rejected": -29.125,
      "step": 9290
    },
    {
      "epoch": 1.1885743497987091,
      "grad_norm": 5.250921899443943,
      "learning_rate": 8.295613557843402e-07,
      "logits/chosen": -0.458984375,
      "logits/rejected": 0.1767578125,
      "logps/chosen": -470.0,
      "logps/rejected": -544.0,
      "loss": 0.0744,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.125,
      "rewards/margins": 10.125,
      "rewards/rejected": -29.25,
      "step": 9300
    },
    {
      "epoch": 1.1898523867339765,
      "grad_norm": 6.654507220297743,
      "learning_rate": 8.291157144114419e-07,
      "logits/chosen": -0.474609375,
      "logits/rejected": 0.32421875,
      "logps/chosen": -456.0,
      "logps/rejected": -504.0,
      "loss": 0.0611,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.0,
      "rewards/margins": 9.9375,
      "rewards/rejected": -27.875,
      "step": 9310
    },
    {
      "epoch": 1.191130423669244,
      "grad_norm": 6.580511838031899,
      "learning_rate": 8.286707904650417e-07,
      "logits/chosen": -0.431640625,
      "logits/rejected": 0.291015625,
      "logps/chosen": -470.0,
      "logps/rejected": -520.0,
      "loss": 0.0641,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -20.0,
      "rewards/margins": 9.75,
      "rewards/rejected": -29.75,
      "step": 9320
    },
    {
      "epoch": 1.1924084606045116,
      "grad_norm": 5.1122608175230715,
      "learning_rate": 8.282265820222593e-07,
      "logits/chosen": -0.5859375,
      "logits/rejected": 0.236328125,
      "logps/chosen": -476.0,
      "logps/rejected": -540.0,
      "loss": 0.062,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.5,
      "rewards/margins": 10.875,
      "rewards/rejected": -31.25,
      "step": 9330
    },
    {
      "epoch": 1.193686497539779,
      "grad_norm": 4.537471794576403,
      "learning_rate": 8.277830871674222e-07,
      "logits/chosen": -0.55078125,
      "logits/rejected": 0.341796875,
      "logps/chosen": -462.0,
      "logps/rejected": -516.0,
      "loss": 0.0544,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.375,
      "rewards/margins": 9.25,
      "rewards/rejected": -28.625,
      "step": 9340
    },
    {
      "epoch": 1.1949645344750464,
      "grad_norm": 4.9598090754411395,
      "learning_rate": 8.273403039920306e-07,
      "logits/chosen": -0.48046875,
      "logits/rejected": 0.1884765625,
      "logps/chosen": -482.0,
      "logps/rejected": -528.0,
      "loss": 0.0558,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.875,
      "rewards/margins": 9.125,
      "rewards/rejected": -30.0,
      "step": 9350
    },
    {
      "epoch": 1.1962425714103138,
      "grad_norm": 6.540758097870689,
      "learning_rate": 8.268982305947231e-07,
      "logits/chosen": -0.57421875,
      "logits/rejected": 0.25,
      "logps/chosen": -490.0,
      "logps/rejected": -544.0,
      "loss": 0.0633,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.5,
      "rewards/margins": 9.125,
      "rewards/rejected": -29.625,
      "step": 9360
    },
    {
      "epoch": 1.1975206083455812,
      "grad_norm": 5.76274044053147,
      "learning_rate": 8.264568650812423e-07,
      "logits/chosen": -0.50390625,
      "logits/rejected": 0.1103515625,
      "logps/chosen": -452.0,
      "logps/rejected": -536.0,
      "loss": 0.063,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.5,
      "rewards/margins": 10.125,
      "rewards/rejected": -28.75,
      "step": 9370
    },
    {
      "epoch": 1.1987986452808486,
      "grad_norm": 4.0928531909618835,
      "learning_rate": 8.26016205564401e-07,
      "logits/chosen": -0.6015625,
      "logits/rejected": 0.26953125,
      "logps/chosen": -456.0,
      "logps/rejected": -496.0,
      "loss": 0.0728,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.875,
      "rewards/margins": 9.875,
      "rewards/rejected": -27.875,
      "step": 9380
    },
    {
      "epoch": 1.200076682216116,
      "grad_norm": 9.578041460880797,
      "learning_rate": 8.25576250164048e-07,
      "logits/chosen": -0.62890625,
      "logits/rejected": 0.1494140625,
      "logps/chosen": -450.0,
      "logps/rejected": -496.0,
      "loss": 0.0608,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.5,
      "rewards/margins": 9.25,
      "rewards/rejected": -27.625,
      "step": 9390
    },
    {
      "epoch": 1.2013547191513836,
      "grad_norm": 6.654335791859341,
      "learning_rate": 8.251369970070346e-07,
      "logits/chosen": -0.37890625,
      "logits/rejected": 0.22265625,
      "logps/chosen": -452.0,
      "logps/rejected": -512.0,
      "loss": 0.0551,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.375,
      "rewards/margins": 8.8125,
      "rewards/rejected": -28.125,
      "step": 9400
    },
    {
      "epoch": 1.202632756086651,
      "grad_norm": 4.747478854630732,
      "learning_rate": 8.246984442271813e-07,
      "logits/chosen": -0.52734375,
      "logits/rejected": 0.208984375,
      "logps/chosen": -478.0,
      "logps/rejected": -536.0,
      "loss": 0.0625,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -20.125,
      "rewards/margins": 9.3125,
      "rewards/rejected": -29.5,
      "step": 9410
    },
    {
      "epoch": 1.2039107930219184,
      "grad_norm": 4.028771554878852,
      "learning_rate": 8.242605899652435e-07,
      "logits/chosen": -0.41796875,
      "logits/rejected": 0.201171875,
      "logps/chosen": -460.0,
      "logps/rejected": -528.0,
      "loss": 0.0615,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.0,
      "rewards/margins": 9.75,
      "rewards/rejected": -28.75,
      "step": 9420
    },
    {
      "epoch": 1.2051888299571858,
      "grad_norm": 2.9128021665864794,
      "learning_rate": 8.238234323688798e-07,
      "logits/chosen": -0.408203125,
      "logits/rejected": 0.24609375,
      "logps/chosen": -476.0,
      "logps/rejected": -520.0,
      "loss": 0.0469,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.625,
      "rewards/margins": 9.5625,
      "rewards/rejected": -29.25,
      "step": 9430
    },
    {
      "epoch": 1.2064668668924532,
      "grad_norm": 3.7386526488242926,
      "learning_rate": 8.233869695926182e-07,
      "logits/chosen": -0.38671875,
      "logits/rejected": 0.326171875,
      "logps/chosen": -472.0,
      "logps/rejected": -524.0,
      "loss": 0.0775,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -19.25,
      "rewards/margins": 9.5625,
      "rewards/rejected": -28.875,
      "step": 9440
    },
    {
      "epoch": 1.2077449038277206,
      "grad_norm": 4.276130717576441,
      "learning_rate": 8.229511997978235e-07,
      "logits/chosen": -0.419921875,
      "logits/rejected": 0.1357421875,
      "logps/chosen": -466.0,
      "logps/rejected": -520.0,
      "loss": 0.0488,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.125,
      "rewards/margins": 9.5,
      "rewards/rejected": -28.625,
      "step": 9450
    },
    {
      "epoch": 1.209022940762988,
      "grad_norm": 3.891106985902184,
      "learning_rate": 8.22516121152665e-07,
      "logits/chosen": -0.53125,
      "logits/rejected": 0.421875,
      "logps/chosen": -474.0,
      "logps/rejected": -532.0,
      "loss": 0.0529,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -20.25,
      "rewards/margins": 11.125,
      "rewards/rejected": -31.375,
      "step": 9460
    },
    {
      "epoch": 1.2103009776982554,
      "grad_norm": 3.9998974604831243,
      "learning_rate": 8.220817318320836e-07,
      "logits/chosen": -0.5234375,
      "logits/rejected": 0.16015625,
      "logps/chosen": -456.0,
      "logps/rejected": -556.0,
      "loss": 0.0559,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.875,
      "rewards/margins": 9.9375,
      "rewards/rejected": -28.875,
      "step": 9470
    },
    {
      "epoch": 1.2115790146335228,
      "grad_norm": 4.0217413510176065,
      "learning_rate": 8.216480300177611e-07,
      "logits/chosen": -0.484375,
      "logits/rejected": 0.1435546875,
      "logps/chosen": -492.0,
      "logps/rejected": -548.0,
      "loss": 0.0657,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.5,
      "rewards/margins": 10.3125,
      "rewards/rejected": -30.75,
      "step": 9480
    },
    {
      "epoch": 1.2128570515687904,
      "grad_norm": 5.119112419312525,
      "learning_rate": 8.212150138980857e-07,
      "logits/chosen": -0.484375,
      "logits/rejected": 0.306640625,
      "logps/chosen": -490.0,
      "logps/rejected": -540.0,
      "loss": 0.0636,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -21.5,
      "rewards/margins": 9.875,
      "rewards/rejected": -31.375,
      "step": 9490
    },
    {
      "epoch": 1.2141350885040578,
      "grad_norm": 3.474149234640574,
      "learning_rate": 8.207826816681233e-07,
      "logits/chosen": -0.6171875,
      "logits/rejected": 0.333984375,
      "logps/chosen": -488.0,
      "logps/rejected": -520.0,
      "loss": 0.0544,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -20.0,
      "rewards/margins": 10.375,
      "rewards/rejected": -30.25,
      "step": 9500
    },
    {
      "epoch": 1.2154131254393252,
      "grad_norm": 3.334078819899853,
      "learning_rate": 8.203510315295829e-07,
      "logits/chosen": -0.55078125,
      "logits/rejected": 0.203125,
      "logps/chosen": -452.0,
      "logps/rejected": -512.0,
      "loss": 0.0619,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.375,
      "rewards/margins": 9.6875,
      "rewards/rejected": -28.125,
      "step": 9510
    },
    {
      "epoch": 1.2166911623745926,
      "grad_norm": 6.293896849399345,
      "learning_rate": 8.199200616907878e-07,
      "logits/chosen": -0.59375,
      "logits/rejected": 0.1982421875,
      "logps/chosen": -462.0,
      "logps/rejected": -528.0,
      "loss": 0.0694,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.75,
      "rewards/margins": 9.5625,
      "rewards/rejected": -28.25,
      "step": 9520
    },
    {
      "epoch": 1.21796919930986,
      "grad_norm": 7.512148174620903,
      "learning_rate": 8.194897703666421e-07,
      "logits/chosen": -0.59375,
      "logits/rejected": 0.19921875,
      "logps/chosen": -498.0,
      "logps/rejected": -548.0,
      "loss": 0.0599,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.75,
      "rewards/margins": 10.25,
      "rewards/rejected": -31.0,
      "step": 9530
    },
    {
      "epoch": 1.2192472362451274,
      "grad_norm": 6.56022432860611,
      "learning_rate": 8.190601557786015e-07,
      "logits/chosen": -0.427734375,
      "logits/rejected": 0.29296875,
      "logps/chosen": -480.0,
      "logps/rejected": -548.0,
      "loss": 0.0516,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.0,
      "rewards/margins": 10.5,
      "rewards/rejected": -31.5,
      "step": 9540
    },
    {
      "epoch": 1.2205252731803948,
      "grad_norm": 4.05692828812449,
      "learning_rate": 8.186312161546413e-07,
      "logits/chosen": -0.56640625,
      "logits/rejected": 0.37890625,
      "logps/chosen": -488.0,
      "logps/rejected": -536.0,
      "loss": 0.0575,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.0,
      "rewards/margins": 11.375,
      "rewards/rejected": -32.5,
      "step": 9550
    },
    {
      "epoch": 1.2218033101156625,
      "grad_norm": 4.677235284704719,
      "learning_rate": 8.182029497292262e-07,
      "logits/chosen": -0.5625,
      "logits/rejected": 0.12255859375,
      "logps/chosen": -468.0,
      "logps/rejected": -540.0,
      "loss": 0.0849,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.5,
      "rewards/margins": 10.25,
      "rewards/rejected": -29.75,
      "step": 9560
    },
    {
      "epoch": 1.2230813470509299,
      "grad_norm": 2.247214247247455,
      "learning_rate": 8.177753547432792e-07,
      "logits/chosen": -0.439453125,
      "logits/rejected": 0.212890625,
      "logps/chosen": -440.0,
      "logps/rejected": -516.0,
      "loss": 0.0564,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.125,
      "rewards/margins": 10.125,
      "rewards/rejected": -28.25,
      "step": 9570
    },
    {
      "epoch": 1.2243593839861973,
      "grad_norm": 3.654150203223324,
      "learning_rate": 8.173484294441524e-07,
      "logits/chosen": -0.546875,
      "logits/rejected": 0.24609375,
      "logps/chosen": -450.0,
      "logps/rejected": -506.0,
      "loss": 0.0433,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -17.625,
      "rewards/margins": 10.875,
      "rewards/rejected": -28.5,
      "step": 9580
    },
    {
      "epoch": 1.2256374209214647,
      "grad_norm": 7.999701081989942,
      "learning_rate": 8.169221720855952e-07,
      "logits/chosen": -0.5703125,
      "logits/rejected": 0.25,
      "logps/chosen": -468.0,
      "logps/rejected": -540.0,
      "loss": 0.0676,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.75,
      "rewards/margins": 9.875,
      "rewards/rejected": -28.625,
      "step": 9590
    },
    {
      "epoch": 1.226915457856732,
      "grad_norm": 9.329006183875958,
      "learning_rate": 8.164965809277261e-07,
      "logits/chosen": -0.48828125,
      "logits/rejected": 0.240234375,
      "logps/chosen": -460.0,
      "logps/rejected": -520.0,
      "loss": 0.0682,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -19.375,
      "rewards/margins": 9.6875,
      "rewards/rejected": -29.125,
      "step": 9600
    },
    {
      "epoch": 1.2281934947919995,
      "grad_norm": 4.449428339800174,
      "learning_rate": 8.160716542370011e-07,
      "logits/chosen": -0.349609375,
      "logits/rejected": 0.345703125,
      "logps/chosen": -480.0,
      "logps/rejected": -516.0,
      "loss": 0.0606,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.875,
      "rewards/margins": 9.3125,
      "rewards/rejected": -29.25,
      "step": 9610
    },
    {
      "epoch": 1.2294715317272669,
      "grad_norm": 1.989978885743653,
      "learning_rate": 8.156473902861856e-07,
      "logits/chosen": -0.3515625,
      "logits/rejected": 0.416015625,
      "logps/chosen": -448.0,
      "logps/rejected": -492.0,
      "loss": 0.0437,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.375,
      "rewards/margins": 8.9375,
      "rewards/rejected": -28.25,
      "step": 9620
    },
    {
      "epoch": 1.2307495686625343,
      "grad_norm": 10.482397340928834,
      "learning_rate": 8.152237873543241e-07,
      "logits/chosen": -0.5078125,
      "logits/rejected": 0.23828125,
      "logps/chosen": -470.0,
      "logps/rejected": -516.0,
      "loss": 0.0612,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.5,
      "rewards/margins": 9.25,
      "rewards/rejected": -27.75,
      "step": 9630
    },
    {
      "epoch": 1.2320276055978017,
      "grad_norm": 10.784843269615523,
      "learning_rate": 8.148008437267104e-07,
      "logits/chosen": -0.39453125,
      "logits/rejected": 0.1796875,
      "logps/chosen": -466.0,
      "logps/rejected": -540.0,
      "loss": 0.0737,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.375,
      "rewards/margins": 9.4375,
      "rewards/rejected": -28.75,
      "step": 9640
    },
    {
      "epoch": 1.2333056425330693,
      "grad_norm": 10.77060662289928,
      "learning_rate": 8.143785576948602e-07,
      "logits/chosen": -0.52734375,
      "logits/rejected": 0.119140625,
      "logps/chosen": -488.0,
      "logps/rejected": -548.0,
      "loss": 0.0859,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.625,
      "rewards/margins": 10.375,
      "rewards/rejected": -30.0,
      "step": 9650
    },
    {
      "epoch": 1.2345836794683367,
      "grad_norm": 7.039660750407491,
      "learning_rate": 8.139569275564796e-07,
      "logits/chosen": -0.341796875,
      "logits/rejected": 0.220703125,
      "logps/chosen": -482.0,
      "logps/rejected": -508.0,
      "loss": 0.0554,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.5,
      "rewards/margins": 9.3125,
      "rewards/rejected": -28.875,
      "step": 9660
    },
    {
      "epoch": 1.235861716403604,
      "grad_norm": 5.086668485801837,
      "learning_rate": 8.135359516154388e-07,
      "logits/chosen": -0.6640625,
      "logits/rejected": 0.1943359375,
      "logps/chosen": -458.0,
      "logps/rejected": -490.0,
      "loss": 0.0512,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.125,
      "rewards/margins": 9.5,
      "rewards/rejected": -27.625,
      "step": 9670
    },
    {
      "epoch": 1.2371397533388715,
      "grad_norm": 5.872106383242348,
      "learning_rate": 8.131156281817418e-07,
      "logits/chosen": -0.5546875,
      "logits/rejected": 0.330078125,
      "logps/chosen": -456.0,
      "logps/rejected": -490.0,
      "loss": 0.0564,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.125,
      "rewards/margins": 9.3125,
      "rewards/rejected": -28.375,
      "step": 9680
    },
    {
      "epoch": 1.238417790274139,
      "grad_norm": 5.473469187940654,
      "learning_rate": 8.126959555714979e-07,
      "logits/chosen": -0.4765625,
      "logits/rejected": 0.27734375,
      "logps/chosen": -438.0,
      "logps/rejected": -512.0,
      "loss": 0.0609,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -18.875,
      "rewards/margins": 9.8125,
      "rewards/rejected": -28.625,
      "step": 9690
    },
    {
      "epoch": 1.2396958272094063,
      "grad_norm": 2.4967203266657756,
      "learning_rate": 8.122769321068952e-07,
      "logits/chosen": -0.47265625,
      "logits/rejected": 0.1044921875,
      "logps/chosen": -430.0,
      "logps/rejected": -516.0,
      "loss": 0.0603,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.125,
      "rewards/margins": 9.125,
      "rewards/rejected": -27.25,
      "step": 9700
    },
    {
      "epoch": 1.2409738641446737,
      "grad_norm": 8.31230928522874,
      "learning_rate": 8.118585561161698e-07,
      "logits/chosen": -0.470703125,
      "logits/rejected": 0.0947265625,
      "logps/chosen": -440.0,
      "logps/rejected": -490.0,
      "loss": 0.0817,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.25,
      "rewards/margins": 8.5,
      "rewards/rejected": -25.75,
      "step": 9710
    },
    {
      "epoch": 1.2422519010799413,
      "grad_norm": 6.093861945485736,
      "learning_rate": 8.114408259335793e-07,
      "logits/chosen": -0.353515625,
      "logits/rejected": 0.1904296875,
      "logps/chosen": -444.0,
      "logps/rejected": -508.0,
      "loss": 0.0584,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.25,
      "rewards/margins": 9.4375,
      "rewards/rejected": -27.625,
      "step": 9720
    },
    {
      "epoch": 1.2435299380152087,
      "grad_norm": 5.334491339066769,
      "learning_rate": 8.110237398993754e-07,
      "logits/chosen": -0.44921875,
      "logits/rejected": 0.2265625,
      "logps/chosen": -442.0,
      "logps/rejected": -508.0,
      "loss": 0.0566,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.5,
      "rewards/margins": 9.9375,
      "rewards/rejected": -27.375,
      "step": 9730
    },
    {
      "epoch": 1.2448079749504761,
      "grad_norm": 3.537917092561329,
      "learning_rate": 8.106072963597751e-07,
      "logits/chosen": -0.44140625,
      "logits/rejected": 0.263671875,
      "logps/chosen": -454.0,
      "logps/rejected": -504.0,
      "loss": 0.0501,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.5,
      "rewards/margins": 9.0625,
      "rewards/rejected": -26.625,
      "step": 9740
    },
    {
      "epoch": 1.2460860118857435,
      "grad_norm": 5.173718162485858,
      "learning_rate": 8.101914936669332e-07,
      "logits/chosen": -0.375,
      "logits/rejected": 0.216796875,
      "logps/chosen": -434.0,
      "logps/rejected": -500.0,
      "loss": 0.0672,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.0,
      "rewards/margins": 9.5625,
      "rewards/rejected": -26.5,
      "step": 9750
    },
    {
      "epoch": 1.247364048821011,
      "grad_norm": 7.617426116385524,
      "learning_rate": 8.09776330178916e-07,
      "logits/chosen": -0.4296875,
      "logits/rejected": 0.236328125,
      "logps/chosen": -450.0,
      "logps/rejected": -488.0,
      "loss": 0.0635,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -16.875,
      "rewards/margins": 9.4375,
      "rewards/rejected": -26.375,
      "step": 9760
    },
    {
      "epoch": 1.2486420857562783,
      "grad_norm": 9.919217834732802,
      "learning_rate": 8.093618042596727e-07,
      "logits/chosen": -0.5078125,
      "logits/rejected": 0.1728515625,
      "logps/chosen": -466.0,
      "logps/rejected": -502.0,
      "loss": 0.0783,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.5,
      "rewards/margins": 8.6875,
      "rewards/rejected": -27.25,
      "step": 9770
    },
    {
      "epoch": 1.2499201226915457,
      "grad_norm": 7.390065807731816,
      "learning_rate": 8.089479142790095e-07,
      "logits/chosen": -0.55859375,
      "logits/rejected": -0.0279541015625,
      "logps/chosen": -450.0,
      "logps/rejected": -540.0,
      "loss": 0.0461,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.5,
      "rewards/margins": 10.0,
      "rewards/rejected": -28.5,
      "step": 9780
    },
    {
      "epoch": 1.2511981596268131,
      "grad_norm": 12.319077087520702,
      "learning_rate": 8.085346586125621e-07,
      "logits/chosen": -0.400390625,
      "logits/rejected": 0.271484375,
      "logps/chosen": -452.0,
      "logps/rejected": -510.0,
      "loss": 0.0658,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.375,
      "rewards/margins": 9.5,
      "rewards/rejected": -27.875,
      "step": 9790
    },
    {
      "epoch": 1.2524761965620805,
      "grad_norm": 5.883873197494254,
      "learning_rate": 8.081220356417685e-07,
      "logits/chosen": -0.353515625,
      "logits/rejected": 0.326171875,
      "logps/chosen": -420.0,
      "logps/rejected": -496.0,
      "loss": 0.0671,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.0,
      "rewards/margins": 9.6875,
      "rewards/rejected": -27.75,
      "step": 9800
    },
    {
      "epoch": 1.253754233497348,
      "grad_norm": 7.731046768460738,
      "learning_rate": 8.077100437538435e-07,
      "logits/chosen": -0.45703125,
      "logits/rejected": 0.1884765625,
      "logps/chosen": -448.0,
      "logps/rejected": -516.0,
      "loss": 0.0774,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.25,
      "rewards/margins": 9.1875,
      "rewards/rejected": -27.375,
      "step": 9810
    },
    {
      "epoch": 1.2550322704326156,
      "grad_norm": 5.989471612565628,
      "learning_rate": 8.072986813417512e-07,
      "logits/chosen": -0.4609375,
      "logits/rejected": 0.18359375,
      "logps/chosen": -438.0,
      "logps/rejected": -506.0,
      "loss": 0.0779,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.375,
      "rewards/margins": 9.8125,
      "rewards/rejected": -27.125,
      "step": 9820
    },
    {
      "epoch": 1.256310307367883,
      "grad_norm": 2.4772085763062632,
      "learning_rate": 8.068879468041791e-07,
      "logits/chosen": -0.498046875,
      "logits/rejected": 0.1943359375,
      "logps/chosen": -440.0,
      "logps/rejected": -504.0,
      "loss": 0.0623,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.5,
      "rewards/margins": 9.1875,
      "rewards/rejected": -26.75,
      "step": 9830
    },
    {
      "epoch": 1.2575883443031504,
      "grad_norm": 3.282874266574362,
      "learning_rate": 8.064778385455118e-07,
      "logits/chosen": -0.5859375,
      "logits/rejected": 0.22265625,
      "logps/chosen": -470.0,
      "logps/rejected": -524.0,
      "loss": 0.0503,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.625,
      "rewards/margins": 10.1875,
      "rewards/rejected": -28.75,
      "step": 9840
    },
    {
      "epoch": 1.2588663812384178,
      "grad_norm": 4.397037140766637,
      "learning_rate": 8.060683549758054e-07,
      "logits/chosen": -0.5625,
      "logits/rejected": 0.1416015625,
      "logps/chosen": -458.0,
      "logps/rejected": -520.0,
      "loss": 0.0683,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.375,
      "rewards/margins": 9.5,
      "rewards/rejected": -28.875,
      "step": 9850
    },
    {
      "epoch": 1.2601444181736852,
      "grad_norm": 5.167626978608734,
      "learning_rate": 8.056594945107608e-07,
      "logits/chosen": -0.71875,
      "logits/rejected": 0.10595703125,
      "logps/chosen": -476.0,
      "logps/rejected": -532.0,
      "loss": 0.0574,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.375,
      "rewards/margins": 9.625,
      "rewards/rejected": -29.0,
      "step": 9860
    },
    {
      "epoch": 1.2614224551089526,
      "grad_norm": 8.132393200954484,
      "learning_rate": 8.052512555716987e-07,
      "logits/chosen": -0.6328125,
      "logits/rejected": 0.05615234375,
      "logps/chosen": -466.0,
      "logps/rejected": -520.0,
      "loss": 0.078,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.375,
      "rewards/margins": 9.75,
      "rewards/rejected": -28.125,
      "step": 9870
    },
    {
      "epoch": 1.2627004920442202,
      "grad_norm": 4.892431987232012,
      "learning_rate": 8.048436365855337e-07,
      "logits/chosen": -0.53125,
      "logits/rejected": 0.16796875,
      "logps/chosen": -434.0,
      "logps/rejected": -504.0,
      "loss": 0.0679,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.125,
      "rewards/margins": 9.1875,
      "rewards/rejected": -26.25,
      "step": 9880
    },
    {
      "epoch": 1.2639785289794876,
      "grad_norm": 7.2081247289812795,
      "learning_rate": 8.044366359847486e-07,
      "logits/chosen": -0.5546875,
      "logits/rejected": 0.11669921875,
      "logps/chosen": -476.0,
      "logps/rejected": -516.0,
      "loss": 0.052,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.5,
      "rewards/margins": 9.375,
      "rewards/rejected": -27.875,
      "step": 9890
    },
    {
      "epoch": 1.265256565914755,
      "grad_norm": 3.4927618789241013,
      "learning_rate": 8.040302522073696e-07,
      "logits/chosen": -0.50390625,
      "logits/rejected": 0.275390625,
      "logps/chosen": -456.0,
      "logps/rejected": -496.0,
      "loss": 0.0632,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.875,
      "rewards/margins": 9.25,
      "rewards/rejected": -27.125,
      "step": 9900
    },
    {
      "epoch": 1.2665346028500224,
      "grad_norm": 8.607930741692055,
      "learning_rate": 8.036244836969407e-07,
      "logits/chosen": -0.451171875,
      "logits/rejected": 0.06982421875,
      "logps/chosen": -462.0,
      "logps/rejected": -528.0,
      "loss": 0.0742,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.25,
      "rewards/margins": 9.375,
      "rewards/rejected": -27.625,
      "step": 9910
    },
    {
      "epoch": 1.2678126397852898,
      "grad_norm": 6.390313002080005,
      "learning_rate": 8.032193289024989e-07,
      "logits/chosen": -0.412109375,
      "logits/rejected": 0.158203125,
      "logps/chosen": -426.0,
      "logps/rejected": -500.0,
      "loss": 0.0783,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.25,
      "rewards/margins": 9.375,
      "rewards/rejected": -26.625,
      "step": 9920
    },
    {
      "epoch": 1.2690906767205572,
      "grad_norm": 6.504653979588834,
      "learning_rate": 8.02814786278549e-07,
      "logits/chosen": -0.443359375,
      "logits/rejected": 0.283203125,
      "logps/chosen": -444.0,
      "logps/rejected": -486.0,
      "loss": 0.0525,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.625,
      "rewards/margins": 9.3125,
      "rewards/rejected": -26.875,
      "step": 9930
    },
    {
      "epoch": 1.2703687136558246,
      "grad_norm": 5.105672552998424,
      "learning_rate": 8.024108542850394e-07,
      "logits/chosen": -0.51171875,
      "logits/rejected": 0.09130859375,
      "logps/chosen": -434.0,
      "logps/rejected": -500.0,
      "loss": 0.0676,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -16.625,
      "rewards/margins": 8.875,
      "rewards/rejected": -25.5,
      "step": 9940
    },
    {
      "epoch": 1.271646750591092,
      "grad_norm": 3.9222731673823,
      "learning_rate": 8.020075313873367e-07,
      "logits/chosen": -0.4296875,
      "logits/rejected": 0.068359375,
      "logps/chosen": -430.0,
      "logps/rejected": -500.0,
      "loss": 0.0705,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.75,
      "rewards/margins": 8.625,
      "rewards/rejected": -25.375,
      "step": 9950
    },
    {
      "epoch": 1.2729247875263594,
      "grad_norm": 3.3355011946354827,
      "learning_rate": 8.016048160562023e-07,
      "logits/chosen": -0.5234375,
      "logits/rejected": 0.068359375,
      "logps/chosen": -470.0,
      "logps/rejected": -508.0,
      "loss": 0.0525,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.0,
      "rewards/margins": 9.25,
      "rewards/rejected": -26.25,
      "step": 9960
    },
    {
      "epoch": 1.2742028244616268,
      "grad_norm": 3.3189371026343353,
      "learning_rate": 8.012027067677667e-07,
      "logits/chosen": -0.490234375,
      "logits/rejected": 0.1826171875,
      "logps/chosen": -466.0,
      "logps/rejected": -496.0,
      "loss": 0.0476,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.5,
      "rewards/margins": 9.375,
      "rewards/rejected": -25.875,
      "step": 9970
    },
    {
      "epoch": 1.2754808613968944,
      "grad_norm": 3.1376450030393,
      "learning_rate": 8.008012020035062e-07,
      "logits/chosen": -0.494140625,
      "logits/rejected": 0.1533203125,
      "logps/chosen": -448.0,
      "logps/rejected": -500.0,
      "loss": 0.0555,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.25,
      "rewards/margins": 9.6875,
      "rewards/rejected": -27.0,
      "step": 9980
    },
    {
      "epoch": 1.2767588983321618,
      "grad_norm": 5.703482799004608,
      "learning_rate": 8.004003002502188e-07,
      "logits/chosen": -0.419921875,
      "logits/rejected": 0.1259765625,
      "logps/chosen": -462.0,
      "logps/rejected": -536.0,
      "loss": 0.0563,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.875,
      "rewards/margins": 10.1875,
      "rewards/rejected": -29.125,
      "step": 9990
    },
    {
      "epoch": 1.2780369352674292,
      "grad_norm": 7.4026837514203745,
      "learning_rate": 8e-07,
      "logits/chosen": -0.52734375,
      "logits/rejected": 0.087890625,
      "logps/chosen": -464.0,
      "logps/rejected": -494.0,
      "loss": 0.0686,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.375,
      "rewards/margins": 9.75,
      "rewards/rejected": -27.0,
      "step": 10000
    },
    {
      "epoch": 1.2780369352674292,
      "eval_logits/chosen": -0.494140625,
      "eval_logits/rejected": 0.12255859375,
      "eval_logps/chosen": -442.0,
      "eval_logps/rejected": -478.0,
      "eval_loss": 0.2714194059371948,
      "eval_rewards/accuracies": 0.897897481918335,
      "eval_rewards/chosen": -17.5,
      "eval_rewards/margins": 7.3125,
      "eval_rewards/rejected": -24.875,
      "eval_runtime": 1499.9181,
      "eval_samples_per_second": 37.006,
      "eval_steps_per_second": 0.579,
      "step": 10000
    },
    {
      "epoch": 1.2793149722026966,
      "grad_norm": 5.766140040325809,
      "learning_rate": 7.996002997502185e-07,
      "logits/chosen": -0.50390625,
      "logits/rejected": 0.09619140625,
      "logps/chosen": -458.0,
      "logps/rejected": -504.0,
      "loss": 0.0607,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.5,
      "rewards/margins": 9.875,
      "rewards/rejected": -27.375,
      "step": 10010
    },
    {
      "epoch": 1.280593009137964,
      "grad_norm": 7.7310825358181,
      "learning_rate": 7.992011980034937e-07,
      "logits/chosen": -0.55078125,
      "logits/rejected": 0.1962890625,
      "logps/chosen": -444.0,
      "logps/rejected": -508.0,
      "loss": 0.0738,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.875,
      "rewards/margins": 9.8125,
      "rewards/rejected": -28.75,
      "step": 10020
    },
    {
      "epoch": 1.2818710460732314,
      "grad_norm": 8.620455900690697,
      "learning_rate": 7.98802693267671e-07,
      "logits/chosen": -0.408203125,
      "logits/rejected": 0.3046875,
      "logps/chosen": -458.0,
      "logps/rejected": -528.0,
      "loss": 0.066,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.625,
      "rewards/margins": 9.9375,
      "rewards/rejected": -29.5,
      "step": 10030
    },
    {
      "epoch": 1.283149083008499,
      "grad_norm": 8.057339965915086,
      "learning_rate": 7.984047840557992e-07,
      "logits/chosen": -0.486328125,
      "logits/rejected": 0.1982421875,
      "logps/chosen": -446.0,
      "logps/rejected": -528.0,
      "loss": 0.0686,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.25,
      "rewards/margins": 9.375,
      "rewards/rejected": -28.625,
      "step": 10040
    },
    {
      "epoch": 1.2844271199437665,
      "grad_norm": 4.21713497125553,
      "learning_rate": 7.980074688861063e-07,
      "logits/chosen": -0.62109375,
      "logits/rejected": 0.050537109375,
      "logps/chosen": -432.0,
      "logps/rejected": -508.0,
      "loss": 0.0504,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.25,
      "rewards/margins": 9.5,
      "rewards/rejected": -26.75,
      "step": 10050
    },
    {
      "epoch": 1.2857051568790339,
      "grad_norm": 1.6523830769610441,
      "learning_rate": 7.976107462819775e-07,
      "logits/chosen": -0.609375,
      "logits/rejected": 0.006500244140625,
      "logps/chosen": -468.0,
      "logps/rejected": -520.0,
      "loss": 0.0571,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -18.375,
      "rewards/margins": 10.3125,
      "rewards/rejected": -28.75,
      "step": 10060
    },
    {
      "epoch": 1.2869831938143013,
      "grad_norm": 5.527960346952958,
      "learning_rate": 7.97214614771931e-07,
      "logits/chosen": -0.6015625,
      "logits/rejected": 0.12109375,
      "logps/chosen": -442.0,
      "logps/rejected": -498.0,
      "loss": 0.0507,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.0,
      "rewards/margins": 10.0625,
      "rewards/rejected": -27.125,
      "step": 10070
    },
    {
      "epoch": 1.2882612307495687,
      "grad_norm": 7.205348354844139,
      "learning_rate": 7.968190728895957e-07,
      "logits/chosen": -0.58203125,
      "logits/rejected": 0.07568359375,
      "logps/chosen": -452.0,
      "logps/rejected": -508.0,
      "loss": 0.0765,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.75,
      "rewards/margins": 10.125,
      "rewards/rejected": -27.875,
      "step": 10080
    },
    {
      "epoch": 1.289539267684836,
      "grad_norm": 7.937229811325548,
      "learning_rate": 7.964241191736886e-07,
      "logits/chosen": -0.65234375,
      "logits/rejected": -0.03125,
      "logps/chosen": -426.0,
      "logps/rejected": -488.0,
      "loss": 0.0938,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.375,
      "rewards/margins": 9.125,
      "rewards/rejected": -25.5,
      "step": 10090
    },
    {
      "epoch": 1.2908173046201035,
      "grad_norm": 5.467030353932639,
      "learning_rate": 7.960297521679913e-07,
      "logits/chosen": -0.451171875,
      "logits/rejected": 0.09912109375,
      "logps/chosen": -394.0,
      "logps/rejected": -472.0,
      "loss": 0.0813,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.25,
      "rewards/margins": 9.6875,
      "rewards/rejected": -25.0,
      "step": 10100
    },
    {
      "epoch": 1.2920953415553709,
      "grad_norm": 5.849124095016742,
      "learning_rate": 7.956359704213283e-07,
      "logits/chosen": -0.63671875,
      "logits/rejected": 0.1513671875,
      "logps/chosen": -446.0,
      "logps/rejected": -492.0,
      "loss": 0.0591,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.5,
      "rewards/margins": 9.5,
      "rewards/rejected": -27.0,
      "step": 10110
    },
    {
      "epoch": 1.2933733784906383,
      "grad_norm": 9.510277546191345,
      "learning_rate": 7.95242772487544e-07,
      "logits/chosen": -0.54296875,
      "logits/rejected": 0.28515625,
      "logps/chosen": -456.0,
      "logps/rejected": -490.0,
      "loss": 0.0589,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -16.5,
      "rewards/margins": 10.3125,
      "rewards/rejected": -26.875,
      "step": 10120
    },
    {
      "epoch": 1.2946514154259057,
      "grad_norm": 3.5162596203308984,
      "learning_rate": 7.94850156925481e-07,
      "logits/chosen": -0.5234375,
      "logits/rejected": 0.08642578125,
      "logps/chosen": -442.0,
      "logps/rejected": -512.0,
      "loss": 0.0609,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.375,
      "rewards/margins": 10.0,
      "rewards/rejected": -27.375,
      "step": 10130
    },
    {
      "epoch": 1.2959294523611733,
      "grad_norm": 3.847966726562354,
      "learning_rate": 7.944581222989574e-07,
      "logits/chosen": -0.50390625,
      "logits/rejected": 0.0908203125,
      "logps/chosen": -458.0,
      "logps/rejected": -516.0,
      "loss": 0.0504,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.875,
      "rewards/margins": 9.8125,
      "rewards/rejected": -27.625,
      "step": 10140
    },
    {
      "epoch": 1.2972074892964407,
      "grad_norm": 3.0942192697005804,
      "learning_rate": 7.940666671767441e-07,
      "logits/chosen": -0.51953125,
      "logits/rejected": 0.09033203125,
      "logps/chosen": -452.0,
      "logps/rejected": -540.0,
      "loss": 0.0577,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.0,
      "rewards/margins": 9.6875,
      "rewards/rejected": -27.625,
      "step": 10150
    },
    {
      "epoch": 1.298485526231708,
      "grad_norm": 2.4484028668975943,
      "learning_rate": 7.936757901325451e-07,
      "logits/chosen": -0.52734375,
      "logits/rejected": 0.197265625,
      "logps/chosen": -440.0,
      "logps/rejected": -508.0,
      "loss": 0.0563,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -16.75,
      "rewards/margins": 10.25,
      "rewards/rejected": -27.0,
      "step": 10160
    },
    {
      "epoch": 1.2997635631669755,
      "grad_norm": 8.078899034890208,
      "learning_rate": 7.932854897449727e-07,
      "logits/chosen": -0.412109375,
      "logits/rejected": 0.296875,
      "logps/chosen": -454.0,
      "logps/rejected": -512.0,
      "loss": 0.0715,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.5,
      "rewards/margins": 10.0,
      "rewards/rejected": -28.5,
      "step": 10170
    },
    {
      "epoch": 1.301041600102243,
      "grad_norm": 4.200652791417481,
      "learning_rate": 7.928957645975286e-07,
      "logits/chosen": -0.39453125,
      "logits/rejected": 0.28515625,
      "logps/chosen": -472.0,
      "logps/rejected": -492.0,
      "loss": 0.0647,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.875,
      "rewards/margins": 9.625,
      "rewards/rejected": -26.5,
      "step": 10180
    },
    {
      "epoch": 1.3023196370375103,
      "grad_norm": 4.501345895493102,
      "learning_rate": 7.925066132785799e-07,
      "logits/chosen": -0.423828125,
      "logits/rejected": 0.240234375,
      "logps/chosen": -418.0,
      "logps/rejected": -476.0,
      "loss": 0.0693,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.0,
      "rewards/margins": 8.6875,
      "rewards/rejected": -24.75,
      "step": 10190
    },
    {
      "epoch": 1.303597673972778,
      "grad_norm": 5.791692215236792,
      "learning_rate": 7.921180343813395e-07,
      "logits/chosen": -0.47265625,
      "logits/rejected": 0.2021484375,
      "logps/chosen": -440.0,
      "logps/rejected": -478.0,
      "loss": 0.046,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.75,
      "rewards/margins": 9.3125,
      "rewards/rejected": -26.125,
      "step": 10200
    },
    {
      "epoch": 1.3048757109080453,
      "grad_norm": 2.9821493618976613,
      "learning_rate": 7.917300265038436e-07,
      "logits/chosen": -0.484375,
      "logits/rejected": 0.2255859375,
      "logps/chosen": -454.0,
      "logps/rejected": -506.0,
      "loss": 0.0549,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -17.5,
      "rewards/margins": 9.6875,
      "rewards/rejected": -27.25,
      "step": 10210
    },
    {
      "epoch": 1.3061537478433127,
      "grad_norm": 3.3409028097450646,
      "learning_rate": 7.913425882489307e-07,
      "logits/chosen": -0.4921875,
      "logits/rejected": 0.2001953125,
      "logps/chosen": -424.0,
      "logps/rejected": -492.0,
      "loss": 0.0529,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.75,
      "rewards/margins": 8.75,
      "rewards/rejected": -26.5,
      "step": 10220
    },
    {
      "epoch": 1.3074317847785801,
      "grad_norm": 7.2331317340901435,
      "learning_rate": 7.909557182242211e-07,
      "logits/chosen": -0.3984375,
      "logits/rejected": 0.1787109375,
      "logps/chosen": -442.0,
      "logps/rejected": -502.0,
      "loss": 0.0609,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -16.625,
      "rewards/margins": 9.875,
      "rewards/rejected": -26.5,
      "step": 10230
    },
    {
      "epoch": 1.3087098217138475,
      "grad_norm": 2.9362421893434796,
      "learning_rate": 7.905694150420947e-07,
      "logits/chosen": -0.376953125,
      "logits/rejected": 0.248046875,
      "logps/chosen": -458.0,
      "logps/rejected": -516.0,
      "loss": 0.052,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.25,
      "rewards/margins": 9.125,
      "rewards/rejected": -28.375,
      "step": 10240
    },
    {
      "epoch": 1.309987858649115,
      "grad_norm": 6.608764110812264,
      "learning_rate": 7.901836773196717e-07,
      "logits/chosen": -0.4609375,
      "logits/rejected": 0.045654296875,
      "logps/chosen": -492.0,
      "logps/rejected": -532.0,
      "loss": 0.0577,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.75,
      "rewards/margins": 9.1875,
      "rewards/rejected": -28.0,
      "step": 10250
    },
    {
      "epoch": 1.3112658955843823,
      "grad_norm": 2.058001749831141,
      "learning_rate": 7.897985036787898e-07,
      "logits/chosen": -0.412109375,
      "logits/rejected": 0.2890625,
      "logps/chosen": -454.0,
      "logps/rejected": -520.0,
      "loss": 0.0624,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.5,
      "rewards/margins": 9.8125,
      "rewards/rejected": -29.25,
      "step": 10260
    },
    {
      "epoch": 1.3125439325196497,
      "grad_norm": 3.4426888064300107,
      "learning_rate": 7.894138927459853e-07,
      "logits/chosen": -0.51171875,
      "logits/rejected": 0.2392578125,
      "logps/chosen": -470.0,
      "logps/rejected": -556.0,
      "loss": 0.0482,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -19.375,
      "rewards/margins": 10.0,
      "rewards/rejected": -29.375,
      "step": 10270
    },
    {
      "epoch": 1.3138219694549171,
      "grad_norm": 4.423800572689863,
      "learning_rate": 7.890298431524716e-07,
      "logits/chosen": -0.53125,
      "logits/rejected": 0.1015625,
      "logps/chosen": -432.0,
      "logps/rejected": -512.0,
      "loss": 0.0548,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.5,
      "rewards/margins": 9.875,
      "rewards/rejected": -26.375,
      "step": 10280
    },
    {
      "epoch": 1.3151000063901845,
      "grad_norm": 7.590223860458519,
      "learning_rate": 7.88646353534119e-07,
      "logits/chosen": -0.494140625,
      "logits/rejected": 0.388671875,
      "logps/chosen": -436.0,
      "logps/rejected": -504.0,
      "loss": 0.0496,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -17.0,
      "rewards/margins": 10.1875,
      "rewards/rejected": -27.25,
      "step": 10290
    },
    {
      "epoch": 1.3163780433254522,
      "grad_norm": 9.20056604444523,
      "learning_rate": 7.882634225314345e-07,
      "logits/chosen": -0.50390625,
      "logits/rejected": 0.057861328125,
      "logps/chosen": -424.0,
      "logps/rejected": -488.0,
      "loss": 0.0617,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.375,
      "rewards/margins": 9.6875,
      "rewards/rejected": -25.0,
      "step": 10300
    },
    {
      "epoch": 1.3176560802607196,
      "grad_norm": 2.7396826167795774,
      "learning_rate": 7.87881048789541e-07,
      "logits/chosen": -0.478515625,
      "logits/rejected": 0.2109375,
      "logps/chosen": -426.0,
      "logps/rejected": -482.0,
      "loss": 0.0468,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -16.125,
      "rewards/margins": 9.0625,
      "rewards/rejected": -25.25,
      "step": 10310
    },
    {
      "epoch": 1.318934117195987,
      "grad_norm": 6.539166783866857,
      "learning_rate": 7.874992309581578e-07,
      "logits/chosen": -0.4375,
      "logits/rejected": 0.2119140625,
      "logps/chosen": -440.0,
      "logps/rejected": -494.0,
      "loss": 0.05,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.125,
      "rewards/margins": 9.4375,
      "rewards/rejected": -26.5,
      "step": 10320
    },
    {
      "epoch": 1.3202121541312544,
      "grad_norm": 5.070092653669181,
      "learning_rate": 7.871179676915801e-07,
      "logits/chosen": -0.484375,
      "logits/rejected": 0.1591796875,
      "logps/chosen": -436.0,
      "logps/rejected": -496.0,
      "loss": 0.077,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -16.875,
      "rewards/margins": 9.25,
      "rewards/rejected": -26.125,
      "step": 10330
    },
    {
      "epoch": 1.3214901910665218,
      "grad_norm": 7.42191664838274,
      "learning_rate": 7.867372576486597e-07,
      "logits/chosen": -0.466796875,
      "logits/rejected": 0.23046875,
      "logps/chosen": -460.0,
      "logps/rejected": -524.0,
      "loss": 0.0778,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.125,
      "rewards/margins": 9.75,
      "rewards/rejected": -26.875,
      "step": 10340
    },
    {
      "epoch": 1.3227682280017892,
      "grad_norm": 6.324871551744725,
      "learning_rate": 7.863570994927847e-07,
      "logits/chosen": -0.5390625,
      "logits/rejected": 0.1318359375,
      "logps/chosen": -480.0,
      "logps/rejected": -548.0,
      "loss": 0.0489,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.625,
      "rewards/margins": 9.75,
      "rewards/rejected": -28.375,
      "step": 10350
    },
    {
      "epoch": 1.3240462649370568,
      "grad_norm": 5.94211998050712,
      "learning_rate": 7.859774918918594e-07,
      "logits/chosen": -0.498046875,
      "logits/rejected": 0.279296875,
      "logps/chosen": -466.0,
      "logps/rejected": -506.0,
      "loss": 0.0662,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.375,
      "rewards/margins": 9.6875,
      "rewards/rejected": -28.125,
      "step": 10360
    },
    {
      "epoch": 1.3253243018723242,
      "grad_norm": 5.692370271069355,
      "learning_rate": 7.855984335182852e-07,
      "logits/chosen": -0.431640625,
      "logits/rejected": 0.185546875,
      "logps/chosen": -442.0,
      "logps/rejected": -488.0,
      "loss": 0.0641,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -16.875,
      "rewards/margins": 8.875,
      "rewards/rejected": -25.75,
      "step": 10370
    },
    {
      "epoch": 1.3266023388075916,
      "grad_norm": 6.452056510712177,
      "learning_rate": 7.852199230489422e-07,
      "logits/chosen": -0.3828125,
      "logits/rejected": 0.203125,
      "logps/chosen": -448.0,
      "logps/rejected": -496.0,
      "loss": 0.0674,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -16.875,
      "rewards/margins": 9.9375,
      "rewards/rejected": -26.875,
      "step": 10380
    },
    {
      "epoch": 1.327880375742859,
      "grad_norm": 2.101276784132653,
      "learning_rate": 7.848419591651668e-07,
      "logits/chosen": -0.435546875,
      "logits/rejected": 0.255859375,
      "logps/chosen": -490.0,
      "logps/rejected": -524.0,
      "loss": 0.0465,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.875,
      "rewards/margins": 9.1875,
      "rewards/rejected": -29.0,
      "step": 10390
    },
    {
      "epoch": 1.3291584126781264,
      "grad_norm": 6.585959680597183,
      "learning_rate": 7.844645405527361e-07,
      "logits/chosen": -0.416015625,
      "logits/rejected": 0.12158203125,
      "logps/chosen": -468.0,
      "logps/rejected": -552.0,
      "loss": 0.0666,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -20.125,
      "rewards/margins": 10.4375,
      "rewards/rejected": -30.625,
      "step": 10400
    },
    {
      "epoch": 1.3304364496133938,
      "grad_norm": 7.942934446749502,
      "learning_rate": 7.840876659018457e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": 0.333984375,
      "logps/chosen": -480.0,
      "logps/rejected": -536.0,
      "loss": 0.0577,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.0,
      "rewards/margins": 10.1875,
      "rewards/rejected": -31.25,
      "step": 10410
    },
    {
      "epoch": 1.3317144865486612,
      "grad_norm": 4.184923282670691,
      "learning_rate": 7.837113339070922e-07,
      "logits/chosen": -0.44140625,
      "logits/rejected": 0.29296875,
      "logps/chosen": -488.0,
      "logps/rejected": -552.0,
      "loss": 0.0509,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.125,
      "rewards/margins": 10.75,
      "rewards/rejected": -31.875,
      "step": 10420
    },
    {
      "epoch": 1.3329925234839286,
      "grad_norm": 9.299536578392104,
      "learning_rate": 7.833355432674538e-07,
      "logits/chosen": -0.640625,
      "logits/rejected": 0.1572265625,
      "logps/chosen": -474.0,
      "logps/rejected": -540.0,
      "loss": 0.0643,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -18.875,
      "rewards/margins": 11.125,
      "rewards/rejected": -30.0,
      "step": 10430
    },
    {
      "epoch": 1.334270560419196,
      "grad_norm": 8.70301160852477,
      "learning_rate": 7.829602926862713e-07,
      "logits/chosen": -0.53125,
      "logits/rejected": 0.150390625,
      "logps/chosen": -472.0,
      "logps/rejected": -536.0,
      "loss": 0.0591,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.25,
      "rewards/margins": 10.1875,
      "rewards/rejected": -29.375,
      "step": 10440
    },
    {
      "epoch": 1.3355485973544634,
      "grad_norm": 4.585901254936114,
      "learning_rate": 7.825855808712296e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": 0.271484375,
      "logps/chosen": -460.0,
      "logps/rejected": -520.0,
      "loss": 0.069,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.5,
      "rewards/margins": 9.0625,
      "rewards/rejected": -28.625,
      "step": 10450
    },
    {
      "epoch": 1.336826634289731,
      "grad_norm": 2.264645047501264,
      "learning_rate": 7.822114065343386e-07,
      "logits/chosen": -0.515625,
      "logits/rejected": 0.201171875,
      "logps/chosen": -482.0,
      "logps/rejected": -552.0,
      "loss": 0.0592,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -20.25,
      "rewards/margins": 10.0625,
      "rewards/rejected": -30.375,
      "step": 10460
    },
    {
      "epoch": 1.3381046712249984,
      "grad_norm": 11.690553210256036,
      "learning_rate": 7.818377683919149e-07,
      "logits/chosen": -0.50390625,
      "logits/rejected": 0.2109375,
      "logps/chosen": -470.0,
      "logps/rejected": -544.0,
      "loss": 0.0589,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.625,
      "rewards/margins": 10.0,
      "rewards/rejected": -29.625,
      "step": 10470
    },
    {
      "epoch": 1.3393827081602658,
      "grad_norm": 6.4967583089289045,
      "learning_rate": 7.814646651645635e-07,
      "logits/chosen": -0.47265625,
      "logits/rejected": 0.234375,
      "logps/chosen": -458.0,
      "logps/rejected": -528.0,
      "loss": 0.0493,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.5,
      "rewards/margins": 9.8125,
      "rewards/rejected": -29.375,
      "step": 10480
    },
    {
      "epoch": 1.3406607450955332,
      "grad_norm": 5.811995735930668,
      "learning_rate": 7.810920955771586e-07,
      "logits/chosen": -0.390625,
      "logits/rejected": 0.373046875,
      "logps/chosen": -462.0,
      "logps/rejected": -532.0,
      "loss": 0.0756,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.625,
      "rewards/margins": 10.25,
      "rewards/rejected": -29.75,
      "step": 10490
    },
    {
      "epoch": 1.3419387820308006,
      "grad_norm": 6.722408298850363,
      "learning_rate": 7.807200583588266e-07,
      "logits/chosen": -0.43359375,
      "logits/rejected": 0.16796875,
      "logps/chosen": -468.0,
      "logps/rejected": -532.0,
      "loss": 0.0689,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.25,
      "rewards/margins": 9.1875,
      "rewards/rejected": -28.5,
      "step": 10500
    },
    {
      "epoch": 1.343216818966068,
      "grad_norm": 8.070639716094007,
      "learning_rate": 7.803485522429261e-07,
      "logits/chosen": -0.5703125,
      "logits/rejected": 0.1611328125,
      "logps/chosen": -468.0,
      "logps/rejected": -532.0,
      "loss": 0.0486,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.0,
      "rewards/margins": 9.25,
      "rewards/rejected": -29.25,
      "step": 10510
    },
    {
      "epoch": 1.3444948559013357,
      "grad_norm": 3.1044051736846727,
      "learning_rate": 7.799775759670318e-07,
      "logits/chosen": -0.56640625,
      "logits/rejected": 0.259765625,
      "logps/chosen": -472.0,
      "logps/rejected": -540.0,
      "loss": 0.0548,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -20.125,
      "rewards/margins": 9.75,
      "rewards/rejected": -29.875,
      "step": 10520
    },
    {
      "epoch": 1.345772892836603,
      "grad_norm": 6.191587333986995,
      "learning_rate": 7.796071282729149e-07,
      "logits/chosen": -0.55859375,
      "logits/rejected": 0.19921875,
      "logps/chosen": -462.0,
      "logps/rejected": -524.0,
      "loss": 0.0452,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -18.875,
      "rewards/margins": 10.125,
      "rewards/rejected": -29.0,
      "step": 10530
    },
    {
      "epoch": 1.3470509297718705,
      "grad_norm": 1.6910350040796234,
      "learning_rate": 7.792372079065259e-07,
      "logits/chosen": -0.427734375,
      "logits/rejected": 0.310546875,
      "logps/chosen": -468.0,
      "logps/rejected": -524.0,
      "loss": 0.0586,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.375,
      "rewards/margins": 9.75,
      "rewards/rejected": -29.125,
      "step": 10540
    },
    {
      "epoch": 1.3483289667071379,
      "grad_norm": 5.480630834752022,
      "learning_rate": 7.788678136179767e-07,
      "logits/chosen": -0.451171875,
      "logits/rejected": 0.33984375,
      "logps/chosen": -458.0,
      "logps/rejected": -520.0,
      "loss": 0.0485,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.125,
      "rewards/margins": 9.875,
      "rewards/rejected": -29.0,
      "step": 10550
    },
    {
      "epoch": 1.3496070036424053,
      "grad_norm": 5.245839408402204,
      "learning_rate": 7.78498944161523e-07,
      "logits/chosen": -0.55859375,
      "logits/rejected": 0.3125,
      "logps/chosen": -488.0,
      "logps/rejected": -516.0,
      "loss": 0.0588,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -19.0,
      "rewards/margins": 10.625,
      "rewards/rejected": -29.625,
      "step": 10560
    },
    {
      "epoch": 1.3508850405776727,
      "grad_norm": 4.8125371644894175,
      "learning_rate": 7.781305982955459e-07,
      "logits/chosen": -0.447265625,
      "logits/rejected": 0.2451171875,
      "logps/chosen": -446.0,
      "logps/rejected": -520.0,
      "loss": 0.0583,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.125,
      "rewards/margins": 10.3125,
      "rewards/rejected": -29.5,
      "step": 10570
    },
    {
      "epoch": 1.35216307751294,
      "grad_norm": 9.765760924534012,
      "learning_rate": 7.777627747825355e-07,
      "logits/chosen": -0.443359375,
      "logits/rejected": 0.28125,
      "logps/chosen": -478.0,
      "logps/rejected": -528.0,
      "loss": 0.0739,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.375,
      "rewards/margins": 9.75,
      "rewards/rejected": -30.125,
      "step": 10580
    },
    {
      "epoch": 1.3534411144482075,
      "grad_norm": 4.379777617867407,
      "learning_rate": 7.773954723890725e-07,
      "logits/chosen": -0.5390625,
      "logits/rejected": 0.07958984375,
      "logps/chosen": -492.0,
      "logps/rejected": -552.0,
      "loss": 0.0417,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.75,
      "rewards/margins": 9.875,
      "rewards/rejected": -29.5,
      "step": 10590
    },
    {
      "epoch": 1.3547191513834749,
      "grad_norm": 5.925064899175481,
      "learning_rate": 7.770286898858113e-07,
      "logits/chosen": -0.46484375,
      "logits/rejected": 0.326171875,
      "logps/chosen": -480.0,
      "logps/rejected": -544.0,
      "loss": 0.0591,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.625,
      "rewards/margins": 9.9375,
      "rewards/rejected": -30.625,
      "step": 10600
    },
    {
      "epoch": 1.3559971883187423,
      "grad_norm": 3.1171185307255307,
      "learning_rate": 7.766624260474625e-07,
      "logits/chosen": -0.375,
      "logits/rejected": 0.244140625,
      "logps/chosen": -476.0,
      "logps/rejected": -540.0,
      "loss": 0.0489,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.25,
      "rewards/margins": 9.9375,
      "rewards/rejected": -30.125,
      "step": 10610
    },
    {
      "epoch": 1.35727522525401,
      "grad_norm": 12.403826614637596,
      "learning_rate": 7.762966796527759e-07,
      "logits/chosen": -0.419921875,
      "logits/rejected": 0.298828125,
      "logps/chosen": -458.0,
      "logps/rejected": -532.0,
      "loss": 0.0805,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.375,
      "rewards/margins": 10.0,
      "rewards/rejected": -29.375,
      "step": 10620
    },
    {
      "epoch": 1.3585532621892773,
      "grad_norm": 4.471005481796046,
      "learning_rate": 7.759314494845234e-07,
      "logits/chosen": -0.380859375,
      "logits/rejected": 0.287109375,
      "logps/chosen": -450.0,
      "logps/rejected": -488.0,
      "loss": 0.0649,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.125,
      "rewards/margins": 9.375,
      "rewards/rejected": -27.5,
      "step": 10630
    },
    {
      "epoch": 1.3598312991245447,
      "grad_norm": 2.95996348736324,
      "learning_rate": 7.755667343294812e-07,
      "logits/chosen": -0.431640625,
      "logits/rejected": 0.201171875,
      "logps/chosen": -448.0,
      "logps/rejected": -512.0,
      "loss": 0.0615,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.0,
      "rewards/margins": 9.6875,
      "rewards/rejected": -27.75,
      "step": 10640
    },
    {
      "epoch": 1.361109336059812,
      "grad_norm": 3.59846037880973,
      "learning_rate": 7.752025329784146e-07,
      "logits/chosen": -0.451171875,
      "logits/rejected": 0.1005859375,
      "logps/chosen": -422.0,
      "logps/rejected": -488.0,
      "loss": 0.0466,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.375,
      "rewards/margins": 9.25,
      "rewards/rejected": -25.625,
      "step": 10650
    },
    {
      "epoch": 1.3623873729950795,
      "grad_norm": 3.7502345949154945,
      "learning_rate": 7.748388442260596e-07,
      "logits/chosen": -0.46484375,
      "logits/rejected": 0.197265625,
      "logps/chosen": -498.0,
      "logps/rejected": -564.0,
      "loss": 0.0519,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.125,
      "rewards/margins": 11.3125,
      "rewards/rejected": -31.375,
      "step": 10660
    },
    {
      "epoch": 1.363665409930347,
      "grad_norm": 9.762916016354124,
      "learning_rate": 7.744756668711065e-07,
      "logits/chosen": -0.423828125,
      "logits/rejected": 0.31640625,
      "logps/chosen": -452.0,
      "logps/rejected": -504.0,
      "loss": 0.0824,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -20.125,
      "rewards/margins": 8.6875,
      "rewards/rejected": -28.75,
      "step": 10670
    },
    {
      "epoch": 1.3649434468656145,
      "grad_norm": 5.906319730976104,
      "learning_rate": 7.741129997161835e-07,
      "logits/chosen": -0.400390625,
      "logits/rejected": 0.251953125,
      "logps/chosen": -460.0,
      "logps/rejected": -516.0,
      "loss": 0.0477,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -18.875,
      "rewards/margins": 10.25,
      "rewards/rejected": -29.125,
      "step": 10680
    },
    {
      "epoch": 1.366221483800882,
      "grad_norm": 2.238544292735042,
      "learning_rate": 7.737508415678403e-07,
      "logits/chosen": -0.515625,
      "logits/rejected": 0.21484375,
      "logps/chosen": -454.0,
      "logps/rejected": -528.0,
      "loss": 0.0567,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.375,
      "rewards/margins": 9.375,
      "rewards/rejected": -26.75,
      "step": 10690
    },
    {
      "epoch": 1.3674995207361493,
      "grad_norm": 6.499853095762623,
      "learning_rate": 7.733891912365308e-07,
      "logits/chosen": -0.443359375,
      "logits/rejected": 0.470703125,
      "logps/chosen": -466.0,
      "logps/rejected": -516.0,
      "loss": 0.0663,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.375,
      "rewards/margins": 9.375,
      "rewards/rejected": -28.625,
      "step": 10700
    },
    {
      "epoch": 1.3687775576714167,
      "grad_norm": 4.126314100137072,
      "learning_rate": 7.730280475365979e-07,
      "logits/chosen": -0.30078125,
      "logits/rejected": 0.388671875,
      "logps/chosen": -450.0,
      "logps/rejected": -516.0,
      "loss": 0.0464,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.875,
      "rewards/margins": 9.8125,
      "rewards/rejected": -28.625,
      "step": 10710
    },
    {
      "epoch": 1.3700555946066841,
      "grad_norm": 7.944436235609646,
      "learning_rate": 7.726674092862557e-07,
      "logits/chosen": -0.4609375,
      "logits/rejected": 0.11865234375,
      "logps/chosen": -460.0,
      "logps/rejected": -516.0,
      "loss": 0.0671,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.375,
      "rewards/margins": 9.5,
      "rewards/rejected": -28.875,
      "step": 10720
    },
    {
      "epoch": 1.3713336315419515,
      "grad_norm": 11.489182316669774,
      "learning_rate": 7.723072753075748e-07,
      "logits/chosen": -0.5078125,
      "logits/rejected": 0.2314453125,
      "logps/chosen": -446.0,
      "logps/rejected": -512.0,
      "loss": 0.0552,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.625,
      "rewards/margins": 9.6875,
      "rewards/rejected": -28.375,
      "step": 10730
    },
    {
      "epoch": 1.372611668477219,
      "grad_norm": 10.531232460750358,
      "learning_rate": 7.719476444264649e-07,
      "logits/chosen": -0.353515625,
      "logits/rejected": 0.2158203125,
      "logps/chosen": -444.0,
      "logps/rejected": -528.0,
      "loss": 0.0661,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.75,
      "rewards/margins": 9.75,
      "rewards/rejected": -29.5,
      "step": 10740
    },
    {
      "epoch": 1.3738897054124863,
      "grad_norm": 7.703932925269038,
      "learning_rate": 7.715885154726593e-07,
      "logits/chosen": -0.69921875,
      "logits/rejected": 0.09814453125,
      "logps/chosen": -468.0,
      "logps/rejected": -532.0,
      "loss": 0.0493,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -20.125,
      "rewards/margins": 9.875,
      "rewards/rejected": -30.0,
      "step": 10750
    },
    {
      "epoch": 1.3751677423477537,
      "grad_norm": 4.849539799085882,
      "learning_rate": 7.71229887279699e-07,
      "logits/chosen": -0.470703125,
      "logits/rejected": 0.328125,
      "logps/chosen": -480.0,
      "logps/rejected": -568.0,
      "loss": 0.0516,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -21.375,
      "rewards/margins": 10.75,
      "rewards/rejected": -32.0,
      "step": 10760
    },
    {
      "epoch": 1.3764457792830211,
      "grad_norm": 7.737405085306414,
      "learning_rate": 7.708717586849164e-07,
      "logits/chosen": -0.55078125,
      "logits/rejected": 0.1484375,
      "logps/chosen": -482.0,
      "logps/rejected": -544.0,
      "loss": 0.0711,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -22.25,
      "rewards/margins": 9.4375,
      "rewards/rejected": -31.625,
      "step": 10770
    },
    {
      "epoch": 1.3777238162182888,
      "grad_norm": 6.013275857816496,
      "learning_rate": 7.705141285294196e-07,
      "logits/chosen": -0.482421875,
      "logits/rejected": 0.2353515625,
      "logps/chosen": -494.0,
      "logps/rejected": -568.0,
      "loss": 0.065,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -22.125,
      "rewards/margins": 11.1875,
      "rewards/rejected": -33.25,
      "step": 10780
    },
    {
      "epoch": 1.3790018531535562,
      "grad_norm": 2.5033530955705556,
      "learning_rate": 7.701569956580767e-07,
      "logits/chosen": -0.4765625,
      "logits/rejected": 0.2734375,
      "logps/chosen": -482.0,
      "logps/rejected": -544.0,
      "loss": 0.0528,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.75,
      "rewards/margins": 10.3125,
      "rewards/rejected": -31.0,
      "step": 10790
    },
    {
      "epoch": 1.3802798900888236,
      "grad_norm": 7.458600702410158,
      "learning_rate": 7.69800358919501e-07,
      "logits/chosen": -0.255859375,
      "logits/rejected": 0.3203125,
      "logps/chosen": -474.0,
      "logps/rejected": -548.0,
      "loss": 0.0614,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -21.875,
      "rewards/margins": 9.4375,
      "rewards/rejected": -31.375,
      "step": 10800
    },
    {
      "epoch": 1.381557927024091,
      "grad_norm": 2.7237080279116435,
      "learning_rate": 7.694442171660333e-07,
      "logits/chosen": -0.53125,
      "logits/rejected": 0.1015625,
      "logps/chosen": -494.0,
      "logps/rejected": -552.0,
      "loss": 0.0411,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.375,
      "rewards/margins": 10.25,
      "rewards/rejected": -29.625,
      "step": 10810
    },
    {
      "epoch": 1.3828359639593584,
      "grad_norm": 9.944625252986386,
      "learning_rate": 7.690885692537282e-07,
      "logits/chosen": -0.54296875,
      "logits/rejected": 0.373046875,
      "logps/chosen": -486.0,
      "logps/rejected": -536.0,
      "loss": 0.0706,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -20.75,
      "rewards/margins": 10.25,
      "rewards/rejected": -31.0,
      "step": 10820
    },
    {
      "epoch": 1.3841140008946258,
      "grad_norm": 10.455048548725705,
      "learning_rate": 7.687334140423383e-07,
      "logits/chosen": -0.4453125,
      "logits/rejected": 0.1435546875,
      "logps/chosen": -468.0,
      "logps/rejected": -532.0,
      "loss": 0.0737,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.625,
      "rewards/margins": 9.625,
      "rewards/rejected": -28.25,
      "step": 10830
    },
    {
      "epoch": 1.3853920378298934,
      "grad_norm": 5.193128489238999,
      "learning_rate": 7.683787503952985e-07,
      "logits/chosen": -0.318359375,
      "logits/rejected": 0.33984375,
      "logps/chosen": -468.0,
      "logps/rejected": -528.0,
      "loss": 0.0668,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -20.375,
      "rewards/margins": 9.25,
      "rewards/rejected": -29.625,
      "step": 10840
    },
    {
      "epoch": 1.3866700747651608,
      "grad_norm": 8.581472098778868,
      "learning_rate": 7.680245771797108e-07,
      "logits/chosen": -0.31640625,
      "logits/rejected": 0.3125,
      "logps/chosen": -466.0,
      "logps/rejected": -548.0,
      "loss": 0.056,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.625,
      "rewards/margins": 10.625,
      "rewards/rejected": -30.25,
      "step": 10850
    },
    {
      "epoch": 1.3879481117004282,
      "grad_norm": 8.219324261787795,
      "learning_rate": 7.676708932663293e-07,
      "logits/chosen": -0.44140625,
      "logits/rejected": 0.27734375,
      "logps/chosen": -490.0,
      "logps/rejected": -560.0,
      "loss": 0.0531,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.75,
      "rewards/margins": 10.875,
      "rewards/rejected": -31.75,
      "step": 10860
    },
    {
      "epoch": 1.3892261486356956,
      "grad_norm": 4.576444999558215,
      "learning_rate": 7.67317697529545e-07,
      "logits/chosen": -0.4296875,
      "logits/rejected": 0.173828125,
      "logps/chosen": -488.0,
      "logps/rejected": -548.0,
      "loss": 0.0435,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.875,
      "rewards/margins": 10.1875,
      "rewards/rejected": -31.125,
      "step": 10870
    },
    {
      "epoch": 1.390504185570963,
      "grad_norm": 2.8014982864819866,
      "learning_rate": 7.669649888473704e-07,
      "logits/chosen": -0.388671875,
      "logits/rejected": 0.349609375,
      "logps/chosen": -468.0,
      "logps/rejected": -552.0,
      "loss": 0.0525,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.25,
      "rewards/margins": 10.375,
      "rewards/rejected": -30.625,
      "step": 10880
    },
    {
      "epoch": 1.3917822225062304,
      "grad_norm": 4.327770400549654,
      "learning_rate": 7.666127661014253e-07,
      "logits/chosen": -0.5078125,
      "logits/rejected": 0.35546875,
      "logps/chosen": -484.0,
      "logps/rejected": -536.0,
      "loss": 0.0555,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.0,
      "rewards/margins": 10.125,
      "rewards/rejected": -30.125,
      "step": 10890
    },
    {
      "epoch": 1.3930602594414978,
      "grad_norm": 3.598107463667701,
      "learning_rate": 7.662610281769211e-07,
      "logits/chosen": -0.39453125,
      "logits/rejected": 0.27734375,
      "logps/chosen": -506.0,
      "logps/rejected": -544.0,
      "loss": 0.0694,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -21.5,
      "rewards/margins": 9.6875,
      "rewards/rejected": -31.25,
      "step": 10900
    },
    {
      "epoch": 1.3943382963767652,
      "grad_norm": 7.277819921890155,
      "learning_rate": 7.659097739626465e-07,
      "logits/chosen": -0.3671875,
      "logits/rejected": 0.46484375,
      "logps/chosen": -454.0,
      "logps/rejected": -496.0,
      "loss": 0.0598,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.625,
      "rewards/margins": 9.875,
      "rewards/rejected": -28.5,
      "step": 10910
    },
    {
      "epoch": 1.3956163333120326,
      "grad_norm": 5.515399069175126,
      "learning_rate": 7.655590023509527e-07,
      "logits/chosen": -0.3984375,
      "logits/rejected": 0.3359375,
      "logps/chosen": -450.0,
      "logps/rejected": -510.0,
      "loss": 0.0516,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.25,
      "rewards/margins": 9.625,
      "rewards/rejected": -27.875,
      "step": 10920
    },
    {
      "epoch": 1.3968943702473,
      "grad_norm": 3.7890279050659696,
      "learning_rate": 7.652087122377384e-07,
      "logits/chosen": -0.498046875,
      "logits/rejected": 0.240234375,
      "logps/chosen": -436.0,
      "logps/rejected": -506.0,
      "loss": 0.0332,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.0,
      "rewards/margins": 10.0625,
      "rewards/rejected": -27.0,
      "step": 10930
    },
    {
      "epoch": 1.3981724071825676,
      "grad_norm": 3.59842174442594,
      "learning_rate": 7.648589025224355e-07,
      "logits/chosen": -0.439453125,
      "logits/rejected": 0.0908203125,
      "logps/chosen": -432.0,
      "logps/rejected": -506.0,
      "loss": 0.0569,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.625,
      "rewards/margins": 9.75,
      "rewards/rejected": -27.25,
      "step": 10940
    },
    {
      "epoch": 1.399450444117835,
      "grad_norm": 6.353526002607336,
      "learning_rate": 7.645095721079945e-07,
      "logits/chosen": -0.498046875,
      "logits/rejected": 0.380859375,
      "logps/chosen": -474.0,
      "logps/rejected": -512.0,
      "loss": 0.0506,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.875,
      "rewards/margins": 10.25,
      "rewards/rejected": -29.125,
      "step": 10950
    },
    {
      "epoch": 1.4007284810531024,
      "grad_norm": 4.133942946859961,
      "learning_rate": 7.641607199008701e-07,
      "logits/chosen": -0.57421875,
      "logits/rejected": 0.11181640625,
      "logps/chosen": -482.0,
      "logps/rejected": -528.0,
      "loss": 0.0573,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -19.875,
      "rewards/margins": 10.0625,
      "rewards/rejected": -30.0,
      "step": 10960
    },
    {
      "epoch": 1.4020065179883698,
      "grad_norm": 6.641274395009525,
      "learning_rate": 7.638123448110066e-07,
      "logits/chosen": -0.400390625,
      "logits/rejected": 0.2060546875,
      "logps/chosen": -448.0,
      "logps/rejected": -528.0,
      "loss": 0.0701,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -19.625,
      "rewards/margins": 10.0625,
      "rewards/rejected": -29.625,
      "step": 10970
    },
    {
      "epoch": 1.4032845549236372,
      "grad_norm": 6.284573608060938,
      "learning_rate": 7.634644457518242e-07,
      "logits/chosen": -0.349609375,
      "logits/rejected": 0.30078125,
      "logps/chosen": -440.0,
      "logps/rejected": -516.0,
      "loss": 0.0588,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.5,
      "rewards/margins": 9.5,
      "rewards/rejected": -29.0,
      "step": 10980
    },
    {
      "epoch": 1.4045625918589046,
      "grad_norm": 5.437970646745664,
      "learning_rate": 7.631170216402039e-07,
      "logits/chosen": -0.546875,
      "logits/rejected": 0.26953125,
      "logps/chosen": -452.0,
      "logps/rejected": -506.0,
      "loss": 0.061,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.625,
      "rewards/margins": 9.3125,
      "rewards/rejected": -28.0,
      "step": 10990
    },
    {
      "epoch": 1.4058406287941723,
      "grad_norm": 7.24305747916149,
      "learning_rate": 7.627700713964739e-07,
      "logits/chosen": -0.5078125,
      "logits/rejected": 0.1630859375,
      "logps/chosen": -488.0,
      "logps/rejected": -510.0,
      "loss": 0.0819,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.375,
      "rewards/margins": 9.5625,
      "rewards/rejected": -28.0,
      "step": 11000
    },
    {
      "epoch": 1.4071186657294397,
      "grad_norm": 2.419506062240875,
      "learning_rate": 7.624235939443953e-07,
      "logits/chosen": -0.50390625,
      "logits/rejected": 0.26953125,
      "logps/chosen": -466.0,
      "logps/rejected": -528.0,
      "loss": 0.0445,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -19.5,
      "rewards/margins": 10.0,
      "rewards/rejected": -29.5,
      "step": 11010
    },
    {
      "epoch": 1.408396702664707,
      "grad_norm": 8.822412851568348,
      "learning_rate": 7.620775882111482e-07,
      "logits/chosen": -0.46484375,
      "logits/rejected": 0.279296875,
      "logps/chosen": -454.0,
      "logps/rejected": -524.0,
      "loss": 0.078,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -18.875,
      "rewards/margins": 9.6875,
      "rewards/rejected": -28.625,
      "step": 11020
    },
    {
      "epoch": 1.4096747395999745,
      "grad_norm": 9.21692330578064,
      "learning_rate": 7.617320531273181e-07,
      "logits/chosen": -0.3984375,
      "logits/rejected": 0.259765625,
      "logps/chosen": -490.0,
      "logps/rejected": -540.0,
      "loss": 0.0576,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.5,
      "rewards/margins": 9.875,
      "rewards/rejected": -30.375,
      "step": 11030
    },
    {
      "epoch": 1.4109527765352419,
      "grad_norm": 6.367899458622755,
      "learning_rate": 7.613869876268809e-07,
      "logits/chosen": -0.294921875,
      "logits/rejected": 0.671875,
      "logps/chosen": -472.0,
      "logps/rejected": -520.0,
      "loss": 0.0645,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.875,
      "rewards/margins": 10.1875,
      "rewards/rejected": -31.0,
      "step": 11040
    },
    {
      "epoch": 1.4122308134705093,
      "grad_norm": 2.939648362092805,
      "learning_rate": 7.610423906471905e-07,
      "logits/chosen": -0.359375,
      "logits/rejected": 0.369140625,
      "logps/chosen": -474.0,
      "logps/rejected": -524.0,
      "loss": 0.0542,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.125,
      "rewards/margins": 9.6875,
      "rewards/rejected": -28.875,
      "step": 11050
    },
    {
      "epoch": 1.4135088504057767,
      "grad_norm": 7.62039467563853,
      "learning_rate": 7.606982611289639e-07,
      "logits/chosen": -0.4453125,
      "logits/rejected": 0.294921875,
      "logps/chosen": -462.0,
      "logps/rejected": -532.0,
      "loss": 0.0569,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.25,
      "rewards/margins": 9.625,
      "rewards/rejected": -29.875,
      "step": 11060
    },
    {
      "epoch": 1.414786887341044,
      "grad_norm": 5.202252493411787,
      "learning_rate": 7.60354598016268e-07,
      "logits/chosen": -0.490234375,
      "logits/rejected": 0.1630859375,
      "logps/chosen": -462.0,
      "logps/rejected": -524.0,
      "loss": 0.0606,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.75,
      "rewards/margins": 9.875,
      "rewards/rejected": -29.75,
      "step": 11070
    },
    {
      "epoch": 1.4160649242763115,
      "grad_norm": 2.9181759218950902,
      "learning_rate": 7.600114002565063e-07,
      "logits/chosen": -0.47265625,
      "logits/rejected": 0.158203125,
      "logps/chosen": -484.0,
      "logps/rejected": -532.0,
      "loss": 0.0664,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -20.5,
      "rewards/margins": 9.75,
      "rewards/rejected": -30.25,
      "step": 11080
    },
    {
      "epoch": 1.4173429612115789,
      "grad_norm": 7.766782597057097,
      "learning_rate": 7.596686668004049e-07,
      "logits/chosen": -0.55859375,
      "logits/rejected": 0.2119140625,
      "logps/chosen": -506.0,
      "logps/rejected": -560.0,
      "loss": 0.0559,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.625,
      "rewards/margins": 10.9375,
      "rewards/rejected": -31.5,
      "step": 11090
    },
    {
      "epoch": 1.4186209981468465,
      "grad_norm": 6.167328102693251,
      "learning_rate": 7.593263966019991e-07,
      "logits/chosen": -0.53515625,
      "logits/rejected": 0.298828125,
      "logps/chosen": -480.0,
      "logps/rejected": -536.0,
      "loss": 0.0406,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.75,
      "rewards/margins": 10.4375,
      "rewards/rejected": -31.25,
      "step": 11100
    },
    {
      "epoch": 1.419899035082114,
      "grad_norm": 6.3780921463640246,
      "learning_rate": 7.589845886186201e-07,
      "logits/chosen": -0.62109375,
      "logits/rejected": 0.005126953125,
      "logps/chosen": -464.0,
      "logps/rejected": -560.0,
      "loss": 0.068,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.5,
      "rewards/margins": 12.25,
      "rewards/rejected": -31.875,
      "step": 11110
    },
    {
      "epoch": 1.4211770720173813,
      "grad_norm": 2.2503348795580678,
      "learning_rate": 7.586432418108816e-07,
      "logits/chosen": -0.486328125,
      "logits/rejected": 0.171875,
      "logps/chosen": -464.0,
      "logps/rejected": -532.0,
      "loss": 0.0499,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.875,
      "rewards/margins": 10.0,
      "rewards/rejected": -28.875,
      "step": 11120
    },
    {
      "epoch": 1.4224551089526487,
      "grad_norm": 5.842677686342807,
      "learning_rate": 7.583023551426664e-07,
      "logits/chosen": -0.455078125,
      "logits/rejected": 0.357421875,
      "logps/chosen": -460.0,
      "logps/rejected": -528.0,
      "loss": 0.058,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.375,
      "rewards/margins": 8.9375,
      "rewards/rejected": -29.375,
      "step": 11130
    },
    {
      "epoch": 1.423733145887916,
      "grad_norm": 7.86806403224109,
      "learning_rate": 7.579619275811138e-07,
      "logits/chosen": -0.30078125,
      "logits/rejected": 0.41015625,
      "logps/chosen": -474.0,
      "logps/rejected": -540.0,
      "loss": 0.0755,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -20.375,
      "rewards/margins": 9.8125,
      "rewards/rejected": -30.25,
      "step": 11140
    },
    {
      "epoch": 1.4250111828231835,
      "grad_norm": 4.451778381952158,
      "learning_rate": 7.576219580966055e-07,
      "logits/chosen": -0.453125,
      "logits/rejected": 0.2412109375,
      "logps/chosen": -442.0,
      "logps/rejected": -520.0,
      "loss": 0.0568,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.5,
      "rewards/margins": 9.625,
      "rewards/rejected": -28.125,
      "step": 11150
    },
    {
      "epoch": 1.4262892197584511,
      "grad_norm": 4.518030507313526,
      "learning_rate": 7.57282445662753e-07,
      "logits/chosen": -0.3984375,
      "logits/rejected": 0.1484375,
      "logps/chosen": -456.0,
      "logps/rejected": -556.0,
      "loss": 0.0751,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.5,
      "rewards/margins": 8.75,
      "rewards/rejected": -27.25,
      "step": 11160
    },
    {
      "epoch": 1.4275672566937185,
      "grad_norm": 8.15761907137179,
      "learning_rate": 7.569433892563852e-07,
      "logits/chosen": -0.373046875,
      "logits/rejected": 0.4375,
      "logps/chosen": -452.0,
      "logps/rejected": -502.0,
      "loss": 0.0673,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -18.125,
      "rewards/margins": 10.0625,
      "rewards/rejected": -28.25,
      "step": 11170
    },
    {
      "epoch": 1.428845293628986,
      "grad_norm": 1.7670586600415394,
      "learning_rate": 7.566047878575343e-07,
      "logits/chosen": -0.3125,
      "logits/rejected": 0.341796875,
      "logps/chosen": -462.0,
      "logps/rejected": -532.0,
      "loss": 0.038,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.875,
      "rewards/margins": 9.625,
      "rewards/rejected": -29.5,
      "step": 11180
    },
    {
      "epoch": 1.4301233305642533,
      "grad_norm": 9.185522984203027,
      "learning_rate": 7.562666404494236e-07,
      "logits/chosen": -0.369140625,
      "logits/rejected": 0.2041015625,
      "logps/chosen": -466.0,
      "logps/rejected": -528.0,
      "loss": 0.0484,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -19.5,
      "rewards/margins": 9.5625,
      "rewards/rejected": -29.0,
      "step": 11190
    },
    {
      "epoch": 1.4314013674995207,
      "grad_norm": 10.300478707775067,
      "learning_rate": 7.559289460184543e-07,
      "logits/chosen": -0.5078125,
      "logits/rejected": 0.08154296875,
      "logps/chosen": -472.0,
      "logps/rejected": -544.0,
      "loss": 0.045,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -19.75,
      "rewards/margins": 10.0625,
      "rewards/rejected": -29.75,
      "step": 11200
    },
    {
      "epoch": 1.4326794044347881,
      "grad_norm": 3.851574200965299,
      "learning_rate": 7.555917035541937e-07,
      "logits/chosen": -0.5390625,
      "logits/rejected": 0.1953125,
      "logps/chosen": -472.0,
      "logps/rejected": -536.0,
      "loss": 0.0693,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.375,
      "rewards/margins": 9.875,
      "rewards/rejected": -29.25,
      "step": 11210
    },
    {
      "epoch": 1.4339574413700555,
      "grad_norm": 3.5775479960369863,
      "learning_rate": 7.552549120493609e-07,
      "logits/chosen": -0.48046875,
      "logits/rejected": 0.2255859375,
      "logps/chosen": -482.0,
      "logps/rejected": -532.0,
      "loss": 0.0751,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.625,
      "rewards/margins": 9.9375,
      "rewards/rejected": -29.625,
      "step": 11220
    },
    {
      "epoch": 1.435235478305323,
      "grad_norm": 6.798174953585432,
      "learning_rate": 7.549185704998158e-07,
      "logits/chosen": -0.380859375,
      "logits/rejected": 0.421875,
      "logps/chosen": -484.0,
      "logps/rejected": -520.0,
      "loss": 0.0571,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.625,
      "rewards/margins": 9.8125,
      "rewards/rejected": -30.375,
      "step": 11230
    },
    {
      "epoch": 1.4365135152405903,
      "grad_norm": 8.460362638453596,
      "learning_rate": 7.545826779045449e-07,
      "logits/chosen": -0.41015625,
      "logits/rejected": 0.439453125,
      "logps/chosen": -432.0,
      "logps/rejected": -494.0,
      "loss": 0.0716,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.375,
      "rewards/margins": 8.9375,
      "rewards/rejected": -28.375,
      "step": 11240
    },
    {
      "epoch": 1.4377915521758577,
      "grad_norm": 2.235601849908752,
      "learning_rate": 7.542472332656506e-07,
      "logits/chosen": -0.4140625,
      "logits/rejected": 0.306640625,
      "logps/chosen": -478.0,
      "logps/rejected": -520.0,
      "loss": 0.0583,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -20.75,
      "rewards/margins": 9.5625,
      "rewards/rejected": -30.375,
      "step": 11250
    },
    {
      "epoch": 1.4390695891111254,
      "grad_norm": 3.0392459428580847,
      "learning_rate": 7.539122355883373e-07,
      "logits/chosen": -0.423828125,
      "logits/rejected": 0.302734375,
      "logps/chosen": -440.0,
      "logps/rejected": -502.0,
      "loss": 0.0567,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.625,
      "rewards/margins": 9.625,
      "rewards/rejected": -28.25,
      "step": 11260
    },
    {
      "epoch": 1.4403476260463928,
      "grad_norm": 2.4038299459612795,
      "learning_rate": 7.535776838808995e-07,
      "logits/chosen": -0.5546875,
      "logits/rejected": 0.1611328125,
      "logps/chosen": -474.0,
      "logps/rejected": -532.0,
      "loss": 0.0503,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.75,
      "rewards/margins": 10.1875,
      "rewards/rejected": -29.875,
      "step": 11270
    },
    {
      "epoch": 1.4416256629816602,
      "grad_norm": 2.3229500370165503,
      "learning_rate": 7.532435771547094e-07,
      "logits/chosen": -0.5390625,
      "logits/rejected": 0.11669921875,
      "logps/chosen": -480.0,
      "logps/rejected": -536.0,
      "loss": 0.0633,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.875,
      "rewards/margins": 9.6875,
      "rewards/rejected": -29.5,
      "step": 11280
    },
    {
      "epoch": 1.4429036999169276,
      "grad_norm": 3.874953863075651,
      "learning_rate": 7.52909914424205e-07,
      "logits/chosen": -0.421875,
      "logits/rejected": 0.265625,
      "logps/chosen": -470.0,
      "logps/rejected": -528.0,
      "loss": 0.0602,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.75,
      "rewards/margins": 9.375,
      "rewards/rejected": -29.125,
      "step": 11290
    },
    {
      "epoch": 1.444181736852195,
      "grad_norm": 9.631738624748776,
      "learning_rate": 7.525766947068777e-07,
      "logits/chosen": -0.416015625,
      "logits/rejected": 0.365234375,
      "logps/chosen": -460.0,
      "logps/rejected": -520.0,
      "loss": 0.0548,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.125,
      "rewards/margins": 10.3125,
      "rewards/rejected": -29.375,
      "step": 11300
    },
    {
      "epoch": 1.4454597737874624,
      "grad_norm": 3.231322263632988,
      "learning_rate": 7.522439170232598e-07,
      "logits/chosen": -0.404296875,
      "logits/rejected": 0.125,
      "logps/chosen": -462.0,
      "logps/rejected": -540.0,
      "loss": 0.0489,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.625,
      "rewards/margins": 9.75,
      "rewards/rejected": -29.375,
      "step": 11310
    },
    {
      "epoch": 1.44673781072273,
      "grad_norm": 7.719971540372682,
      "learning_rate": 7.519115803969124e-07,
      "logits/chosen": -0.55078125,
      "logits/rejected": 0.08935546875,
      "logps/chosen": -464.0,
      "logps/rejected": -544.0,
      "loss": 0.0716,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.0,
      "rewards/margins": 10.8125,
      "rewards/rejected": -29.75,
      "step": 11320
    },
    {
      "epoch": 1.4480158476579974,
      "grad_norm": 2.7884754695287195,
      "learning_rate": 7.515796838544139e-07,
      "logits/chosen": -0.396484375,
      "logits/rejected": 0.279296875,
      "logps/chosen": -486.0,
      "logps/rejected": -532.0,
      "loss": 0.0629,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -21.625,
      "rewards/margins": 9.5625,
      "rewards/rejected": -31.25,
      "step": 11330
    },
    {
      "epoch": 1.4492938845932648,
      "grad_norm": 8.252595189273752,
      "learning_rate": 7.51248226425348e-07,
      "logits/chosen": -0.45703125,
      "logits/rejected": 0.333984375,
      "logps/chosen": -498.0,
      "logps/rejected": -544.0,
      "loss": 0.0607,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.375,
      "rewards/margins": 10.3125,
      "rewards/rejected": -31.75,
      "step": 11340
    },
    {
      "epoch": 1.4505719215285322,
      "grad_norm": 7.071425717745052,
      "learning_rate": 7.509172071422913e-07,
      "logits/chosen": -0.4375,
      "logits/rejected": 0.373046875,
      "logps/chosen": -496.0,
      "logps/rejected": -552.0,
      "loss": 0.0549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -21.875,
      "rewards/margins": 11.5625,
      "rewards/rejected": -33.5,
      "step": 11350
    },
    {
      "epoch": 1.4518499584637996,
      "grad_norm": 6.4449613999592374,
      "learning_rate": 7.505866250408015e-07,
      "logits/chosen": -0.44140625,
      "logits/rejected": 0.322265625,
      "logps/chosen": -466.0,
      "logps/rejected": -536.0,
      "loss": 0.0789,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.375,
      "rewards/margins": 10.3125,
      "rewards/rejected": -30.75,
      "step": 11360
    },
    {
      "epoch": 1.453127995399067,
      "grad_norm": 1.1145999358481975,
      "learning_rate": 7.50256479159406e-07,
      "logits/chosen": -0.46875,
      "logits/rejected": 0.2578125,
      "logps/chosen": -488.0,
      "logps/rejected": -540.0,
      "loss": 0.0539,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.375,
      "rewards/margins": 11.0625,
      "rewards/rejected": -30.5,
      "step": 11370
    },
    {
      "epoch": 1.4544060323343344,
      "grad_norm": 5.936335277314598,
      "learning_rate": 7.499267685395902e-07,
      "logits/chosen": -0.314453125,
      "logits/rejected": 0.57421875,
      "logps/chosen": -456.0,
      "logps/rejected": -536.0,
      "loss": 0.08,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.5,
      "rewards/margins": 10.625,
      "rewards/rejected": -31.0,
      "step": 11380
    },
    {
      "epoch": 1.4556840692696018,
      "grad_norm": 5.649200475370777,
      "learning_rate": 7.495974922257845e-07,
      "logits/chosen": -0.318359375,
      "logits/rejected": 0.140625,
      "logps/chosen": -480.0,
      "logps/rejected": -548.0,
      "loss": 0.0572,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.625,
      "rewards/margins": 9.5625,
      "rewards/rejected": -30.25,
      "step": 11390
    },
    {
      "epoch": 1.4569621062048692,
      "grad_norm": 6.042976266345139,
      "learning_rate": 7.492686492653552e-07,
      "logits/chosen": -0.50390625,
      "logits/rejected": 0.15234375,
      "logps/chosen": -478.0,
      "logps/rejected": -510.0,
      "loss": 0.0551,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.125,
      "rewards/margins": 9.875,
      "rewards/rejected": -28.0,
      "step": 11400
    },
    {
      "epoch": 1.4582401431401366,
      "grad_norm": 3.029506327151923,
      "learning_rate": 7.489402387085902e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": 0.162109375,
      "logps/chosen": -460.0,
      "logps/rejected": -524.0,
      "loss": 0.0401,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.5,
      "rewards/margins": 10.125,
      "rewards/rejected": -28.625,
      "step": 11410
    },
    {
      "epoch": 1.4595181800754042,
      "grad_norm": 7.941771055995715,
      "learning_rate": 7.486122596086891e-07,
      "logits/chosen": -0.58203125,
      "logits/rejected": 0.28515625,
      "logps/chosen": -446.0,
      "logps/rejected": -510.0,
      "loss": 0.0517,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.125,
      "rewards/margins": 10.875,
      "rewards/rejected": -29.0,
      "step": 11420
    },
    {
      "epoch": 1.4607962170106716,
      "grad_norm": 7.154035866476568,
      "learning_rate": 7.482847110217516e-07,
      "logits/chosen": -0.41796875,
      "logits/rejected": 0.1337890625,
      "logps/chosen": -454.0,
      "logps/rejected": -544.0,
      "loss": 0.0746,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.75,
      "rewards/margins": 10.875,
      "rewards/rejected": -29.5,
      "step": 11430
    },
    {
      "epoch": 1.462074253945939,
      "grad_norm": 5.162144048039336,
      "learning_rate": 7.479575920067657e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": 0.23046875,
      "logps/chosen": -490.0,
      "logps/rejected": -552.0,
      "loss": 0.0612,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.875,
      "rewards/margins": 10.1875,
      "rewards/rejected": -31.125,
      "step": 11440
    },
    {
      "epoch": 1.4633522908812064,
      "grad_norm": 6.468506547942546,
      "learning_rate": 7.476309016255964e-07,
      "logits/chosen": -0.453125,
      "logits/rejected": 0.126953125,
      "logps/chosen": -476.0,
      "logps/rejected": -512.0,
      "loss": 0.0619,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.25,
      "rewards/margins": 9.75,
      "rewards/rejected": -28.0,
      "step": 11450
    },
    {
      "epoch": 1.4646303278164738,
      "grad_norm": 5.812971415660119,
      "learning_rate": 7.473046389429744e-07,
      "logits/chosen": -0.37890625,
      "logits/rejected": 0.1953125,
      "logps/chosen": -452.0,
      "logps/rejected": -512.0,
      "loss": 0.0612,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.75,
      "rewards/margins": 10.0625,
      "rewards/rejected": -27.75,
      "step": 11460
    },
    {
      "epoch": 1.4659083647517412,
      "grad_norm": 6.559247033159352,
      "learning_rate": 7.469788030264852e-07,
      "logits/chosen": -0.4765625,
      "logits/rejected": 0.24609375,
      "logps/chosen": -476.0,
      "logps/rejected": -528.0,
      "loss": 0.0512,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -20.125,
      "rewards/margins": 9.3125,
      "rewards/rejected": -29.5,
      "step": 11470
    },
    {
      "epoch": 1.4671864016870089,
      "grad_norm": 6.17673126225513,
      "learning_rate": 7.466533929465574e-07,
      "logits/chosen": -0.36328125,
      "logits/rejected": 0.201171875,
      "logps/chosen": -486.0,
      "logps/rejected": -528.0,
      "loss": 0.0525,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.125,
      "rewards/margins": 9.8125,
      "rewards/rejected": -29.0,
      "step": 11480
    },
    {
      "epoch": 1.4684644386222763,
      "grad_norm": 8.152252819641033,
      "learning_rate": 7.463284077764519e-07,
      "logits/chosen": -0.275390625,
      "logits/rejected": 0.33203125,
      "logps/chosen": -456.0,
      "logps/rejected": -506.0,
      "loss": 0.0439,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.375,
      "rewards/margins": 9.625,
      "rewards/rejected": -28.0,
      "step": 11490
    },
    {
      "epoch": 1.4697424755575437,
      "grad_norm": 9.244702806199848,
      "learning_rate": 7.460038465922511e-07,
      "logits/chosen": -0.35546875,
      "logits/rejected": 0.25,
      "logps/chosen": -450.0,
      "logps/rejected": -508.0,
      "loss": 0.0851,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.25,
      "rewards/margins": 9.375,
      "rewards/rejected": -27.625,
      "step": 11500
    },
    {
      "epoch": 1.471020512492811,
      "grad_norm": 2.754223364298068,
      "learning_rate": 7.456797084728466e-07,
      "logits/chosen": -0.380859375,
      "logits/rejected": 0.1708984375,
      "logps/chosen": -432.0,
      "logps/rejected": -520.0,
      "loss": 0.044,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.5,
      "rewards/margins": 10.25,
      "rewards/rejected": -28.875,
      "step": 11510
    },
    {
      "epoch": 1.4722985494280785,
      "grad_norm": 4.432365292052453,
      "learning_rate": 7.453559924999299e-07,
      "logits/chosen": -0.490234375,
      "logits/rejected": 0.3515625,
      "logps/chosen": -482.0,
      "logps/rejected": -520.0,
      "loss": 0.0465,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.625,
      "rewards/margins": 10.25,
      "rewards/rejected": -29.875,
      "step": 11520
    },
    {
      "epoch": 1.4735765863633459,
      "grad_norm": 7.470448229769635,
      "learning_rate": 7.450326977579804e-07,
      "logits/chosen": -0.33984375,
      "logits/rejected": 0.283203125,
      "logps/chosen": -486.0,
      "logps/rejected": -528.0,
      "loss": 0.0605,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -21.0,
      "rewards/margins": 9.75,
      "rewards/rejected": -30.75,
      "step": 11530
    },
    {
      "epoch": 1.4748546232986133,
      "grad_norm": 7.134940238636114,
      "learning_rate": 7.447098233342549e-07,
      "logits/chosen": -0.4140625,
      "logits/rejected": 0.27734375,
      "logps/chosen": -450.0,
      "logps/rejected": -520.0,
      "loss": 0.0594,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.375,
      "rewards/margins": 10.3125,
      "rewards/rejected": -28.75,
      "step": 11540
    },
    {
      "epoch": 1.4761326602338807,
      "grad_norm": 6.590339541623433,
      "learning_rate": 7.443873683187767e-07,
      "logits/chosen": -0.357421875,
      "logits/rejected": 0.302734375,
      "logps/chosen": -458.0,
      "logps/rejected": -516.0,
      "loss": 0.063,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.875,
      "rewards/margins": 9.8125,
      "rewards/rejected": -28.75,
      "step": 11550
    },
    {
      "epoch": 1.477410697169148,
      "grad_norm": 4.722550800383918,
      "learning_rate": 7.440653318043245e-07,
      "logits/chosen": -0.53515625,
      "logits/rejected": 0.1904296875,
      "logps/chosen": -470.0,
      "logps/rejected": -528.0,
      "loss": 0.0605,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.375,
      "rewards/margins": 10.5,
      "rewards/rejected": -29.0,
      "step": 11560
    },
    {
      "epoch": 1.4786887341044155,
      "grad_norm": 5.5953826352862075,
      "learning_rate": 7.437437128864224e-07,
      "logits/chosen": -0.37890625,
      "logits/rejected": 0.26953125,
      "logps/chosen": -452.0,
      "logps/rejected": -516.0,
      "loss": 0.0698,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.25,
      "rewards/margins": 9.5625,
      "rewards/rejected": -27.875,
      "step": 11570
    },
    {
      "epoch": 1.479966771039683,
      "grad_norm": 2.7954326783896195,
      "learning_rate": 7.434225106633287e-07,
      "logits/chosen": -0.5,
      "logits/rejected": 0.2421875,
      "logps/chosen": -454.0,
      "logps/rejected": -492.0,
      "loss": 0.0505,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.25,
      "rewards/margins": 9.625,
      "rewards/rejected": -26.75,
      "step": 11580
    },
    {
      "epoch": 1.4812448079749505,
      "grad_norm": 5.309819284858215,
      "learning_rate": 7.431017242360253e-07,
      "logits/chosen": -0.421875,
      "logits/rejected": 0.2470703125,
      "logps/chosen": -438.0,
      "logps/rejected": -536.0,
      "loss": 0.0813,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.75,
      "rewards/margins": 10.5625,
      "rewards/rejected": -29.25,
      "step": 11590
    },
    {
      "epoch": 1.482522844910218,
      "grad_norm": 5.505721672410207,
      "learning_rate": 7.427813527082074e-07,
      "logits/chosen": -0.54296875,
      "logits/rejected": 0.1484375,
      "logps/chosen": -468.0,
      "logps/rejected": -516.0,
      "loss": 0.0647,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.625,
      "rewards/margins": 9.5625,
      "rewards/rejected": -28.25,
      "step": 11600
    },
    {
      "epoch": 1.4838008818454853,
      "grad_norm": 5.337809876674101,
      "learning_rate": 7.424613951862727e-07,
      "logits/chosen": -0.62890625,
      "logits/rejected": 0.1025390625,
      "logps/chosen": -504.0,
      "logps/rejected": -568.0,
      "loss": 0.0703,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.375,
      "rewards/margins": 11.0625,
      "rewards/rejected": -32.5,
      "step": 11610
    },
    {
      "epoch": 1.4850789187807527,
      "grad_norm": 2.057891515717941,
      "learning_rate": 7.42141850779311e-07,
      "logits/chosen": -0.375,
      "logits/rejected": 0.328125,
      "logps/chosen": -482.0,
      "logps/rejected": -536.0,
      "loss": 0.0478,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -21.0,
      "rewards/margins": 9.3125,
      "rewards/rejected": -30.25,
      "step": 11620
    },
    {
      "epoch": 1.4863569557160201,
      "grad_norm": 8.077905909486397,
      "learning_rate": 7.418227185990941e-07,
      "logits/chosen": -0.267578125,
      "logits/rejected": 0.2734375,
      "logps/chosen": -496.0,
      "logps/rejected": -560.0,
      "loss": 0.0742,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -22.125,
      "rewards/margins": 9.1875,
      "rewards/rejected": -31.25,
      "step": 11630
    },
    {
      "epoch": 1.4876349926512877,
      "grad_norm": 4.849778781680199,
      "learning_rate": 7.415039977600647e-07,
      "logits/chosen": -0.40625,
      "logits/rejected": 0.45703125,
      "logps/chosen": -478.0,
      "logps/rejected": -532.0,
      "loss": 0.0474,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -21.5,
      "rewards/margins": 9.6875,
      "rewards/rejected": -31.25,
      "step": 11640
    },
    {
      "epoch": 1.4889130295865551,
      "grad_norm": 6.928741639947218,
      "learning_rate": 7.411856873793271e-07,
      "logits/chosen": -0.37890625,
      "logits/rejected": 0.416015625,
      "logps/chosen": -480.0,
      "logps/rejected": -516.0,
      "loss": 0.0659,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.625,
      "rewards/margins": 10.5625,
      "rewards/rejected": -30.25,
      "step": 11650
    },
    {
      "epoch": 1.4901910665218225,
      "grad_norm": 5.978275346946316,
      "learning_rate": 7.408677865766361e-07,
      "logits/chosen": -0.474609375,
      "logits/rejected": 0.287109375,
      "logps/chosen": -488.0,
      "logps/rejected": -540.0,
      "loss": 0.0646,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -21.125,
      "rewards/margins": 10.3125,
      "rewards/rejected": -31.375,
      "step": 11660
    },
    {
      "epoch": 1.49146910345709,
      "grad_norm": 5.004923959062698,
      "learning_rate": 7.405502944743868e-07,
      "logits/chosen": -0.30078125,
      "logits/rejected": 0.28125,
      "logps/chosen": -434.0,
      "logps/rejected": -500.0,
      "loss": 0.0557,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.5,
      "rewards/margins": 9.5625,
      "rewards/rejected": -27.125,
      "step": 11670
    },
    {
      "epoch": 1.4927471403923573,
      "grad_norm": 13.997488927453984,
      "learning_rate": 7.402332101976052e-07,
      "logits/chosen": -0.326171875,
      "logits/rejected": 0.2392578125,
      "logps/chosen": -454.0,
      "logps/rejected": -510.0,
      "loss": 0.075,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.625,
      "rewards/margins": 9.0,
      "rewards/rejected": -27.625,
      "step": 11680
    },
    {
      "epoch": 1.4940251773276247,
      "grad_norm": 8.996493264284561,
      "learning_rate": 7.399165328739372e-07,
      "logits/chosen": -0.376953125,
      "logits/rejected": 0.3828125,
      "logps/chosen": -448.0,
      "logps/rejected": -506.0,
      "loss": 0.0471,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.0,
      "rewards/margins": 9.375,
      "rewards/rejected": -28.375,
      "step": 11690
    },
    {
      "epoch": 1.4953032142628921,
      "grad_norm": 5.505500137566294,
      "learning_rate": 7.396002616336387e-07,
      "logits/chosen": -0.38671875,
      "logits/rejected": 0.322265625,
      "logps/chosen": -440.0,
      "logps/rejected": -528.0,
      "loss": 0.0763,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.5,
      "rewards/margins": 10.8125,
      "rewards/rejected": -29.25,
      "step": 11700
    },
    {
      "epoch": 1.4965812511981595,
      "grad_norm": 10.90355964121012,
      "learning_rate": 7.392843956095663e-07,
      "logits/chosen": -0.458984375,
      "logits/rejected": 0.1748046875,
      "logps/chosen": -486.0,
      "logps/rejected": -532.0,
      "loss": 0.0586,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.375,
      "rewards/margins": 9.625,
      "rewards/rejected": -30.0,
      "step": 11710
    },
    {
      "epoch": 1.497859288133427,
      "grad_norm": 3.883148752829714,
      "learning_rate": 7.389689339371664e-07,
      "logits/chosen": -0.287109375,
      "logits/rejected": 0.3125,
      "logps/chosen": -458.0,
      "logps/rejected": -510.0,
      "loss": 0.0579,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.5,
      "rewards/margins": 10.0,
      "rewards/rejected": -28.5,
      "step": 11720
    },
    {
      "epoch": 1.4991373250686943,
      "grad_norm": 4.0353142365991985,
      "learning_rate": 7.386538757544653e-07,
      "logits/chosen": -0.451171875,
      "logits/rejected": 0.1552734375,
      "logps/chosen": -462.0,
      "logps/rejected": -524.0,
      "loss": 0.0503,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.875,
      "rewards/margins": 10.125,
      "rewards/rejected": -28.0,
      "step": 11730
    },
    {
      "epoch": 1.5004153620039618,
      "grad_norm": 4.476877679110095,
      "learning_rate": 7.3833922020206e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": 0.328125,
      "logps/chosen": -454.0,
      "logps/rejected": -506.0,
      "loss": 0.0536,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.0,
      "rewards/margins": 9.0625,
      "rewards/rejected": -27.0,
      "step": 11740
    },
    {
      "epoch": 1.5016933989392294,
      "grad_norm": 7.492542159988202,
      "learning_rate": 7.38024966423108e-07,
      "logits/chosen": -0.578125,
      "logits/rejected": 0.0859375,
      "logps/chosen": -452.0,
      "logps/rejected": -512.0,
      "loss": 0.048,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -16.875,
      "rewards/margins": 9.5,
      "rewards/rejected": -26.25,
      "step": 11750
    },
    {
      "epoch": 1.5029714358744968,
      "grad_norm": 5.134079297095461,
      "learning_rate": 7.377111135633174e-07,
      "logits/chosen": -0.55859375,
      "logits/rejected": -0.007232666015625,
      "logps/chosen": -442.0,
      "logps/rejected": -524.0,
      "loss": 0.0628,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.875,
      "rewards/margins": 9.8125,
      "rewards/rejected": -27.75,
      "step": 11760
    },
    {
      "epoch": 1.5042494728097642,
      "grad_norm": 12.704186230103746,
      "learning_rate": 7.373976607709372e-07,
      "logits/chosen": -0.484375,
      "logits/rejected": 0.14453125,
      "logps/chosen": -428.0,
      "logps/rejected": -490.0,
      "loss": 0.075,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.875,
      "rewards/margins": 10.1875,
      "rewards/rejected": -27.125,
      "step": 11770
    },
    {
      "epoch": 1.5055275097450316,
      "grad_norm": 17.029101799331805,
      "learning_rate": 7.370846071967476e-07,
      "logits/chosen": -0.486328125,
      "logits/rejected": 0.15234375,
      "logps/chosen": -468.0,
      "logps/rejected": -520.0,
      "loss": 0.0753,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -19.125,
      "rewards/margins": 9.8125,
      "rewards/rejected": -28.875,
      "step": 11780
    },
    {
      "epoch": 1.5068055466802992,
      "grad_norm": 4.379387407834821,
      "learning_rate": 7.367719519940501e-07,
      "logits/chosen": -0.484375,
      "logits/rejected": 0.2236328125,
      "logps/chosen": -484.0,
      "logps/rejected": -560.0,
      "loss": 0.0448,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.875,
      "rewards/margins": 10.5625,
      "rewards/rejected": -31.5,
      "step": 11790
    },
    {
      "epoch": 1.5080835836155666,
      "grad_norm": 5.955650226669275,
      "learning_rate": 7.364596943186587e-07,
      "logits/chosen": -0.546875,
      "logits/rejected": 0.267578125,
      "logps/chosen": -450.0,
      "logps/rejected": -502.0,
      "loss": 0.0644,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.625,
      "rewards/margins": 9.8125,
      "rewards/rejected": -27.375,
      "step": 11800
    },
    {
      "epoch": 1.509361620550834,
      "grad_norm": 9.695418304671364,
      "learning_rate": 7.36147833328889e-07,
      "logits/chosen": -0.58984375,
      "logits/rejected": 0.2021484375,
      "logps/chosen": -460.0,
      "logps/rejected": -516.0,
      "loss": 0.0721,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.625,
      "rewards/margins": 10.1875,
      "rewards/rejected": -28.75,
      "step": 11810
    },
    {
      "epoch": 1.5106396574861014,
      "grad_norm": 7.529805153777354,
      "learning_rate": 7.358363681855503e-07,
      "logits/chosen": -0.3984375,
      "logits/rejected": 0.1767578125,
      "logps/chosen": -436.0,
      "logps/rejected": -524.0,
      "loss": 0.0609,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.25,
      "rewards/margins": 9.6875,
      "rewards/rejected": -28.0,
      "step": 11820
    },
    {
      "epoch": 1.5119176944213688,
      "grad_norm": 4.611606369263192,
      "learning_rate": 7.355252980519345e-07,
      "logits/chosen": -0.466796875,
      "logits/rejected": 0.11083984375,
      "logps/chosen": -454.0,
      "logps/rejected": -504.0,
      "loss": 0.061,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.5,
      "rewards/margins": 10.1875,
      "rewards/rejected": -27.75,
      "step": 11830
    },
    {
      "epoch": 1.5131957313566362,
      "grad_norm": 8.084210786202288,
      "learning_rate": 7.352146220938078e-07,
      "logits/chosen": -0.5625,
      "logits/rejected": 0.06494140625,
      "logps/chosen": -450.0,
      "logps/rejected": -520.0,
      "loss": 0.0474,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.0,
      "rewards/margins": 9.125,
      "rewards/rejected": -27.125,
      "step": 11840
    },
    {
      "epoch": 1.5144737682919036,
      "grad_norm": 3.73322770913082,
      "learning_rate": 7.349043394794005e-07,
      "logits/chosen": -0.55859375,
      "logits/rejected": 0.1240234375,
      "logps/chosen": -464.0,
      "logps/rejected": -524.0,
      "loss": 0.0669,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.5,
      "rewards/margins": 9.5,
      "rewards/rejected": -28.0,
      "step": 11850
    },
    {
      "epoch": 1.515751805227171,
      "grad_norm": 5.6468841333433595,
      "learning_rate": 7.345944493793987e-07,
      "logits/chosen": -0.40625,
      "logits/rejected": 0.16796875,
      "logps/chosen": -444.0,
      "logps/rejected": -504.0,
      "loss": 0.0626,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.375,
      "rewards/margins": 9.25,
      "rewards/rejected": -26.625,
      "step": 11860
    },
    {
      "epoch": 1.5170298421624384,
      "grad_norm": 4.042301689595233,
      "learning_rate": 7.342849509669337e-07,
      "logits/chosen": -0.51171875,
      "logits/rejected": 0.20703125,
      "logps/chosen": -446.0,
      "logps/rejected": -500.0,
      "loss": 0.0491,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.375,
      "rewards/margins": 9.4375,
      "rewards/rejected": -26.875,
      "step": 11870
    },
    {
      "epoch": 1.5183078790977058,
      "grad_norm": 6.831098447872624,
      "learning_rate": 7.339758434175737e-07,
      "logits/chosen": -0.376953125,
      "logits/rejected": 0.283203125,
      "logps/chosen": -452.0,
      "logps/rejected": -512.0,
      "loss": 0.0665,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.375,
      "rewards/margins": 9.0625,
      "rewards/rejected": -28.375,
      "step": 11880
    },
    {
      "epoch": 1.5195859160329732,
      "grad_norm": 4.267549802274819,
      "learning_rate": 7.336671259093143e-07,
      "logits/chosen": -0.396484375,
      "logits/rejected": 0.275390625,
      "logps/chosen": -454.0,
      "logps/rejected": -524.0,
      "loss": 0.0691,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -19.25,
      "rewards/margins": 9.5625,
      "rewards/rejected": -28.875,
      "step": 11890
    },
    {
      "epoch": 1.5208639529682406,
      "grad_norm": 5.621939023737571,
      "learning_rate": 7.33358797622569e-07,
      "logits/chosen": -0.515625,
      "logits/rejected": 0.228515625,
      "logps/chosen": -440.0,
      "logps/rejected": -510.0,
      "loss": 0.0566,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -18.0,
      "rewards/margins": 9.375,
      "rewards/rejected": -27.375,
      "step": 11900
    },
    {
      "epoch": 1.5221419899035082,
      "grad_norm": 5.219729416797891,
      "learning_rate": 7.330508577401606e-07,
      "logits/chosen": -0.357421875,
      "logits/rejected": 0.302734375,
      "logps/chosen": -464.0,
      "logps/rejected": -510.0,
      "loss": 0.0609,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.5,
      "rewards/margins": 9.375,
      "rewards/rejected": -28.875,
      "step": 11910
    },
    {
      "epoch": 1.5234200268387756,
      "grad_norm": 4.170185584690525,
      "learning_rate": 7.327433054473117e-07,
      "logits/chosen": -0.478515625,
      "logits/rejected": 0.2255859375,
      "logps/chosen": -470.0,
      "logps/rejected": -524.0,
      "loss": 0.0443,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.25,
      "rewards/margins": 9.6875,
      "rewards/rejected": -29.0,
      "step": 11920
    },
    {
      "epoch": 1.524698063774043,
      "grad_norm": 9.027787644004698,
      "learning_rate": 7.324361399316357e-07,
      "logits/chosen": -0.48046875,
      "logits/rejected": 0.09716796875,
      "logps/chosen": -486.0,
      "logps/rejected": -548.0,
      "loss": 0.076,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.875,
      "rewards/margins": 9.9375,
      "rewards/rejected": -29.875,
      "step": 11930
    },
    {
      "epoch": 1.5259761007093104,
      "grad_norm": 7.651624522099067,
      "learning_rate": 7.321293603831281e-07,
      "logits/chosen": -0.474609375,
      "logits/rejected": 0.416015625,
      "logps/chosen": -504.0,
      "logps/rejected": -548.0,
      "loss": 0.0587,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -21.75,
      "rewards/margins": 9.875,
      "rewards/rejected": -31.625,
      "step": 11940
    },
    {
      "epoch": 1.527254137644578,
      "grad_norm": 7.463591999954172,
      "learning_rate": 7.318229659941572e-07,
      "logits/chosen": -0.470703125,
      "logits/rejected": 0.265625,
      "logps/chosen": -492.0,
      "logps/rejected": -540.0,
      "loss": 0.0646,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.125,
      "rewards/margins": 9.8125,
      "rewards/rejected": -30.0,
      "step": 11950
    },
    {
      "epoch": 1.5285321745798455,
      "grad_norm": 3.5285816284778297,
      "learning_rate": 7.315169559594551e-07,
      "logits/chosen": -0.41015625,
      "logits/rejected": 0.26953125,
      "logps/chosen": -458.0,
      "logps/rejected": -528.0,
      "loss": 0.0464,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.5,
      "rewards/margins": 9.75,
      "rewards/rejected": -29.25,
      "step": 11960
    },
    {
      "epoch": 1.5298102115151129,
      "grad_norm": 2.778350174519842,
      "learning_rate": 7.31211329476109e-07,
      "logits/chosen": -0.609375,
      "logits/rejected": 0.345703125,
      "logps/chosen": -442.0,
      "logps/rejected": -528.0,
      "loss": 0.0674,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -18.75,
      "rewards/margins": 11.375,
      "rewards/rejected": -30.125,
      "step": 11970
    },
    {
      "epoch": 1.5310882484503803,
      "grad_norm": 3.2802404541470462,
      "learning_rate": 7.309060857435526e-07,
      "logits/chosen": -0.5234375,
      "logits/rejected": 0.1494140625,
      "logps/chosen": -458.0,
      "logps/rejected": -532.0,
      "loss": 0.0635,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.375,
      "rewards/margins": 9.625,
      "rewards/rejected": -28.0,
      "step": 11980
    },
    {
      "epoch": 1.5323662853856477,
      "grad_norm": 6.205524597997288,
      "learning_rate": 7.30601223963557e-07,
      "logits/chosen": -0.5,
      "logits/rejected": 0.302734375,
      "logps/chosen": -450.0,
      "logps/rejected": -480.0,
      "loss": 0.0557,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.125,
      "rewards/margins": 9.375,
      "rewards/rejected": -27.5,
      "step": 11990
    },
    {
      "epoch": 1.533644322320915,
      "grad_norm": 4.513028329383234,
      "learning_rate": 7.302967433402214e-07,
      "logits/chosen": -0.421875,
      "logits/rejected": 0.19140625,
      "logps/chosen": -436.0,
      "logps/rejected": -512.0,
      "loss": 0.0616,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.875,
      "rewards/margins": 9.75,
      "rewards/rejected": -27.625,
      "step": 12000
    },
    {
      "epoch": 1.5349223592561825,
      "grad_norm": 7.569033583664133,
      "learning_rate": 7.299926430799657e-07,
      "logits/chosen": -0.53125,
      "logits/rejected": 0.201171875,
      "logps/chosen": -470.0,
      "logps/rejected": -520.0,
      "loss": 0.0583,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.0,
      "rewards/margins": 9.875,
      "rewards/rejected": -27.75,
      "step": 12010
    },
    {
      "epoch": 1.5362003961914499,
      "grad_norm": 6.3752060199944625,
      "learning_rate": 7.296889223915205e-07,
      "logits/chosen": -0.47265625,
      "logits/rejected": 0.294921875,
      "logps/chosen": -474.0,
      "logps/rejected": -520.0,
      "loss": 0.0525,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -19.375,
      "rewards/margins": 10.1875,
      "rewards/rejected": -29.625,
      "step": 12020
    },
    {
      "epoch": 1.5374784331267173,
      "grad_norm": 3.266855315674966,
      "learning_rate": 7.293855804859192e-07,
      "logits/chosen": -0.42578125,
      "logits/rejected": 0.341796875,
      "logps/chosen": -470.0,
      "logps/rejected": -548.0,
      "loss": 0.0568,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.0,
      "rewards/margins": 10.0,
      "rewards/rejected": -31.0,
      "step": 12030
    },
    {
      "epoch": 1.5387564700619847,
      "grad_norm": 6.472668380109194,
      "learning_rate": 7.290826165764892e-07,
      "logits/chosen": -0.48828125,
      "logits/rejected": 0.328125,
      "logps/chosen": -474.0,
      "logps/rejected": -536.0,
      "loss": 0.0655,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -20.125,
      "rewards/margins": 9.5625,
      "rewards/rejected": -29.75,
      "step": 12040
    },
    {
      "epoch": 1.540034506997252,
      "grad_norm": 5.681142286661591,
      "learning_rate": 7.28780029878843e-07,
      "logits/chosen": -0.408203125,
      "logits/rejected": 0.234375,
      "logps/chosen": -484.0,
      "logps/rejected": -540.0,
      "loss": 0.0579,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -21.25,
      "rewards/margins": 9.25,
      "rewards/rejected": -30.625,
      "step": 12050
    },
    {
      "epoch": 1.5413125439325195,
      "grad_norm": 8.268785352533218,
      "learning_rate": 7.284778196108706e-07,
      "logits/chosen": -0.4296875,
      "logits/rejected": 0.150390625,
      "logps/chosen": -454.0,
      "logps/rejected": -516.0,
      "loss": 0.0596,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.0,
      "rewards/margins": 9.3125,
      "rewards/rejected": -28.375,
      "step": 12060
    },
    {
      "epoch": 1.542590580867787,
      "grad_norm": 3.121583637750701,
      "learning_rate": 7.281759849927299e-07,
      "logits/chosen": -0.345703125,
      "logits/rejected": 0.314453125,
      "logps/chosen": -446.0,
      "logps/rejected": -506.0,
      "loss": 0.0644,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.25,
      "rewards/margins": 9.6875,
      "rewards/rejected": -28.875,
      "step": 12070
    },
    {
      "epoch": 1.5438686178030545,
      "grad_norm": 6.1559453087103115,
      "learning_rate": 7.278745252468389e-07,
      "logits/chosen": -0.30078125,
      "logits/rejected": 0.2001953125,
      "logps/chosen": -444.0,
      "logps/rejected": -516.0,
      "loss": 0.0692,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.375,
      "rewards/margins": 8.9375,
      "rewards/rejected": -27.375,
      "step": 12080
    },
    {
      "epoch": 1.545146654738322,
      "grad_norm": 4.84888832052554,
      "learning_rate": 7.275734395978672e-07,
      "logits/chosen": -0.3359375,
      "logits/rejected": 0.298828125,
      "logps/chosen": -418.0,
      "logps/rejected": -488.0,
      "loss": 0.0521,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.125,
      "rewards/margins": 9.25,
      "rewards/rejected": -25.375,
      "step": 12090
    },
    {
      "epoch": 1.5464246916735893,
      "grad_norm": 7.115745785472235,
      "learning_rate": 7.272727272727272e-07,
      "logits/chosen": -0.46484375,
      "logits/rejected": 0.267578125,
      "logps/chosen": -426.0,
      "logps/rejected": -494.0,
      "loss": 0.062,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -16.875,
      "rewards/margins": 10.0625,
      "rewards/rejected": -27.0,
      "step": 12100
    },
    {
      "epoch": 1.547702728608857,
      "grad_norm": 6.193728695602573,
      "learning_rate": 7.269723875005668e-07,
      "logits/chosen": -0.4375,
      "logits/rejected": 0.1376953125,
      "logps/chosen": -458.0,
      "logps/rejected": -520.0,
      "loss": 0.0676,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.875,
      "rewards/margins": 9.9375,
      "rewards/rejected": -27.75,
      "step": 12110
    },
    {
      "epoch": 1.5489807655441243,
      "grad_norm": 7.582190368997634,
      "learning_rate": 7.266724195127595e-07,
      "logits/chosen": -0.326171875,
      "logits/rejected": 0.33984375,
      "logps/chosen": -436.0,
      "logps/rejected": -508.0,
      "loss": 0.0707,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.875,
      "rewards/margins": 9.1875,
      "rewards/rejected": -28.0,
      "step": 12120
    },
    {
      "epoch": 1.5502588024793917,
      "grad_norm": 6.818793993567014,
      "learning_rate": 7.26372822542898e-07,
      "logits/chosen": -0.36328125,
      "logits/rejected": 0.2392578125,
      "logps/chosen": -480.0,
      "logps/rejected": -528.0,
      "loss": 0.0624,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.0,
      "rewards/margins": 9.9375,
      "rewards/rejected": -29.875,
      "step": 12130
    },
    {
      "epoch": 1.5515368394146591,
      "grad_norm": 7.627493421072433,
      "learning_rate": 7.260735958267845e-07,
      "logits/chosen": -0.4140625,
      "logits/rejected": 0.365234375,
      "logps/chosen": -462.0,
      "logps/rejected": -494.0,
      "loss": 0.0723,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.25,
      "rewards/margins": 9.3125,
      "rewards/rejected": -27.5,
      "step": 12140
    },
    {
      "epoch": 1.5528148763499265,
      "grad_norm": 7.850767081496267,
      "learning_rate": 7.257747386024231e-07,
      "logits/chosen": -0.267578125,
      "logits/rejected": 0.3203125,
      "logps/chosen": -442.0,
      "logps/rejected": -502.0,
      "loss": 0.0516,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.875,
      "rewards/margins": 10.5625,
      "rewards/rejected": -28.375,
      "step": 12150
    },
    {
      "epoch": 1.554092913285194,
      "grad_norm": 4.860062571649734,
      "learning_rate": 7.254762501100117e-07,
      "logits/chosen": -0.322265625,
      "logits/rejected": 0.337890625,
      "logps/chosen": -426.0,
      "logps/rejected": -490.0,
      "loss": 0.0619,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.0,
      "rewards/margins": 9.4375,
      "rewards/rejected": -26.375,
      "step": 12160
    },
    {
      "epoch": 1.5553709502204613,
      "grad_norm": 9.216294108110231,
      "learning_rate": 7.251781295919335e-07,
      "logits/chosen": -0.50390625,
      "logits/rejected": 0.2216796875,
      "logps/chosen": -450.0,
      "logps/rejected": -502.0,
      "loss": 0.0765,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.125,
      "rewards/margins": 9.8125,
      "rewards/rejected": -26.875,
      "step": 12170
    },
    {
      "epoch": 1.5566489871557287,
      "grad_norm": 1.464685002971601,
      "learning_rate": 7.248803762927498e-07,
      "logits/chosen": -0.4375,
      "logits/rejected": 0.059326171875,
      "logps/chosen": -456.0,
      "logps/rejected": -508.0,
      "loss": 0.0662,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.75,
      "rewards/margins": 9.5,
      "rewards/rejected": -26.25,
      "step": 12180
    },
    {
      "epoch": 1.5579270240909961,
      "grad_norm": 7.078486693475819,
      "learning_rate": 7.245829894591907e-07,
      "logits/chosen": -0.34765625,
      "logits/rejected": 0.2421875,
      "logps/chosen": -466.0,
      "logps/rejected": -528.0,
      "loss": 0.0386,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.25,
      "rewards/margins": 9.5625,
      "rewards/rejected": -28.875,
      "step": 12190
    },
    {
      "epoch": 1.5592050610262636,
      "grad_norm": 4.4732823835668265,
      "learning_rate": 7.242859683401482e-07,
      "logits/chosen": -0.240234375,
      "logits/rejected": 0.240234375,
      "logps/chosen": -460.0,
      "logps/rejected": -520.0,
      "loss": 0.0872,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -20.25,
      "rewards/margins": 8.5,
      "rewards/rejected": -28.75,
      "step": 12200
    },
    {
      "epoch": 1.560483097961531,
      "grad_norm": 4.556909337120037,
      "learning_rate": 7.239893121866677e-07,
      "logits/chosen": -0.26953125,
      "logits/rejected": 0.267578125,
      "logps/chosen": -454.0,
      "logps/rejected": -516.0,
      "loss": 0.0564,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.375,
      "rewards/margins": 10.375,
      "rewards/rejected": -28.75,
      "step": 12210
    },
    {
      "epoch": 1.5617611348967984,
      "grad_norm": 10.176497345715216,
      "learning_rate": 7.236930202519399e-07,
      "logits/chosen": -0.375,
      "logits/rejected": 0.328125,
      "logps/chosen": -460.0,
      "logps/rejected": -516.0,
      "loss": 0.0917,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.25,
      "rewards/margins": 10.6875,
      "rewards/rejected": -28.875,
      "step": 12220
    },
    {
      "epoch": 1.563039171832066,
      "grad_norm": 3.261234539448462,
      "learning_rate": 7.233970917912936e-07,
      "logits/chosen": -0.3046875,
      "logits/rejected": 0.2412109375,
      "logps/chosen": -450.0,
      "logps/rejected": -516.0,
      "loss": 0.0668,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.0,
      "rewards/margins": 9.625,
      "rewards/rejected": -27.625,
      "step": 12230
    },
    {
      "epoch": 1.5643172087673334,
      "grad_norm": 5.688118903093301,
      "learning_rate": 7.231015260621871e-07,
      "logits/chosen": -0.40625,
      "logits/rejected": 0.2041015625,
      "logps/chosen": -446.0,
      "logps/rejected": -494.0,
      "loss": 0.047,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.25,
      "rewards/margins": 9.875,
      "rewards/rejected": -27.125,
      "step": 12240
    },
    {
      "epoch": 1.5655952457026008,
      "grad_norm": 7.389311125621238,
      "learning_rate": 7.228063223242011e-07,
      "logits/chosen": -0.337890625,
      "logits/rejected": 0.419921875,
      "logps/chosen": -434.0,
      "logps/rejected": -506.0,
      "loss": 0.0586,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.5,
      "rewards/margins": 9.5,
      "rewards/rejected": -28.0,
      "step": 12250
    },
    {
      "epoch": 1.5668732826378682,
      "grad_norm": 10.040908393190984,
      "learning_rate": 7.225114798390295e-07,
      "logits/chosen": -0.419921875,
      "logits/rejected": 0.30859375,
      "logps/chosen": -458.0,
      "logps/rejected": -496.0,
      "loss": 0.0581,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -17.875,
      "rewards/margins": 9.625,
      "rewards/rejected": -27.5,
      "step": 12260
    },
    {
      "epoch": 1.5681513195731358,
      "grad_norm": 8.313878612804029,
      "learning_rate": 7.222169978704737e-07,
      "logits/chosen": -0.267578125,
      "logits/rejected": 0.455078125,
      "logps/chosen": -454.0,
      "logps/rejected": -502.0,
      "loss": 0.0501,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -19.125,
      "rewards/margins": 9.5625,
      "rewards/rejected": -28.75,
      "step": 12270
    },
    {
      "epoch": 1.5694293565084032,
      "grad_norm": 3.8430511093815816,
      "learning_rate": 7.219228756844335e-07,
      "logits/chosen": -0.212890625,
      "logits/rejected": 0.251953125,
      "logps/chosen": -468.0,
      "logps/rejected": -524.0,
      "loss": 0.075,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -20.0,
      "rewards/margins": 9.0625,
      "rewards/rejected": -29.0,
      "step": 12280
    },
    {
      "epoch": 1.5707073934436706,
      "grad_norm": 7.371848136866327,
      "learning_rate": 7.216291125488994e-07,
      "logits/chosen": -0.2294921875,
      "logits/rejected": 0.45703125,
      "logps/chosen": -486.0,
      "logps/rejected": -544.0,
      "loss": 0.065,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -23.0,
      "rewards/margins": 9.5,
      "rewards/rejected": -32.5,
      "step": 12290
    },
    {
      "epoch": 1.571985430378938,
      "grad_norm": 8.53335610961687,
      "learning_rate": 7.213357077339458e-07,
      "logits/chosen": -0.197265625,
      "logits/rejected": 0.453125,
      "logps/chosen": -474.0,
      "logps/rejected": -548.0,
      "loss": 0.076,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.625,
      "rewards/margins": 10.0,
      "rewards/rejected": -31.625,
      "step": 12300
    },
    {
      "epoch": 1.5732634673142054,
      "grad_norm": 7.204489126525912,
      "learning_rate": 7.210426605117224e-07,
      "logits/chosen": -0.224609375,
      "logits/rejected": 0.369140625,
      "logps/chosen": -478.0,
      "logps/rejected": -564.0,
      "loss": 0.053,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -20.75,
      "rewards/margins": 9.75,
      "rewards/rejected": -30.5,
      "step": 12310
    },
    {
      "epoch": 1.5745415042494728,
      "grad_norm": 4.923561459339351,
      "learning_rate": 7.207499701564471e-07,
      "logits/chosen": -0.396484375,
      "logits/rejected": 0.251953125,
      "logps/chosen": -480.0,
      "logps/rejected": -532.0,
      "loss": 0.0505,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.5,
      "rewards/margins": 10.125,
      "rewards/rejected": -30.625,
      "step": 12320
    },
    {
      "epoch": 1.5758195411847402,
      "grad_norm": 1.1809145729369595,
      "learning_rate": 7.204576359443989e-07,
      "logits/chosen": -0.19140625,
      "logits/rejected": 0.498046875,
      "logps/chosen": -476.0,
      "logps/rejected": -524.0,
      "loss": 0.0568,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -21.75,
      "rewards/margins": 9.125,
      "rewards/rejected": -30.875,
      "step": 12330
    },
    {
      "epoch": 1.5770975781200076,
      "grad_norm": 4.644403888998864,
      "learning_rate": 7.201656571539094e-07,
      "logits/chosen": -0.3828125,
      "logits/rejected": 0.330078125,
      "logps/chosen": -448.0,
      "logps/rejected": -512.0,
      "loss": 0.0571,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.125,
      "rewards/margins": 9.875,
      "rewards/rejected": -29.0,
      "step": 12340
    },
    {
      "epoch": 1.578375615055275,
      "grad_norm": 6.721722826494369,
      "learning_rate": 7.19874033065356e-07,
      "logits/chosen": -0.33984375,
      "logits/rejected": 0.345703125,
      "logps/chosen": -462.0,
      "logps/rejected": -540.0,
      "loss": 0.0574,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.875,
      "rewards/margins": 9.8125,
      "rewards/rejected": -29.75,
      "step": 12350
    },
    {
      "epoch": 1.5796536519905424,
      "grad_norm": 4.093102757370707,
      "learning_rate": 7.195827629611545e-07,
      "logits/chosen": -0.30078125,
      "logits/rejected": 0.375,
      "logps/chosen": -480.0,
      "logps/rejected": -524.0,
      "loss": 0.0691,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.625,
      "rewards/margins": 10.0625,
      "rewards/rejected": -29.75,
      "step": 12360
    },
    {
      "epoch": 1.5809316889258098,
      "grad_norm": 8.829460011671374,
      "learning_rate": 7.19291846125751e-07,
      "logits/chosen": -0.267578125,
      "logits/rejected": 0.1767578125,
      "logps/chosen": -462.0,
      "logps/rejected": -532.0,
      "loss": 0.0564,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -19.625,
      "rewards/margins": 10.0625,
      "rewards/rejected": -29.75,
      "step": 12370
    },
    {
      "epoch": 1.5822097258610772,
      "grad_norm": 7.233018374443183,
      "learning_rate": 7.190012818456154e-07,
      "logits/chosen": -0.38671875,
      "logits/rejected": 0.1884765625,
      "logps/chosen": -462.0,
      "logps/rejected": -504.0,
      "loss": 0.072,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.125,
      "rewards/margins": 9.0625,
      "rewards/rejected": -28.25,
      "step": 12380
    },
    {
      "epoch": 1.5834877627963448,
      "grad_norm": 2.076975777827147,
      "learning_rate": 7.187110694092334e-07,
      "logits/chosen": -0.400390625,
      "logits/rejected": 0.263671875,
      "logps/chosen": -472.0,
      "logps/rejected": -536.0,
      "loss": 0.0502,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.625,
      "rewards/margins": 9.5,
      "rewards/rejected": -29.125,
      "step": 12390
    },
    {
      "epoch": 1.5847657997316122,
      "grad_norm": 9.174137390739746,
      "learning_rate": 7.184212081070996e-07,
      "logits/chosen": -0.3515625,
      "logits/rejected": 0.267578125,
      "logps/chosen": -480.0,
      "logps/rejected": -540.0,
      "loss": 0.0732,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.125,
      "rewards/margins": 9.9375,
      "rewards/rejected": -31.125,
      "step": 12400
    },
    {
      "epoch": 1.5860438366668796,
      "grad_norm": 10.10041397977177,
      "learning_rate": 7.181316972317097e-07,
      "logits/chosen": -0.279296875,
      "logits/rejected": 0.255859375,
      "logps/chosen": -464.0,
      "logps/rejected": -536.0,
      "loss": 0.058,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.75,
      "rewards/margins": 10.375,
      "rewards/rejected": -30.0,
      "step": 12410
    },
    {
      "epoch": 1.587321873602147,
      "grad_norm": 2.1403658163215336,
      "learning_rate": 7.17842536077554e-07,
      "logits/chosen": -0.427734375,
      "logits/rejected": 0.1875,
      "logps/chosen": -452.0,
      "logps/rejected": -512.0,
      "loss": 0.063,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.375,
      "rewards/margins": 10.5,
      "rewards/rejected": -28.875,
      "step": 12420
    },
    {
      "epoch": 1.5885999105374147,
      "grad_norm": 6.749581329335853,
      "learning_rate": 7.175537239411094e-07,
      "logits/chosen": -0.39453125,
      "logits/rejected": 0.2578125,
      "logps/chosen": -480.0,
      "logps/rejected": -532.0,
      "loss": 0.0596,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -20.375,
      "rewards/margins": 9.625,
      "rewards/rejected": -30.0,
      "step": 12430
    },
    {
      "epoch": 1.589877947472682,
      "grad_norm": 10.108666795792255,
      "learning_rate": 7.172652601208325e-07,
      "logits/chosen": -0.357421875,
      "logits/rejected": 0.27734375,
      "logps/chosen": -442.0,
      "logps/rejected": -504.0,
      "loss": 0.0533,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.0,
      "rewards/margins": 9.8125,
      "rewards/rejected": -27.75,
      "step": 12440
    },
    {
      "epoch": 1.5911559844079495,
      "grad_norm": 4.116958683811359,
      "learning_rate": 7.169771439171534e-07,
      "logits/chosen": -0.380859375,
      "logits/rejected": 0.2373046875,
      "logps/chosen": -436.0,
      "logps/rejected": -480.0,
      "loss": 0.0599,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -16.25,
      "rewards/margins": 9.5625,
      "rewards/rejected": -25.75,
      "step": 12450
    },
    {
      "epoch": 1.5924340213432169,
      "grad_norm": 10.495555392885342,
      "learning_rate": 7.166893746324661e-07,
      "logits/chosen": -0.5625,
      "logits/rejected": 0.0181884765625,
      "logps/chosen": -446.0,
      "logps/rejected": -520.0,
      "loss": 0.0501,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.25,
      "rewards/margins": 9.8125,
      "rewards/rejected": -27.0,
      "step": 12460
    },
    {
      "epoch": 1.5937120582784843,
      "grad_norm": 4.783766230342951,
      "learning_rate": 7.164019515711245e-07,
      "logits/chosen": -0.5390625,
      "logits/rejected": 0.10009765625,
      "logps/chosen": -464.0,
      "logps/rejected": -552.0,
      "loss": 0.0523,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.5,
      "rewards/margins": 10.625,
      "rewards/rejected": -30.125,
      "step": 12470
    },
    {
      "epoch": 1.5949900952137517,
      "grad_norm": 3.6294395979969223,
      "learning_rate": 7.161148740394328e-07,
      "logits/chosen": -0.3984375,
      "logits/rejected": 0.337890625,
      "logps/chosen": -464.0,
      "logps/rejected": -510.0,
      "loss": 0.0527,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -19.0,
      "rewards/margins": 9.875,
      "rewards/rejected": -28.875,
      "step": 12480
    },
    {
      "epoch": 1.596268132149019,
      "grad_norm": 5.229806634580297,
      "learning_rate": 7.158281413456402e-07,
      "logits/chosen": -0.33984375,
      "logits/rejected": 0.28515625,
      "logps/chosen": -474.0,
      "logps/rejected": -528.0,
      "loss": 0.0639,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.5,
      "rewards/margins": 9.5,
      "rewards/rejected": -30.0,
      "step": 12490
    },
    {
      "epoch": 1.5975461690842865,
      "grad_norm": 7.184182060354819,
      "learning_rate": 7.155417527999326e-07,
      "logits/chosen": -0.390625,
      "logits/rejected": 0.4140625,
      "logps/chosen": -474.0,
      "logps/rejected": -520.0,
      "loss": 0.0577,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -21.25,
      "rewards/margins": 10.0,
      "rewards/rejected": -31.25,
      "step": 12500
    },
    {
      "epoch": 1.5988242060195539,
      "grad_norm": 7.875457103789657,
      "learning_rate": 7.152557077144268e-07,
      "logits/chosen": -0.3046875,
      "logits/rejected": 0.373046875,
      "logps/chosen": -502.0,
      "logps/rejected": -548.0,
      "loss": 0.052,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -22.875,
      "rewards/margins": 9.75,
      "rewards/rejected": -32.75,
      "step": 12510
    },
    {
      "epoch": 1.6001022429548213,
      "grad_norm": 7.247637538209507,
      "learning_rate": 7.149700054031623e-07,
      "logits/chosen": -0.236328125,
      "logits/rejected": 0.396484375,
      "logps/chosen": -484.0,
      "logps/rejected": -556.0,
      "loss": 0.054,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -21.875,
      "rewards/margins": 10.375,
      "rewards/rejected": -32.25,
      "step": 12520
    },
    {
      "epoch": 1.6013802798900887,
      "grad_norm": 5.812576093751782,
      "learning_rate": 7.146846451820958e-07,
      "logits/chosen": -0.314453125,
      "logits/rejected": 0.43359375,
      "logps/chosen": -468.0,
      "logps/rejected": -532.0,
      "loss": 0.0611,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.125,
      "rewards/margins": 9.25,
      "rewards/rejected": -30.375,
      "step": 12530
    },
    {
      "epoch": 1.602658316825356,
      "grad_norm": 5.241663008275666,
      "learning_rate": 7.14399626369093e-07,
      "logits/chosen": -0.1640625,
      "logits/rejected": 0.416015625,
      "logps/chosen": -466.0,
      "logps/rejected": -532.0,
      "loss": 0.0525,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -21.75,
      "rewards/margins": 9.375,
      "rewards/rejected": -31.125,
      "step": 12540
    },
    {
      "epoch": 1.6039363537606237,
      "grad_norm": 4.725534470001813,
      "learning_rate": 7.141149482839228e-07,
      "logits/chosen": -0.318359375,
      "logits/rejected": 0.302734375,
      "logps/chosen": -498.0,
      "logps/rejected": -552.0,
      "loss": 0.0649,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.25,
      "rewards/margins": 10.4375,
      "rewards/rejected": -31.75,
      "step": 12550
    },
    {
      "epoch": 1.6052143906958911,
      "grad_norm": 3.9517501752428497,
      "learning_rate": 7.138306102482496e-07,
      "logits/chosen": -0.28125,
      "logits/rejected": 0.255859375,
      "logps/chosen": -442.0,
      "logps/rejected": -512.0,
      "loss": 0.065,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.375,
      "rewards/margins": 9.9375,
      "rewards/rejected": -28.375,
      "step": 12560
    },
    {
      "epoch": 1.6064924276311585,
      "grad_norm": 7.278696347784279,
      "learning_rate": 7.135466115856274e-07,
      "logits/chosen": -0.322265625,
      "logits/rejected": 0.26953125,
      "logps/chosen": -472.0,
      "logps/rejected": -524.0,
      "loss": 0.0431,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -19.75,
      "rewards/margins": 9.1875,
      "rewards/rejected": -29.0,
      "step": 12570
    },
    {
      "epoch": 1.607770464566426,
      "grad_norm": 3.646966937959973,
      "learning_rate": 7.13262951621492e-07,
      "logits/chosen": -0.41015625,
      "logits/rejected": 0.29296875,
      "logps/chosen": -450.0,
      "logps/rejected": -512.0,
      "loss": 0.0613,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.875,
      "rewards/margins": 9.0625,
      "rewards/rejected": -26.875,
      "step": 12580
    },
    {
      "epoch": 1.6090485015016935,
      "grad_norm": 8.809131884010402,
      "learning_rate": 7.129796296831554e-07,
      "logits/chosen": -0.373046875,
      "logits/rejected": 0.2099609375,
      "logps/chosen": -436.0,
      "logps/rejected": -520.0,
      "loss": 0.0547,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.375,
      "rewards/margins": 9.75,
      "rewards/rejected": -28.125,
      "step": 12590
    },
    {
      "epoch": 1.610326538436961,
      "grad_norm": 2.345218579030221,
      "learning_rate": 7.126966450997984e-07,
      "logits/chosen": -0.40625,
      "logits/rejected": 0.275390625,
      "logps/chosen": -476.0,
      "logps/rejected": -520.0,
      "loss": 0.0613,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.75,
      "rewards/margins": 9.125,
      "rewards/rejected": -28.875,
      "step": 12600
    },
    {
      "epoch": 1.6116045753722283,
      "grad_norm": 3.420682311137076,
      "learning_rate": 7.124139972024637e-07,
      "logits/chosen": -0.4765625,
      "logits/rejected": 0.177734375,
      "logps/chosen": -484.0,
      "logps/rejected": -532.0,
      "loss": 0.0568,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -19.25,
      "rewards/margins": 10.0625,
      "rewards/rejected": -29.375,
      "step": 12610
    },
    {
      "epoch": 1.6128826123074957,
      "grad_norm": 4.035010482384416,
      "learning_rate": 7.121316853240503e-07,
      "logits/chosen": -0.470703125,
      "logits/rejected": 0.265625,
      "logps/chosen": -444.0,
      "logps/rejected": -516.0,
      "loss": 0.0643,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.75,
      "rewards/margins": 9.875,
      "rewards/rejected": -29.625,
      "step": 12620
    },
    {
      "epoch": 1.6141606492427631,
      "grad_norm": 11.032014375257722,
      "learning_rate": 7.118497087993057e-07,
      "logits/chosen": -0.5,
      "logits/rejected": 0.33203125,
      "logps/chosen": -504.0,
      "logps/rejected": -560.0,
      "loss": 0.0646,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.875,
      "rewards/margins": 10.0,
      "rewards/rejected": -31.875,
      "step": 12630
    },
    {
      "epoch": 1.6154386861780305,
      "grad_norm": 7.0007061141659195,
      "learning_rate": 7.1156806696482e-07,
      "logits/chosen": -0.380859375,
      "logits/rejected": 0.29296875,
      "logps/chosen": -450.0,
      "logps/rejected": -488.0,
      "loss": 0.0645,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.375,
      "rewards/margins": 9.3125,
      "rewards/rejected": -27.625,
      "step": 12640
    },
    {
      "epoch": 1.616716723113298,
      "grad_norm": 7.507948344822005,
      "learning_rate": 7.112867591590192e-07,
      "logits/chosen": -0.474609375,
      "logits/rejected": 0.275390625,
      "logps/chosen": -438.0,
      "logps/rejected": -552.0,
      "loss": 0.058,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -18.75,
      "rewards/margins": 10.875,
      "rewards/rejected": -29.625,
      "step": 12650
    },
    {
      "epoch": 1.6179947600485654,
      "grad_norm": 6.340290390361648,
      "learning_rate": 7.110057847221588e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": 0.232421875,
      "logps/chosen": -470.0,
      "logps/rejected": -540.0,
      "loss": 0.0714,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.25,
      "rewards/margins": 10.25,
      "rewards/rejected": -29.5,
      "step": 12660
    },
    {
      "epoch": 1.6192727969838328,
      "grad_norm": 6.303233758769828,
      "learning_rate": 7.107251429963166e-07,
      "logits/chosen": -0.46484375,
      "logits/rejected": 0.162109375,
      "logps/chosen": -452.0,
      "logps/rejected": -516.0,
      "loss": 0.0734,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.625,
      "rewards/margins": 10.4375,
      "rewards/rejected": -29.0,
      "step": 12670
    },
    {
      "epoch": 1.6205508339191002,
      "grad_norm": 5.822227897368257,
      "learning_rate": 7.104448333253878e-07,
      "logits/chosen": -0.47265625,
      "logits/rejected": 0.2177734375,
      "logps/chosen": -470.0,
      "logps/rejected": -532.0,
      "loss": 0.0523,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.0,
      "rewards/margins": 10.1875,
      "rewards/rejected": -29.25,
      "step": 12680
    },
    {
      "epoch": 1.6218288708543676,
      "grad_norm": 6.700967691407302,
      "learning_rate": 7.101648550550766e-07,
      "logits/chosen": -0.392578125,
      "logits/rejected": 0.2265625,
      "logps/chosen": -470.0,
      "logps/rejected": -540.0,
      "loss": 0.0319,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.875,
      "rewards/margins": 11.0625,
      "rewards/rejected": -30.875,
      "step": 12690
    },
    {
      "epoch": 1.623106907789635,
      "grad_norm": 3.2085955406591062,
      "learning_rate": 7.098852075328911e-07,
      "logits/chosen": -0.4140625,
      "logits/rejected": 0.2119140625,
      "logps/chosen": -472.0,
      "logps/rejected": -540.0,
      "loss": 0.0574,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.5,
      "rewards/margins": 10.25,
      "rewards/rejected": -30.75,
      "step": 12700
    },
    {
      "epoch": 1.6243849447249026,
      "grad_norm": 9.077335106152951,
      "learning_rate": 7.096058901081364e-07,
      "logits/chosen": -0.2041015625,
      "logits/rejected": 0.365234375,
      "logps/chosen": -454.0,
      "logps/rejected": -520.0,
      "loss": 0.068,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -20.25,
      "rewards/margins": 10.25,
      "rewards/rejected": -30.5,
      "step": 12710
    },
    {
      "epoch": 1.62566298166017,
      "grad_norm": 8.100548822098531,
      "learning_rate": 7.093269021319087e-07,
      "logits/chosen": -0.404296875,
      "logits/rejected": 0.26953125,
      "logps/chosen": -478.0,
      "logps/rejected": -532.0,
      "loss": 0.0667,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.75,
      "rewards/margins": 9.875,
      "rewards/rejected": -29.625,
      "step": 12720
    },
    {
      "epoch": 1.6269410185954374,
      "grad_norm": 4.355405866193097,
      "learning_rate": 7.090482429570884e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": 0.2294921875,
      "logps/chosen": -456.0,
      "logps/rejected": -516.0,
      "loss": 0.0711,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.5,
      "rewards/margins": 9.625,
      "rewards/rejected": -28.125,
      "step": 12730
    },
    {
      "epoch": 1.6282190555307048,
      "grad_norm": 7.155438633100244,
      "learning_rate": 7.087699119383339e-07,
      "logits/chosen": -0.29296875,
      "logits/rejected": 0.4140625,
      "logps/chosen": -460.0,
      "logps/rejected": -532.0,
      "loss": 0.0649,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.375,
      "rewards/margins": 10.75,
      "rewards/rejected": -31.0,
      "step": 12740
    },
    {
      "epoch": 1.6294970924659724,
      "grad_norm": 8.023889904564118,
      "learning_rate": 7.084919084320762e-07,
      "logits/chosen": -0.408203125,
      "logits/rejected": 0.388671875,
      "logps/chosen": -460.0,
      "logps/rejected": -506.0,
      "loss": 0.0695,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.375,
      "rewards/margins": 9.9375,
      "rewards/rejected": -29.25,
      "step": 12750
    },
    {
      "epoch": 1.6307751294012398,
      "grad_norm": 6.437290945567123,
      "learning_rate": 7.08214231796511e-07,
      "logits/chosen": -0.30859375,
      "logits/rejected": 0.265625,
      "logps/chosen": -484.0,
      "logps/rejected": -544.0,
      "loss": 0.0599,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -22.0,
      "rewards/margins": 8.875,
      "rewards/rejected": -30.875,
      "step": 12760
    },
    {
      "epoch": 1.6320531663365072,
      "grad_norm": 6.336837329775398,
      "learning_rate": 7.079368813915939e-07,
      "logits/chosen": -0.474609375,
      "logits/rejected": 0.2333984375,
      "logps/chosen": -488.0,
      "logps/rejected": -544.0,
      "loss": 0.0644,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.5,
      "rewards/margins": 10.625,
      "rewards/rejected": -31.0,
      "step": 12770
    },
    {
      "epoch": 1.6333312032717746,
      "grad_norm": 4.26392929410953,
      "learning_rate": 7.076598565790337e-07,
      "logits/chosen": -0.40625,
      "logits/rejected": 0.142578125,
      "logps/chosen": -464.0,
      "logps/rejected": -536.0,
      "loss": 0.0608,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.625,
      "rewards/margins": 9.625,
      "rewards/rejected": -29.25,
      "step": 12780
    },
    {
      "epoch": 1.634609240207042,
      "grad_norm": 8.566904025981069,
      "learning_rate": 7.073831567222859e-07,
      "logits/chosen": -0.40625,
      "logits/rejected": 0.181640625,
      "logps/chosen": -452.0,
      "logps/rejected": -520.0,
      "loss": 0.07,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.75,
      "rewards/margins": 9.1875,
      "rewards/rejected": -29.0,
      "step": 12790
    },
    {
      "epoch": 1.6358872771423094,
      "grad_norm": 3.206419809791158,
      "learning_rate": 7.071067811865475e-07,
      "logits/chosen": -0.482421875,
      "logits/rejected": 0.2275390625,
      "logps/chosen": -494.0,
      "logps/rejected": -544.0,
      "loss": 0.0635,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.375,
      "rewards/margins": 9.9375,
      "rewards/rejected": -31.25,
      "step": 12800
    },
    {
      "epoch": 1.6371653140775768,
      "grad_norm": 6.479870488435939,
      "learning_rate": 7.068307293387497e-07,
      "logits/chosen": -0.38671875,
      "logits/rejected": 0.2265625,
      "logps/chosen": -460.0,
      "logps/rejected": -524.0,
      "loss": 0.0577,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -20.125,
      "rewards/margins": 9.5625,
      "rewards/rejected": -29.75,
      "step": 12810
    },
    {
      "epoch": 1.6384433510128442,
      "grad_norm": 7.989363004834523,
      "learning_rate": 7.065550005475526e-07,
      "logits/chosen": -0.5390625,
      "logits/rejected": 0.05029296875,
      "logps/chosen": -484.0,
      "logps/rejected": -556.0,
      "loss": 0.0683,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -20.5,
      "rewards/margins": 9.4375,
      "rewards/rejected": -29.875,
      "step": 12820
    },
    {
      "epoch": 1.6397213879481116,
      "grad_norm": 3.2199110295114686,
      "learning_rate": 7.062795941833388e-07,
      "logits/chosen": -0.3515625,
      "logits/rejected": 0.197265625,
      "logps/chosen": -446.0,
      "logps/rejected": -510.0,
      "loss": 0.0481,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.25,
      "rewards/margins": 9.25,
      "rewards/rejected": -28.5,
      "step": 12830
    },
    {
      "epoch": 1.640999424883379,
      "grad_norm": 4.453454637722317,
      "learning_rate": 7.060045096182077e-07,
      "logits/chosen": -0.60546875,
      "logits/rejected": 0.12890625,
      "logps/chosen": -472.0,
      "logps/rejected": -528.0,
      "loss": 0.057,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.5,
      "rewards/margins": 9.3125,
      "rewards/rejected": -27.875,
      "step": 12840
    },
    {
      "epoch": 1.6422774618186464,
      "grad_norm": 3.9161429976538145,
      "learning_rate": 7.057297462259693e-07,
      "logits/chosen": -0.60546875,
      "logits/rejected": 0.2138671875,
      "logps/chosen": -486.0,
      "logps/rejected": -528.0,
      "loss": 0.0581,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.25,
      "rewards/margins": 9.5,
      "rewards/rejected": -30.75,
      "step": 12850
    },
    {
      "epoch": 1.6435554987539138,
      "grad_norm": 4.679616132345097,
      "learning_rate": 7.05455303382138e-07,
      "logits/chosen": -0.59765625,
      "logits/rejected": 0.051025390625,
      "logps/chosen": -474.0,
      "logps/rejected": -548.0,
      "loss": 0.0525,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.375,
      "rewards/margins": 11.3125,
      "rewards/rejected": -31.625,
      "step": 12860
    },
    {
      "epoch": 1.6448335356891814,
      "grad_norm": 8.742287766787392,
      "learning_rate": 7.051811804639268e-07,
      "logits/chosen": -0.48828125,
      "logits/rejected": 0.33203125,
      "logps/chosen": -480.0,
      "logps/rejected": -532.0,
      "loss": 0.0695,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.5,
      "rewards/margins": 10.4375,
      "rewards/rejected": -31.0,
      "step": 12870
    },
    {
      "epoch": 1.6461115726244488,
      "grad_norm": 2.9172588361919427,
      "learning_rate": 7.049073768502414e-07,
      "logits/chosen": -0.5390625,
      "logits/rejected": 0.1552734375,
      "logps/chosen": -468.0,
      "logps/rejected": -532.0,
      "loss": 0.06,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.5,
      "rewards/margins": 9.75,
      "rewards/rejected": -30.25,
      "step": 12880
    },
    {
      "epoch": 1.6473896095597163,
      "grad_norm": 7.069493409756147,
      "learning_rate": 7.046338919216742e-07,
      "logits/chosen": -0.47265625,
      "logits/rejected": 0.2470703125,
      "logps/chosen": -488.0,
      "logps/rejected": -532.0,
      "loss": 0.0569,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.0,
      "rewards/margins": 10.5625,
      "rewards/rejected": -30.625,
      "step": 12890
    },
    {
      "epoch": 1.6486676464949837,
      "grad_norm": 5.951729681028049,
      "learning_rate": 7.04360725060499e-07,
      "logits/chosen": -0.482421875,
      "logits/rejected": 0.326171875,
      "logps/chosen": -478.0,
      "logps/rejected": -536.0,
      "loss": 0.0706,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -21.0,
      "rewards/margins": 10.0,
      "rewards/rejected": -31.0,
      "step": 12900
    },
    {
      "epoch": 1.6499456834302513,
      "grad_norm": 1.8563419878471223,
      "learning_rate": 7.040878756506639e-07,
      "logits/chosen": -0.486328125,
      "logits/rejected": 0.27734375,
      "logps/chosen": -466.0,
      "logps/rejected": -516.0,
      "loss": 0.0649,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.25,
      "rewards/margins": 9.375,
      "rewards/rejected": -28.625,
      "step": 12910
    },
    {
      "epoch": 1.6512237203655187,
      "grad_norm": 3.002167718361166,
      "learning_rate": 7.038153430777867e-07,
      "logits/chosen": -0.498046875,
      "logits/rejected": 0.080078125,
      "logps/chosen": -454.0,
      "logps/rejected": -544.0,
      "loss": 0.063,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.875,
      "rewards/margins": 9.5,
      "rewards/rejected": -28.375,
      "step": 12920
    },
    {
      "epoch": 1.652501757300786,
      "grad_norm": 3.9112978217828953,
      "learning_rate": 7.035431267291484e-07,
      "logits/chosen": -0.53125,
      "logits/rejected": 0.062255859375,
      "logps/chosen": -490.0,
      "logps/rejected": -544.0,
      "loss": 0.0565,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -20.0,
      "rewards/margins": 9.625,
      "rewards/rejected": -29.625,
      "step": 12930
    },
    {
      "epoch": 1.6537797942360535,
      "grad_norm": 7.317802665026511,
      "learning_rate": 7.032712259936877e-07,
      "logits/chosen": -0.58203125,
      "logits/rejected": 0.057861328125,
      "logps/chosen": -494.0,
      "logps/rejected": -564.0,
      "loss": 0.0544,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.375,
      "rewards/margins": 9.8125,
      "rewards/rejected": -29.125,
      "step": 12940
    },
    {
      "epoch": 1.6550578311713209,
      "grad_norm": 7.245569189457586,
      "learning_rate": 7.029996402619949e-07,
      "logits/chosen": -0.55078125,
      "logits/rejected": 0.1337890625,
      "logps/chosen": -488.0,
      "logps/rejected": -524.0,
      "loss": 0.0835,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.0,
      "rewards/margins": 8.9375,
      "rewards/rejected": -28.875,
      "step": 12950
    },
    {
      "epoch": 1.6563358681065883,
      "grad_norm": 8.384271904613941,
      "learning_rate": 7.027283689263065e-07,
      "logits/chosen": -0.61328125,
      "logits/rejected": 0.1767578125,
      "logps/chosen": -468.0,
      "logps/rejected": -528.0,
      "loss": 0.0566,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.5,
      "rewards/margins": 9.8125,
      "rewards/rejected": -28.25,
      "step": 12960
    },
    {
      "epoch": 1.6576139050418557,
      "grad_norm": 8.118371563012614,
      "learning_rate": 7.024574113804996e-07,
      "logits/chosen": -0.35546875,
      "logits/rejected": 0.1748046875,
      "logps/chosen": -432.0,
      "logps/rejected": -500.0,
      "loss": 0.0788,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.5,
      "rewards/margins": 9.0,
      "rewards/rejected": -26.375,
      "step": 12970
    },
    {
      "epoch": 1.658891941977123,
      "grad_norm": 10.076197955922249,
      "learning_rate": 7.021867670200857e-07,
      "logits/chosen": -0.55078125,
      "logits/rejected": 0.14453125,
      "logps/chosen": -446.0,
      "logps/rejected": -496.0,
      "loss": 0.0656,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.0,
      "rewards/margins": 9.8125,
      "rewards/rejected": -26.875,
      "step": 12980
    },
    {
      "epoch": 1.6601699789123905,
      "grad_norm": 9.021135322925003,
      "learning_rate": 7.019164352422057e-07,
      "logits/chosen": -0.54296875,
      "logits/rejected": 0.1279296875,
      "logps/chosen": -454.0,
      "logps/rejected": -516.0,
      "loss": 0.0606,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.375,
      "rewards/margins": 9.75,
      "rewards/rejected": -28.0,
      "step": 12990
    },
    {
      "epoch": 1.6614480158476579,
      "grad_norm": 3.7768594648351543,
      "learning_rate": 7.016464154456234e-07,
      "logits/chosen": -0.328125,
      "logits/rejected": 0.376953125,
      "logps/chosen": -426.0,
      "logps/rejected": -488.0,
      "loss": 0.039,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.125,
      "rewards/margins": 9.125,
      "rewards/rejected": -27.25,
      "step": 13000
    },
    {
      "epoch": 1.6627260527829253,
      "grad_norm": 6.03796549916879,
      "learning_rate": 7.013767070307207e-07,
      "logits/chosen": -0.357421875,
      "logits/rejected": 0.2119140625,
      "logps/chosen": -462.0,
      "logps/rejected": -516.0,
      "loss": 0.0548,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.5,
      "rewards/margins": 9.0625,
      "rewards/rejected": -28.625,
      "step": 13010
    },
    {
      "epoch": 1.6640040897181927,
      "grad_norm": 7.8782692043494,
      "learning_rate": 7.011073093994919e-07,
      "logits/chosen": -0.427734375,
      "logits/rejected": 0.203125,
      "logps/chosen": -486.0,
      "logps/rejected": -548.0,
      "loss": 0.0533,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -20.375,
      "rewards/margins": 9.9375,
      "rewards/rejected": -30.375,
      "step": 13020
    },
    {
      "epoch": 1.6652821266534603,
      "grad_norm": 4.118822743171375,
      "learning_rate": 7.008382219555372e-07,
      "logits/chosen": -0.447265625,
      "logits/rejected": 0.2236328125,
      "logps/chosen": -470.0,
      "logps/rejected": -528.0,
      "loss": 0.0608,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.625,
      "rewards/margins": 9.6875,
      "rewards/rejected": -29.375,
      "step": 13030
    },
    {
      "epoch": 1.6665601635887277,
      "grad_norm": 10.641931344903787,
      "learning_rate": 7.005694441040588e-07,
      "logits/chosen": -0.408203125,
      "logits/rejected": 0.1904296875,
      "logps/chosen": -468.0,
      "logps/rejected": -548.0,
      "loss": 0.0528,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.0,
      "rewards/margins": 9.8125,
      "rewards/rejected": -29.875,
      "step": 13040
    },
    {
      "epoch": 1.6678382005239951,
      "grad_norm": 7.317037101782204,
      "learning_rate": 7.003009752518536e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": 0.380859375,
      "logps/chosen": -468.0,
      "logps/rejected": -524.0,
      "loss": 0.0646,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.0,
      "rewards/margins": 9.4375,
      "rewards/rejected": -29.375,
      "step": 13050
    },
    {
      "epoch": 1.6691162374592625,
      "grad_norm": 7.373051909724516,
      "learning_rate": 7.000328148073091e-07,
      "logits/chosen": -0.474609375,
      "logits/rejected": 0.255859375,
      "logps/chosen": -456.0,
      "logps/rejected": -502.0,
      "loss": 0.049,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.625,
      "rewards/margins": 9.4375,
      "rewards/rejected": -29.125,
      "step": 13060
    },
    {
      "epoch": 1.6703942743945301,
      "grad_norm": 4.712137275317087,
      "learning_rate": 6.997649621803973e-07,
      "logits/chosen": -0.498046875,
      "logits/rejected": 0.1845703125,
      "logps/chosen": -472.0,
      "logps/rejected": -544.0,
      "loss": 0.0599,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.25,
      "rewards/margins": 9.875,
      "rewards/rejected": -30.125,
      "step": 13070
    },
    {
      "epoch": 1.6716723113297975,
      "grad_norm": 3.581454832924529,
      "learning_rate": 6.994974167826689e-07,
      "logits/chosen": -0.4921875,
      "logits/rejected": 0.33203125,
      "logps/chosen": -452.0,
      "logps/rejected": -512.0,
      "loss": 0.0446,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.375,
      "rewards/margins": 9.3125,
      "rewards/rejected": -29.625,
      "step": 13080
    },
    {
      "epoch": 1.672950348265065,
      "grad_norm": 9.238154795705977,
      "learning_rate": 6.99230178027249e-07,
      "logits/chosen": -0.5,
      "logits/rejected": 0.10888671875,
      "logps/chosen": -482.0,
      "logps/rejected": -532.0,
      "loss": 0.0643,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.875,
      "rewards/margins": 9.375,
      "rewards/rejected": -30.25,
      "step": 13090
    },
    {
      "epoch": 1.6742283852003323,
      "grad_norm": 7.210997593033501,
      "learning_rate": 6.989632453288303e-07,
      "logits/chosen": -0.4609375,
      "logits/rejected": 0.29296875,
      "logps/chosen": -450.0,
      "logps/rejected": -508.0,
      "loss": 0.0521,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.375,
      "rewards/margins": 9.75,
      "rewards/rejected": -30.125,
      "step": 13100
    },
    {
      "epoch": 1.6755064221355997,
      "grad_norm": 4.203323742491104,
      "learning_rate": 6.98696618103669e-07,
      "logits/chosen": -0.357421875,
      "logits/rejected": 0.193359375,
      "logps/chosen": -480.0,
      "logps/rejected": -544.0,
      "loss": 0.0766,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.875,
      "rewards/margins": 10.5,
      "rewards/rejected": -30.375,
      "step": 13110
    },
    {
      "epoch": 1.6767844590708672,
      "grad_norm": 8.44213697477983,
      "learning_rate": 6.984302957695782e-07,
      "logits/chosen": -0.5625,
      "logits/rejected": -0.0230712890625,
      "logps/chosen": -464.0,
      "logps/rejected": -524.0,
      "loss": 0.0544,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.625,
      "rewards/margins": 9.875,
      "rewards/rejected": -28.5,
      "step": 13120
    },
    {
      "epoch": 1.6780624960061346,
      "grad_norm": 3.670178358424512,
      "learning_rate": 6.981642777459237e-07,
      "logits/chosen": -0.65625,
      "logits/rejected": 0.1904296875,
      "logps/chosen": -478.0,
      "logps/rejected": -536.0,
      "loss": 0.0494,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -20.125,
      "rewards/margins": 9.875,
      "rewards/rejected": -30.0,
      "step": 13130
    },
    {
      "epoch": 1.679340532941402,
      "grad_norm": 6.930953773457424,
      "learning_rate": 6.978985634536182e-07,
      "logits/chosen": -0.578125,
      "logits/rejected": 0.142578125,
      "logps/chosen": -490.0,
      "logps/rejected": -556.0,
      "loss": 0.0726,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -21.75,
      "rewards/margins": 10.5,
      "rewards/rejected": -32.25,
      "step": 13140
    },
    {
      "epoch": 1.6806185698766694,
      "grad_norm": 3.3882889312760587,
      "learning_rate": 6.976331523151157e-07,
      "logits/chosen": -0.33203125,
      "logits/rejected": 0.26171875,
      "logps/chosen": -462.0,
      "logps/rejected": -536.0,
      "loss": 0.0451,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.25,
      "rewards/margins": 10.25,
      "rewards/rejected": -30.625,
      "step": 13150
    },
    {
      "epoch": 1.6818966068119368,
      "grad_norm": 6.2099762310949895,
      "learning_rate": 6.973680437544066e-07,
      "logits/chosen": -0.376953125,
      "logits/rejected": 0.275390625,
      "logps/chosen": -444.0,
      "logps/rejected": -532.0,
      "loss": 0.0556,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.0,
      "rewards/margins": 9.875,
      "rewards/rejected": -30.875,
      "step": 13160
    },
    {
      "epoch": 1.6831746437472042,
      "grad_norm": 5.53434187598213,
      "learning_rate": 6.971032371970126e-07,
      "logits/chosen": -0.318359375,
      "logits/rejected": 0.31640625,
      "logps/chosen": -482.0,
      "logps/rejected": -532.0,
      "loss": 0.0568,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -20.75,
      "rewards/margins": 9.8125,
      "rewards/rejected": -30.5,
      "step": 13170
    },
    {
      "epoch": 1.6844526806824716,
      "grad_norm": 1.0876018761446804,
      "learning_rate": 6.968387320699806e-07,
      "logits/chosen": -0.494140625,
      "logits/rejected": 0.1748046875,
      "logps/chosen": -488.0,
      "logps/rejected": -528.0,
      "loss": 0.0474,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -21.125,
      "rewards/margins": 9.1875,
      "rewards/rejected": -30.25,
      "step": 13180
    },
    {
      "epoch": 1.6857307176177392,
      "grad_norm": 7.348092893682535,
      "learning_rate": 6.965745278018791e-07,
      "logits/chosen": -0.43359375,
      "logits/rejected": 0.1728515625,
      "logps/chosen": -454.0,
      "logps/rejected": -516.0,
      "loss": 0.0584,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -19.5,
      "rewards/margins": 9.4375,
      "rewards/rejected": -28.875,
      "step": 13190
    },
    {
      "epoch": 1.6870087545530066,
      "grad_norm": 9.333708658573174,
      "learning_rate": 6.963106238227914e-07,
      "logits/chosen": -0.58203125,
      "logits/rejected": 0.2216796875,
      "logps/chosen": -484.0,
      "logps/rejected": -544.0,
      "loss": 0.0891,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.375,
      "rewards/margins": 10.6875,
      "rewards/rejected": -31.125,
      "step": 13200
    },
    {
      "epoch": 1.688286791488274,
      "grad_norm": 4.394423238368176,
      "learning_rate": 6.96047019564311e-07,
      "logits/chosen": -0.376953125,
      "logits/rejected": 0.2109375,
      "logps/chosen": -482.0,
      "logps/rejected": -556.0,
      "loss": 0.0529,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -19.5,
      "rewards/margins": 11.4375,
      "rewards/rejected": -31.0,
      "step": 13210
    },
    {
      "epoch": 1.6895648284235414,
      "grad_norm": 1.8328494487236981,
      "learning_rate": 6.957837144595368e-07,
      "logits/chosen": -0.48828125,
      "logits/rejected": 0.04736328125,
      "logps/chosen": -434.0,
      "logps/rejected": -528.0,
      "loss": 0.0543,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.0,
      "rewards/margins": 10.0,
      "rewards/rejected": -28.125,
      "step": 13220
    },
    {
      "epoch": 1.690842865358809,
      "grad_norm": 7.806618912717069,
      "learning_rate": 6.955207079430681e-07,
      "logits/chosen": -0.546875,
      "logits/rejected": 0.1708984375,
      "logps/chosen": -482.0,
      "logps/rejected": -512.0,
      "loss": 0.0615,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.0,
      "rewards/margins": 9.8125,
      "rewards/rejected": -28.875,
      "step": 13230
    },
    {
      "epoch": 1.6921209022940764,
      "grad_norm": 5.904018046349294,
      "learning_rate": 6.952579994509982e-07,
      "logits/chosen": -0.55078125,
      "logits/rejected": 0.0830078125,
      "logps/chosen": -450.0,
      "logps/rejected": -516.0,
      "loss": 0.0521,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.25,
      "rewards/margins": 9.4375,
      "rewards/rejected": -27.625,
      "step": 13240
    },
    {
      "epoch": 1.6933989392293438,
      "grad_norm": 6.480276454484764,
      "learning_rate": 6.94995588420911e-07,
      "logits/chosen": -0.455078125,
      "logits/rejected": 0.1689453125,
      "logps/chosen": -466.0,
      "logps/rejected": -520.0,
      "loss": 0.0559,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.0,
      "rewards/margins": 9.4375,
      "rewards/rejected": -28.5,
      "step": 13250
    },
    {
      "epoch": 1.6946769761646112,
      "grad_norm": 5.0578638664116875,
      "learning_rate": 6.947334742918749e-07,
      "logits/chosen": -0.39453125,
      "logits/rejected": 0.396484375,
      "logps/chosen": -468.0,
      "logps/rejected": -544.0,
      "loss": 0.0368,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -20.875,
      "rewards/margins": 10.6875,
      "rewards/rejected": -31.625,
      "step": 13260
    },
    {
      "epoch": 1.6959550130998786,
      "grad_norm": 4.882113093086125,
      "learning_rate": 6.94471656504438e-07,
      "logits/chosen": -0.48828125,
      "logits/rejected": 0.1591796875,
      "logps/chosen": -480.0,
      "logps/rejected": -564.0,
      "loss": 0.0489,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -21.125,
      "rewards/margins": 10.375,
      "rewards/rejected": -31.5,
      "step": 13270
    },
    {
      "epoch": 1.697233050035146,
      "grad_norm": 3.572898454708742,
      "learning_rate": 6.942101345006232e-07,
      "logits/chosen": -0.412109375,
      "logits/rejected": 0.330078125,
      "logps/chosen": -468.0,
      "logps/rejected": -520.0,
      "loss": 0.0621,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -22.0,
      "rewards/margins": 9.625,
      "rewards/rejected": -31.625,
      "step": 13280
    },
    {
      "epoch": 1.6985110869704134,
      "grad_norm": 3.0436196333848913,
      "learning_rate": 6.939489077239235e-07,
      "logits/chosen": -0.427734375,
      "logits/rejected": 0.314453125,
      "logps/chosen": -460.0,
      "logps/rejected": -548.0,
      "loss": 0.0475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -19.875,
      "rewards/margins": 10.6875,
      "rewards/rejected": -30.625,
      "step": 13290
    },
    {
      "epoch": 1.6997891239056808,
      "grad_norm": 5.210799835637349,
      "learning_rate": 6.936879756192959e-07,
      "logits/chosen": -0.55078125,
      "logits/rejected": 0.2197265625,
      "logps/chosen": -474.0,
      "logps/rejected": -544.0,
      "loss": 0.0653,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.375,
      "rewards/margins": 9.75,
      "rewards/rejected": -30.25,
      "step": 13300
    },
    {
      "epoch": 1.7010671608409482,
      "grad_norm": 4.318166072044879,
      "learning_rate": 6.934273376331579e-07,
      "logits/chosen": -0.4921875,
      "logits/rejected": 0.1611328125,
      "logps/chosen": -444.0,
      "logps/rejected": -510.0,
      "loss": 0.073,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.125,
      "rewards/margins": 9.6875,
      "rewards/rejected": -27.75,
      "step": 13310
    },
    {
      "epoch": 1.7023451977762156,
      "grad_norm": 4.266117892410533,
      "learning_rate": 6.931669932133818e-07,
      "logits/chosen": -0.447265625,
      "logits/rejected": 0.23828125,
      "logps/chosen": -454.0,
      "logps/rejected": -496.0,
      "loss": 0.0697,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.5,
      "rewards/margins": 9.1875,
      "rewards/rejected": -27.75,
      "step": 13320
    },
    {
      "epoch": 1.703623234711483,
      "grad_norm": 3.804633879715066,
      "learning_rate": 6.929069418092892e-07,
      "logits/chosen": -0.431640625,
      "logits/rejected": 0.08447265625,
      "logps/chosen": -468.0,
      "logps/rejected": -524.0,
      "loss": 0.0469,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.0,
      "rewards/margins": 9.375,
      "rewards/rejected": -27.375,
      "step": 13330
    },
    {
      "epoch": 1.7049012716467504,
      "grad_norm": 8.17594653518881,
      "learning_rate": 6.926471828716478e-07,
      "logits/chosen": -0.390625,
      "logits/rejected": 0.3046875,
      "logps/chosen": -464.0,
      "logps/rejected": -532.0,
      "loss": 0.0762,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.75,
      "rewards/margins": 9.8125,
      "rewards/rejected": -29.625,
      "step": 13340
    },
    {
      "epoch": 1.706179308582018,
      "grad_norm": 6.632049891378289,
      "learning_rate": 6.923877158526646e-07,
      "logits/chosen": -0.38671875,
      "logits/rejected": 0.298828125,
      "logps/chosen": -468.0,
      "logps/rejected": -536.0,
      "loss": 0.0556,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -20.125,
      "rewards/margins": 9.875,
      "rewards/rejected": -30.0,
      "step": 13350
    },
    {
      "epoch": 1.7074573455172855,
      "grad_norm": 7.045218002129889,
      "learning_rate": 6.921285402059827e-07,
      "logits/chosen": -0.431640625,
      "logits/rejected": 0.23828125,
      "logps/chosen": -480.0,
      "logps/rejected": -544.0,
      "loss": 0.0558,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.75,
      "rewards/margins": 10.125,
      "rewards/rejected": -30.875,
      "step": 13360
    },
    {
      "epoch": 1.7087353824525529,
      "grad_norm": 8.812015261882118,
      "learning_rate": 6.918696553866751e-07,
      "logits/chosen": -0.416015625,
      "logits/rejected": 0.3359375,
      "logps/chosen": -488.0,
      "logps/rejected": -564.0,
      "loss": 0.0465,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.875,
      "rewards/margins": 11.125,
      "rewards/rejected": -32.0,
      "step": 13370
    },
    {
      "epoch": 1.7100134193878203,
      "grad_norm": 1.3734926405579382,
      "learning_rate": 6.916110608512408e-07,
      "logits/chosen": -0.435546875,
      "logits/rejected": 0.236328125,
      "logps/chosen": -464.0,
      "logps/rejected": -552.0,
      "loss": 0.0389,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.25,
      "rewards/margins": 10.0,
      "rewards/rejected": -30.25,
      "step": 13380
    },
    {
      "epoch": 1.7112914563230879,
      "grad_norm": 2.409390512693395,
      "learning_rate": 6.913527560575998e-07,
      "logits/chosen": -0.3984375,
      "logits/rejected": 0.1416015625,
      "logps/chosen": -472.0,
      "logps/rejected": -564.0,
      "loss": 0.0359,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.75,
      "rewards/margins": 10.4375,
      "rewards/rejected": -31.125,
      "step": 13390
    },
    {
      "epoch": 1.7125694932583553,
      "grad_norm": 5.056358611832604,
      "learning_rate": 6.910947404650881e-07,
      "logits/chosen": -0.345703125,
      "logits/rejected": 0.10986328125,
      "logps/chosen": -438.0,
      "logps/rejected": -524.0,
      "loss": 0.0351,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -19.0,
      "rewards/margins": 9.75,
      "rewards/rejected": -28.75,
      "step": 13400
    },
    {
      "epoch": 1.7138475301936227,
      "grad_norm": 11.469441482457388,
      "learning_rate": 6.90837013534453e-07,
      "logits/chosen": -0.443359375,
      "logits/rejected": 0.30859375,
      "logps/chosen": -434.0,
      "logps/rejected": -492.0,
      "loss": 0.0621,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.25,
      "rewards/margins": 10.4375,
      "rewards/rejected": -27.625,
      "step": 13410
    },
    {
      "epoch": 1.71512556712889,
      "grad_norm": 10.064691245823004,
      "learning_rate": 6.905795747278488e-07,
      "logits/chosen": -0.443359375,
      "logits/rejected": 0.205078125,
      "logps/chosen": -490.0,
      "logps/rejected": -560.0,
      "loss": 0.0638,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.875,
      "rewards/margins": 11.3125,
      "rewards/rejected": -31.125,
      "step": 13420
    },
    {
      "epoch": 1.7164036040641575,
      "grad_norm": 6.8211084094131476,
      "learning_rate": 6.903224235088314e-07,
      "logits/chosen": -0.39453125,
      "logits/rejected": 0.365234375,
      "logps/chosen": -494.0,
      "logps/rejected": -528.0,
      "loss": 0.06,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -21.125,
      "rewards/margins": 9.625,
      "rewards/rejected": -30.875,
      "step": 13430
    },
    {
      "epoch": 1.7176816409994249,
      "grad_norm": 3.730598164907703,
      "learning_rate": 6.900655593423541e-07,
      "logits/chosen": -0.39453125,
      "logits/rejected": 0.376953125,
      "logps/chosen": -452.0,
      "logps/rejected": -516.0,
      "loss": 0.0572,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.0,
      "rewards/margins": 10.625,
      "rewards/rejected": -29.625,
      "step": 13440
    },
    {
      "epoch": 1.7189596779346923,
      "grad_norm": 5.254674103307178,
      "learning_rate": 6.89808981694763e-07,
      "logits/chosen": -0.48046875,
      "logits/rejected": 0.26171875,
      "logps/chosen": -452.0,
      "logps/rejected": -520.0,
      "loss": 0.0474,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -17.5,
      "rewards/margins": 10.5,
      "rewards/rejected": -28.0,
      "step": 13450
    },
    {
      "epoch": 1.7202377148699597,
      "grad_norm": 4.546431535462404,
      "learning_rate": 6.895526900337915e-07,
      "logits/chosen": -0.470703125,
      "logits/rejected": 0.1787109375,
      "logps/chosen": -456.0,
      "logps/rejected": -520.0,
      "loss": 0.0663,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.125,
      "rewards/margins": 10.0,
      "rewards/rejected": -29.125,
      "step": 13460
    },
    {
      "epoch": 1.721515751805227,
      "grad_norm": 9.181628076624467,
      "learning_rate": 6.892966838285567e-07,
      "logits/chosen": -0.5234375,
      "logits/rejected": 0.06884765625,
      "logps/chosen": -454.0,
      "logps/rejected": -516.0,
      "loss": 0.072,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.125,
      "rewards/margins": 9.9375,
      "rewards/rejected": -28.125,
      "step": 13470
    },
    {
      "epoch": 1.7227937887404945,
      "grad_norm": 9.068351365222885,
      "learning_rate": 6.890409625495545e-07,
      "logits/chosen": -0.51171875,
      "logits/rejected": 0.134765625,
      "logps/chosen": -464.0,
      "logps/rejected": -524.0,
      "loss": 0.0583,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.75,
      "rewards/margins": 10.1875,
      "rewards/rejected": -29.0,
      "step": 13480
    },
    {
      "epoch": 1.724071825675762,
      "grad_norm": 6.672375482635719,
      "learning_rate": 6.887855256686546e-07,
      "logits/chosen": -0.4765625,
      "logits/rejected": 0.048583984375,
      "logps/chosen": -452.0,
      "logps/rejected": -516.0,
      "loss": 0.0538,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.5,
      "rewards/margins": 10.125,
      "rewards/rejected": -28.5,
      "step": 13490
    },
    {
      "epoch": 1.7253498626110293,
      "grad_norm": 7.570772079973231,
      "learning_rate": 6.885303726590963e-07,
      "logits/chosen": -0.5,
      "logits/rejected": 0.2578125,
      "logps/chosen": -456.0,
      "logps/rejected": -524.0,
      "loss": 0.0624,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.875,
      "rewards/margins": 10.125,
      "rewards/rejected": -29.0,
      "step": 13500
    },
    {
      "epoch": 1.726627899546297,
      "grad_norm": 9.110399806215359,
      "learning_rate": 6.882755029954837e-07,
      "logits/chosen": -0.6484375,
      "logits/rejected": 0.1826171875,
      "logps/chosen": -454.0,
      "logps/rejected": -520.0,
      "loss": 0.042,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -18.375,
      "rewards/margins": 10.3125,
      "rewards/rejected": -28.75,
      "step": 13510
    },
    {
      "epoch": 1.7279059364815643,
      "grad_norm": 6.302327217088703,
      "learning_rate": 6.880209161537815e-07,
      "logits/chosen": -0.45703125,
      "logits/rejected": 0.279296875,
      "logps/chosen": -450.0,
      "logps/rejected": -520.0,
      "loss": 0.0509,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.125,
      "rewards/margins": 9.5,
      "rewards/rejected": -29.625,
      "step": 13520
    },
    {
      "epoch": 1.7291839734168317,
      "grad_norm": 8.462041643234226,
      "learning_rate": 6.877666116113097e-07,
      "logits/chosen": -0.30859375,
      "logits/rejected": 0.173828125,
      "logps/chosen": -446.0,
      "logps/rejected": -528.0,
      "loss": 0.0642,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.25,
      "rewards/margins": 10.0,
      "rewards/rejected": -29.25,
      "step": 13530
    },
    {
      "epoch": 1.7304620103520991,
      "grad_norm": 6.1314939140824825,
      "learning_rate": 6.875125888467405e-07,
      "logits/chosen": -0.4296875,
      "logits/rejected": 0.1357421875,
      "logps/chosen": -454.0,
      "logps/rejected": -498.0,
      "loss": 0.0625,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.375,
      "rewards/margins": 9.75,
      "rewards/rejected": -28.125,
      "step": 13540
    },
    {
      "epoch": 1.7317400472873667,
      "grad_norm": 3.06920625762654,
      "learning_rate": 6.872588473400923e-07,
      "logits/chosen": -0.458984375,
      "logits/rejected": 0.2353515625,
      "logps/chosen": -506.0,
      "logps/rejected": -556.0,
      "loss": 0.0667,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.5,
      "rewards/margins": 10.3125,
      "rewards/rejected": -31.875,
      "step": 13550
    },
    {
      "epoch": 1.7330180842226341,
      "grad_norm": 5.429941889933024,
      "learning_rate": 6.870053865727262e-07,
      "logits/chosen": -0.40625,
      "logits/rejected": 0.1884765625,
      "logps/chosen": -500.0,
      "logps/rejected": -568.0,
      "loss": 0.0673,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -22.25,
      "rewards/margins": 10.1875,
      "rewards/rejected": -32.5,
      "step": 13560
    },
    {
      "epoch": 1.7342961211579015,
      "grad_norm": 4.959971074041916,
      "learning_rate": 6.867522060273408e-07,
      "logits/chosen": -0.419921875,
      "logits/rejected": 0.2314453125,
      "logps/chosen": -460.0,
      "logps/rejected": -532.0,
      "loss": 0.0465,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -21.0,
      "rewards/margins": 9.4375,
      "rewards/rejected": -30.5,
      "step": 13570
    },
    {
      "epoch": 1.735574158093169,
      "grad_norm": 3.840569194003064,
      "learning_rate": 6.864993051879688e-07,
      "logits/chosen": -0.275390625,
      "logits/rejected": 0.44140625,
      "logps/chosen": -476.0,
      "logps/rejected": -536.0,
      "loss": 0.0734,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -21.875,
      "rewards/margins": 9.4375,
      "rewards/rejected": -31.25,
      "step": 13580
    },
    {
      "epoch": 1.7368521950284364,
      "grad_norm": 8.559153957260108,
      "learning_rate": 6.862466835399716e-07,
      "logits/chosen": -0.357421875,
      "logits/rejected": 0.158203125,
      "logps/chosen": -478.0,
      "logps/rejected": -572.0,
      "loss": 0.05,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -22.125,
      "rewards/margins": 10.5625,
      "rewards/rejected": -32.75,
      "step": 13590
    },
    {
      "epoch": 1.7381302319637038,
      "grad_norm": 3.6449771220933416,
      "learning_rate": 6.859943405700353e-07,
      "logits/chosen": -0.341796875,
      "logits/rejected": 0.2255859375,
      "logps/chosen": -500.0,
      "logps/rejected": -572.0,
      "loss": 0.0553,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -22.625,
      "rewards/margins": 10.4375,
      "rewards/rejected": -33.0,
      "step": 13600
    },
    {
      "epoch": 1.7394082688989712,
      "grad_norm": 8.226250621042935,
      "learning_rate": 6.857422757661664e-07,
      "logits/chosen": -0.42578125,
      "logits/rejected": 0.23046875,
      "logps/chosen": -490.0,
      "logps/rejected": -572.0,
      "loss": 0.079,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -22.5,
      "rewards/margins": 11.0625,
      "rewards/rejected": -33.5,
      "step": 13610
    },
    {
      "epoch": 1.7406863058342386,
      "grad_norm": 11.600758396691623,
      "learning_rate": 6.854904886176873e-07,
      "logits/chosen": -0.53125,
      "logits/rejected": 0.1845703125,
      "logps/chosen": -488.0,
      "logps/rejected": -556.0,
      "loss": 0.0481,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.75,
      "rewards/margins": 10.25,
      "rewards/rejected": -32.0,
      "step": 13620
    },
    {
      "epoch": 1.741964342769506,
      "grad_norm": 6.555507396887581,
      "learning_rate": 6.85238978615232e-07,
      "logits/chosen": -0.47265625,
      "logits/rejected": 0.169921875,
      "logps/chosen": -476.0,
      "logps/rejected": -552.0,
      "loss": 0.0533,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -21.0,
      "rewards/margins": 11.5625,
      "rewards/rejected": -32.5,
      "step": 13630
    },
    {
      "epoch": 1.7432423797047734,
      "grad_norm": 3.092198704323131,
      "learning_rate": 6.849877452507417e-07,
      "logits/chosen": -0.37890625,
      "logits/rejected": 0.330078125,
      "logps/chosen": -462.0,
      "logps/rejected": -532.0,
      "loss": 0.0305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -20.875,
      "rewards/margins": 10.125,
      "rewards/rejected": -31.0,
      "step": 13640
    },
    {
      "epoch": 1.7445204166400408,
      "grad_norm": 9.85919971527004,
      "learning_rate": 6.847367880174605e-07,
      "logits/chosen": -0.478515625,
      "logits/rejected": 0.22265625,
      "logps/chosen": -476.0,
      "logps/rejected": -536.0,
      "loss": 0.0604,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -21.625,
      "rewards/margins": 10.1875,
      "rewards/rejected": -31.875,
      "step": 13650
    },
    {
      "epoch": 1.7457984535753082,
      "grad_norm": 8.180357467164844,
      "learning_rate": 6.844861064099317e-07,
      "logits/chosen": -0.4375,
      "logits/rejected": 0.185546875,
      "logps/chosen": -486.0,
      "logps/rejected": -548.0,
      "loss": 0.0497,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -21.5,
      "rewards/margins": 10.0,
      "rewards/rejected": -31.5,
      "step": 13660
    },
    {
      "epoch": 1.7470764905105758,
      "grad_norm": 6.28875929153802,
      "learning_rate": 6.842356999239922e-07,
      "logits/chosen": -0.5234375,
      "logits/rejected": 0.353515625,
      "logps/chosen": -480.0,
      "logps/rejected": -536.0,
      "loss": 0.0493,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.5,
      "rewards/margins": 10.75,
      "rewards/rejected": -32.25,
      "step": 13670
    },
    {
      "epoch": 1.7483545274458432,
      "grad_norm": 7.339424811591869,
      "learning_rate": 6.839855680567694e-07,
      "logits/chosen": -0.474609375,
      "logits/rejected": 0.3828125,
      "logps/chosen": -470.0,
      "logps/rejected": -540.0,
      "loss": 0.0827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -21.0,
      "rewards/margins": 11.375,
      "rewards/rejected": -32.25,
      "step": 13680
    },
    {
      "epoch": 1.7496325643811106,
      "grad_norm": 1.7917445280091502,
      "learning_rate": 6.837357103066766e-07,
      "logits/chosen": -0.478515625,
      "logits/rejected": 0.154296875,
      "logps/chosen": -470.0,
      "logps/rejected": -528.0,
      "loss": 0.0523,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -20.75,
      "rewards/margins": 10.0625,
      "rewards/rejected": -30.75,
      "step": 13690
    },
    {
      "epoch": 1.750910601316378,
      "grad_norm": 3.882881732555117,
      "learning_rate": 6.834861261734087e-07,
      "logits/chosen": -0.58984375,
      "logits/rejected": 0.306640625,
      "logps/chosen": -464.0,
      "logps/rejected": -532.0,
      "loss": 0.068,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.5,
      "rewards/margins": 10.625,
      "rewards/rejected": -31.125,
      "step": 13700
    },
    {
      "epoch": 1.7521886382516456,
      "grad_norm": 2.5198065657769093,
      "learning_rate": 6.832368151579382e-07,
      "logits/chosen": -0.318359375,
      "logits/rejected": 0.30859375,
      "logps/chosen": -472.0,
      "logps/rejected": -540.0,
      "loss": 0.0751,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -21.0,
      "rewards/margins": 10.0,
      "rewards/rejected": -31.0,
      "step": 13710
    },
    {
      "epoch": 1.753466675186913,
      "grad_norm": 5.729934375725355,
      "learning_rate": 6.829877767625106e-07,
      "logits/chosen": -0.3828125,
      "logits/rejected": 0.345703125,
      "logps/chosen": -476.0,
      "logps/rejected": -524.0,
      "loss": 0.056,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -21.75,
      "rewards/margins": 9.125,
      "rewards/rejected": -30.75,
      "step": 13720
    },
    {
      "epoch": 1.7547447121221804,
      "grad_norm": 7.2210070581015815,
      "learning_rate": 6.827390104906408e-07,
      "logits/chosen": -0.37109375,
      "logits/rejected": 0.376953125,
      "logps/chosen": -480.0,
      "logps/rejected": -536.0,
      "loss": 0.0478,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -21.25,
      "rewards/margins": 10.375,
      "rewards/rejected": -31.625,
      "step": 13730
    },
    {
      "epoch": 1.7560227490574478,
      "grad_norm": 3.271736680694404,
      "learning_rate": 6.824905158471082e-07,
      "logits/chosen": -0.453125,
      "logits/rejected": 0.201171875,
      "logps/chosen": -484.0,
      "logps/rejected": -556.0,
      "loss": 0.0629,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.5,
      "rewards/margins": 10.4375,
      "rewards/rejected": -31.875,
      "step": 13740
    },
    {
      "epoch": 1.7573007859927152,
      "grad_norm": 6.301466094315794,
      "learning_rate": 6.822422923379532e-07,
      "logits/chosen": -0.37890625,
      "logits/rejected": 0.310546875,
      "logps/chosen": -498.0,
      "logps/rejected": -556.0,
      "loss": 0.0688,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -21.875,
      "rewards/margins": 10.3125,
      "rewards/rejected": -32.25,
      "step": 13750
    },
    {
      "epoch": 1.7585788229279826,
      "grad_norm": 6.9437356945009,
      "learning_rate": 6.819943394704734e-07,
      "logits/chosen": -0.41015625,
      "logits/rejected": 0.1923828125,
      "logps/chosen": -478.0,
      "logps/rejected": -528.0,
      "loss": 0.0633,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -20.75,
      "rewards/margins": 9.25,
      "rewards/rejected": -30.125,
      "step": 13760
    },
    {
      "epoch": 1.75985685986325,
      "grad_norm": 2.9381545820470785,
      "learning_rate": 6.817466567532181e-07,
      "logits/chosen": -0.33203125,
      "logits/rejected": 0.337890625,
      "logps/chosen": -502.0,
      "logps/rejected": -552.0,
      "loss": 0.052,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -21.0,
      "rewards/margins": 10.4375,
      "rewards/rejected": -31.375,
      "step": 13770
    },
    {
      "epoch": 1.7611348967985174,
      "grad_norm": 7.600611481163961,
      "learning_rate": 6.814992436959855e-07,
      "logits/chosen": -0.29296875,
      "logits/rejected": 0.443359375,
      "logps/chosen": -502.0,
      "logps/rejected": -548.0,
      "loss": 0.0574,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -22.125,
      "rewards/margins": 10.3125,
      "rewards/rejected": -32.5,
      "step": 13780
    },
    {
      "epoch": 1.7624129337337848,
      "grad_norm": 3.929139461858747,
      "learning_rate": 6.812520998098182e-07,
      "logits/chosen": -0.30859375,
      "logits/rejected": 0.296875,
      "logps/chosen": -488.0,
      "logps/rejected": -548.0,
      "loss": 0.0429,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -22.0,
      "rewards/margins": 9.875,
      "rewards/rejected": -32.0,
      "step": 13790
    },
    {
      "epoch": 1.7636909706690522,
      "grad_norm": 3.5694838589726436,
      "learning_rate": 6.810052246069989e-07,
      "logits/chosen": -0.314453125,
      "logits/rejected": 0.384765625,
      "logps/chosen": -484.0,
      "logps/rejected": -544.0,
      "loss": 0.0462,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.0,
      "rewards/margins": 10.125,
      "rewards/rejected": -31.125,
      "step": 13800
    },
    {
      "epoch": 1.7649690076043196,
      "grad_norm": 5.534692732815137,
      "learning_rate": 6.807586176010469e-07,
      "logits/chosen": -0.302734375,
      "logits/rejected": 0.279296875,
      "logps/chosen": -532.0,
      "logps/rejected": -584.0,
      "loss": 0.051,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -23.625,
      "rewards/margins": 10.25,
      "rewards/rejected": -34.0,
      "step": 13810
    },
    {
      "epoch": 1.766247044539587,
      "grad_norm": 5.877185826059291,
      "learning_rate": 6.805122783067135e-07,
      "logits/chosen": -0.259765625,
      "logits/rejected": 0.40625,
      "logps/chosen": -500.0,
      "logps/rejected": -580.0,
      "loss": 0.0451,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -23.125,
      "rewards/margins": 10.5625,
      "rewards/rejected": -33.75,
      "step": 13820
    },
    {
      "epoch": 1.7675250814748547,
      "grad_norm": 11.277200366123196,
      "learning_rate": 6.802662062399785e-07,
      "logits/chosen": -0.361328125,
      "logits/rejected": 0.322265625,
      "logps/chosen": -520.0,
      "logps/rejected": -580.0,
      "loss": 0.0624,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -23.625,
      "rewards/margins": 10.25,
      "rewards/rejected": -34.0,
      "step": 13830
    },
    {
      "epoch": 1.768803118410122,
      "grad_norm": 9.62738747982709,
      "learning_rate": 6.800204009180459e-07,
      "logits/chosen": -0.18359375,
      "logits/rejected": 0.353515625,
      "logps/chosen": -508.0,
      "logps/rejected": -580.0,
      "loss": 0.0429,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -23.75,
      "rewards/margins": 10.0,
      "rewards/rejected": -33.75,
      "step": 13840
    },
    {
      "epoch": 1.7700811553453895,
      "grad_norm": 3.293769414840479,
      "learning_rate": 6.797748618593397e-07,
      "logits/chosen": -0.37890625,
      "logits/rejected": 0.2392578125,
      "logps/chosen": -528.0,
      "logps/rejected": -596.0,
      "loss": 0.063,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -24.25,
      "rewards/margins": 10.25,
      "rewards/rejected": -34.5,
      "step": 13850
    },
    {
      "epoch": 1.7713591922806569,
      "grad_norm": 5.195493139431868,
      "learning_rate": 6.795295885835008e-07,
      "logits/chosen": -0.396484375,
      "logits/rejected": 0.1796875,
      "logps/chosen": -482.0,
      "logps/rejected": -556.0,
      "loss": 0.0556,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -21.75,
      "rewards/margins": 10.6875,
      "rewards/rejected": -32.5,
      "step": 13860
    },
    {
      "epoch": 1.7726372292159245,
      "grad_norm": 7.365913666733674,
      "learning_rate": 6.792845806113816e-07,
      "logits/chosen": -0.283203125,
      "logits/rejected": 0.2236328125,
      "logps/chosen": -476.0,
      "logps/rejected": -560.0,
      "loss": 0.0638,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -21.875,
      "rewards/margins": 10.1875,
      "rewards/rejected": -32.0,
      "step": 13870
    },
    {
      "epoch": 1.7739152661511919,
      "grad_norm": 3.8897850649520134,
      "learning_rate": 6.790398374650439e-07,
      "logits/chosen": -0.302734375,
      "logits/rejected": 0.38671875,
      "logps/chosen": -482.0,
      "logps/rejected": -540.0,
      "loss": 0.0517,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.875,
      "rewards/margins": 9.9375,
      "rewards/rejected": -30.75,
      "step": 13880
    },
    {
      "epoch": 1.7751933030864593,
      "grad_norm": 5.935544166383281,
      "learning_rate": 6.787953586677534e-07,
      "logits/chosen": -0.26953125,
      "logits/rejected": 0.48828125,
      "logps/chosen": -464.0,
      "logps/rejected": -536.0,
      "loss": 0.0739,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -21.625,
      "rewards/margins": 10.75,
      "rewards/rejected": -32.5,
      "step": 13890
    },
    {
      "epoch": 1.7764713400217267,
      "grad_norm": 1.231432456443922,
      "learning_rate": 6.785511437439767e-07,
      "logits/chosen": -0.28515625,
      "logits/rejected": 0.474609375,
      "logps/chosen": -498.0,
      "logps/rejected": -572.0,
      "loss": 0.0756,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -22.75,
      "rewards/margins": 10.5,
      "rewards/rejected": -33.25,
      "step": 13900
    },
    {
      "epoch": 1.777749376956994,
      "grad_norm": 5.332880032603217,
      "learning_rate": 6.78307192219377e-07,
      "logits/chosen": -0.2470703125,
      "logits/rejected": 0.259765625,
      "logps/chosen": -516.0,
      "logps/rejected": -576.0,
      "loss": 0.0459,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -22.625,
      "rewards/margins": 10.4375,
      "rewards/rejected": -33.0,
      "step": 13910
    },
    {
      "epoch": 1.7790274138922615,
      "grad_norm": 6.1931928290405,
      "learning_rate": 6.780635036208104e-07,
      "logits/chosen": -0.1298828125,
      "logits/rejected": 0.486328125,
      "logps/chosen": -476.0,
      "logps/rejected": -560.0,
      "loss": 0.0524,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -23.5,
      "rewards/margins": 10.625,
      "rewards/rejected": -34.0,
      "step": 13920
    },
    {
      "epoch": 1.780305450827529,
      "grad_norm": 3.0856131391518176,
      "learning_rate": 6.77820077476322e-07,
      "logits/chosen": -0.294921875,
      "logits/rejected": 0.330078125,
      "logps/chosen": -506.0,
      "logps/rejected": -564.0,
      "loss": 0.071,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -22.625,
      "rewards/margins": 10.0,
      "rewards/rejected": -32.5,
      "step": 13930
    },
    {
      "epoch": 1.7815834877627963,
      "grad_norm": 4.426878321974366,
      "learning_rate": 6.775769133151421e-07,
      "logits/chosen": -0.263671875,
      "logits/rejected": 0.400390625,
      "logps/chosen": -488.0,
      "logps/rejected": -552.0,
      "loss": 0.0606,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -22.75,
      "rewards/margins": 10.0,
      "rewards/rejected": -32.75,
      "step": 13940
    },
    {
      "epoch": 1.7828615246980637,
      "grad_norm": 7.040579901888245,
      "learning_rate": 6.773340106676827e-07,
      "logits/chosen": -0.251953125,
      "logits/rejected": 0.515625,
      "logps/chosen": -520.0,
      "logps/rejected": -572.0,
      "loss": 0.0742,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -23.875,
      "rewards/margins": 11.25,
      "rewards/rejected": -35.0,
      "step": 13950
    },
    {
      "epoch": 1.784139561633331,
      "grad_norm": 10.42795401573382,
      "learning_rate": 6.770913690655326e-07,
      "logits/chosen": -0.357421875,
      "logits/rejected": 0.265625,
      "logps/chosen": -488.0,
      "logps/rejected": -544.0,
      "loss": 0.0791,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.75,
      "rewards/margins": 10.8125,
      "rewards/rejected": -31.5,
      "step": 13960
    },
    {
      "epoch": 1.7854175985685985,
      "grad_norm": 6.21839012657328,
      "learning_rate": 6.768489880414549e-07,
      "logits/chosen": -0.361328125,
      "logits/rejected": 0.30859375,
      "logps/chosen": -496.0,
      "logps/rejected": -568.0,
      "loss": 0.0719,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -21.5,
      "rewards/margins": 10.375,
      "rewards/rejected": -31.875,
      "step": 13970
    },
    {
      "epoch": 1.786695635503866,
      "grad_norm": 7.656146372700508,
      "learning_rate": 6.766068671293825e-07,
      "logits/chosen": -0.228515625,
      "logits/rejected": 0.470703125,
      "logps/chosen": -492.0,
      "logps/rejected": -556.0,
      "loss": 0.0624,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -22.5,
      "rewards/margins": 10.1875,
      "rewards/rejected": -32.75,
      "step": 13980
    },
    {
      "epoch": 1.7879736724391335,
      "grad_norm": 5.145286493907716,
      "learning_rate": 6.763650058644149e-07,
      "logits/chosen": -0.166015625,
      "logits/rejected": 0.384765625,
      "logps/chosen": -494.0,
      "logps/rejected": -540.0,
      "loss": 0.0624,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.75,
      "rewards/margins": 10.1875,
      "rewards/rejected": -32.0,
      "step": 13990
    },
    {
      "epoch": 1.789251709374401,
      "grad_norm": 4.04334846032706,
      "learning_rate": 6.761234037828132e-07,
      "logits/chosen": -0.2119140625,
      "logits/rejected": 0.38671875,
      "logps/chosen": -498.0,
      "logps/rejected": -560.0,
      "loss": 0.0513,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -23.0,
      "rewards/margins": 9.875,
      "rewards/rejected": -33.0,
      "step": 14000
    },
    {
      "epoch": 1.7905297463096683,
      "grad_norm": 5.5582914420466185,
      "learning_rate": 6.758820604219981e-07,
      "logits/chosen": -0.12158203125,
      "logits/rejected": 0.447265625,
      "logps/chosen": -506.0,
      "logps/rejected": -560.0,
      "loss": 0.0584,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -24.25,
      "rewards/margins": 8.75,
      "rewards/rejected": -33.0,
      "step": 14010
    },
    {
      "epoch": 1.7918077832449357,
      "grad_norm": 3.4160114371850585,
      "learning_rate": 6.756409753205447e-07,
      "logits/chosen": -0.107421875,
      "logits/rejected": 0.451171875,
      "logps/chosen": -476.0,
      "logps/rejected": -560.0,
      "loss": 0.0495,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -22.75,
      "rewards/margins": 9.375,
      "rewards/rejected": -32.25,
      "step": 14020
    },
    {
      "epoch": 1.7930858201802033,
      "grad_norm": 4.857783356819474,
      "learning_rate": 6.754001480181798e-07,
      "logits/chosen": -0.1943359375,
      "logits/rejected": 0.4453125,
      "logps/chosen": -484.0,
      "logps/rejected": -540.0,
      "loss": 0.0729,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -21.25,
      "rewards/margins": 9.25,
      "rewards/rejected": -30.625,
      "step": 14030
    },
    {
      "epoch": 1.7943638571154708,
      "grad_norm": 5.648033749586228,
      "learning_rate": 6.751595780557777e-07,
      "logits/chosen": -0.31640625,
      "logits/rejected": 0.1865234375,
      "logps/chosen": -466.0,
      "logps/rejected": -540.0,
      "loss": 0.0776,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.5,
      "rewards/margins": 10.1875,
      "rewards/rejected": -30.75,
      "step": 14040
    },
    {
      "epoch": 1.7956418940507382,
      "grad_norm": 9.423801575399565,
      "learning_rate": 6.749192649753564e-07,
      "logits/chosen": -0.240234375,
      "logits/rejected": 0.255859375,
      "logps/chosen": -466.0,
      "logps/rejected": -544.0,
      "loss": 0.0453,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.125,
      "rewards/margins": 9.0,
      "rewards/rejected": -30.125,
      "step": 14050
    },
    {
      "epoch": 1.7969199309860056,
      "grad_norm": 4.484408140183514,
      "learning_rate": 6.746792083200745e-07,
      "logits/chosen": -0.439453125,
      "logits/rejected": 0.248046875,
      "logps/chosen": -504.0,
      "logps/rejected": -556.0,
      "loss": 0.0458,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -21.5,
      "rewards/margins": 10.75,
      "rewards/rejected": -32.25,
      "step": 14060
    },
    {
      "epoch": 1.798197967921273,
      "grad_norm": 4.312598366978668,
      "learning_rate": 6.74439407634227e-07,
      "logits/chosen": -0.421875,
      "logits/rejected": 0.40625,
      "logps/chosen": -482.0,
      "logps/rejected": -544.0,
      "loss": 0.0643,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -22.125,
      "rewards/margins": 9.625,
      "rewards/rejected": -31.75,
      "step": 14070
    },
    {
      "epoch": 1.7994760048565404,
      "grad_norm": 5.933055360218132,
      "learning_rate": 6.74199862463242e-07,
      "logits/chosen": -0.3671875,
      "logits/rejected": 0.2265625,
      "logps/chosen": -496.0,
      "logps/rejected": -576.0,
      "loss": 0.0569,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -22.125,
      "rewards/margins": 10.875,
      "rewards/rejected": -33.0,
      "step": 14080
    },
    {
      "epoch": 1.8007540417918078,
      "grad_norm": 5.745421942795816,
      "learning_rate": 6.739605723536771e-07,
      "logits/chosen": -0.177734375,
      "logits/rejected": 0.41015625,
      "logps/chosen": -500.0,
      "logps/rejected": -572.0,
      "loss": 0.0644,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -23.875,
      "rewards/margins": 10.125,
      "rewards/rejected": -34.0,
      "step": 14090
    },
    {
      "epoch": 1.8020320787270752,
      "grad_norm": 5.241369339439487,
      "learning_rate": 6.737215368532152e-07,
      "logits/chosen": -0.30859375,
      "logits/rejected": 0.37109375,
      "logps/chosen": -506.0,
      "logps/rejected": -560.0,
      "loss": 0.0599,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -23.375,
      "rewards/margins": 10.0625,
      "rewards/rejected": -33.5,
      "step": 14100
    },
    {
      "epoch": 1.8033101156623426,
      "grad_norm": 2.3287985046674233,
      "learning_rate": 6.734827555106618e-07,
      "logits/chosen": -0.314453125,
      "logits/rejected": 0.224609375,
      "logps/chosen": -520.0,
      "logps/rejected": -556.0,
      "loss": 0.071,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -23.125,
      "rewards/margins": 9.5,
      "rewards/rejected": -32.5,
      "step": 14110
    },
    {
      "epoch": 1.80458815259761,
      "grad_norm": 2.273237160784909,
      "learning_rate": 6.73244227875941e-07,
      "logits/chosen": -0.1904296875,
      "logits/rejected": 0.357421875,
      "logps/chosen": -480.0,
      "logps/rejected": -564.0,
      "loss": 0.0657,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -22.875,
      "rewards/margins": 10.0625,
      "rewards/rejected": -33.0,
      "step": 14120
    },
    {
      "epoch": 1.8058661895328774,
      "grad_norm": 7.531903592882603,
      "learning_rate": 6.730059535000916e-07,
      "logits/chosen": -0.337890625,
      "logits/rejected": 0.419921875,
      "logps/chosen": -532.0,
      "logps/rejected": -592.0,
      "loss": 0.0595,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -24.25,
      "rewards/margins": 11.0,
      "rewards/rejected": -35.25,
      "step": 14130
    },
    {
      "epoch": 1.8071442264681448,
      "grad_norm": 5.717181769251435,
      "learning_rate": 6.727679319352644e-07,
      "logits/chosen": -0.244140625,
      "logits/rejected": 0.3515625,
      "logps/chosen": -498.0,
      "logps/rejected": -568.0,
      "loss": 0.0448,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -23.0,
      "rewards/margins": 10.5,
      "rewards/rejected": -33.5,
      "step": 14140
    },
    {
      "epoch": 1.8084222634034124,
      "grad_norm": 4.7874696517226045,
      "learning_rate": 6.725301627347177e-07,
      "logits/chosen": -0.2734375,
      "logits/rejected": 0.59375,
      "logps/chosen": -488.0,
      "logps/rejected": -536.0,
      "loss": 0.0555,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -22.0,
      "rewards/margins": 10.5,
      "rewards/rejected": -32.5,
      "step": 14150
    },
    {
      "epoch": 1.8097003003386798,
      "grad_norm": 3.6754611893164197,
      "learning_rate": 6.722926454528143e-07,
      "logits/chosen": -0.265625,
      "logits/rejected": 0.458984375,
      "logps/chosen": -496.0,
      "logps/rejected": -556.0,
      "loss": 0.0582,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -23.25,
      "rewards/margins": 9.5625,
      "rewards/rejected": -33.0,
      "step": 14160
    },
    {
      "epoch": 1.8109783372739472,
      "grad_norm": 6.89737470007493,
      "learning_rate": 6.720553796450181e-07,
      "logits/chosen": -0.306640625,
      "logits/rejected": 0.30078125,
      "logps/chosen": -470.0,
      "logps/rejected": -544.0,
      "loss": 0.0652,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -21.0,
      "rewards/margins": 10.9375,
      "rewards/rejected": -31.875,
      "step": 14170
    },
    {
      "epoch": 1.8122563742092146,
      "grad_norm": 5.494871832190991,
      "learning_rate": 6.718183648678903e-07,
      "logits/chosen": -0.322265625,
      "logits/rejected": 0.36328125,
      "logps/chosen": -458.0,
      "logps/rejected": -516.0,
      "loss": 0.0634,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.625,
      "rewards/margins": 10.3125,
      "rewards/rejected": -30.0,
      "step": 14180
    },
    {
      "epoch": 1.8135344111444822,
      "grad_norm": 4.742978592999589,
      "learning_rate": 6.715816006790862e-07,
      "logits/chosen": -0.291015625,
      "logits/rejected": 0.19140625,
      "logps/chosen": -486.0,
      "logps/rejected": -552.0,
      "loss": 0.0727,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.5,
      "rewards/margins": 9.9375,
      "rewards/rejected": -30.5,
      "step": 14190
    },
    {
      "epoch": 1.8148124480797496,
      "grad_norm": 6.3507296588062605,
      "learning_rate": 6.713450866373512e-07,
      "logits/chosen": -0.130859375,
      "logits/rejected": 0.3203125,
      "logps/chosen": -460.0,
      "logps/rejected": -548.0,
      "loss": 0.0415,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -21.25,
      "rewards/margins": 9.375,
      "rewards/rejected": -30.625,
      "step": 14200
    },
    {
      "epoch": 1.816090485015017,
      "grad_norm": 6.444015702180477,
      "learning_rate": 6.711088223025183e-07,
      "logits/chosen": -0.2392578125,
      "logits/rejected": 0.484375,
      "logps/chosen": -474.0,
      "logps/rejected": -528.0,
      "loss": 0.0479,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.25,
      "rewards/margins": 9.9375,
      "rewards/rejected": -31.25,
      "step": 14210
    },
    {
      "epoch": 1.8173685219502844,
      "grad_norm": 10.594355018938765,
      "learning_rate": 6.708728072355036e-07,
      "logits/chosen": -0.3671875,
      "logits/rejected": 0.326171875,
      "logps/chosen": -508.0,
      "logps/rejected": -596.0,
      "loss": 0.0646,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -22.25,
      "rewards/margins": 10.6875,
      "rewards/rejected": -32.75,
      "step": 14220
    },
    {
      "epoch": 1.8186465588855518,
      "grad_norm": 3.2602929804592646,
      "learning_rate": 6.706370409983032e-07,
      "logits/chosen": -0.37109375,
      "logits/rejected": 0.271484375,
      "logps/chosen": -470.0,
      "logps/rejected": -552.0,
      "loss": 0.0476,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.625,
      "rewards/margins": 10.6875,
      "rewards/rejected": -31.375,
      "step": 14230
    },
    {
      "epoch": 1.8199245958208192,
      "grad_norm": 9.626351693033685,
      "learning_rate": 6.704015231539909e-07,
      "logits/chosen": -0.51171875,
      "logits/rejected": 0.1826171875,
      "logps/chosen": -492.0,
      "logps/rejected": -528.0,
      "loss": 0.0553,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -19.875,
      "rewards/margins": 10.1875,
      "rewards/rejected": -30.125,
      "step": 14240
    },
    {
      "epoch": 1.8212026327560866,
      "grad_norm": 5.689836280681443,
      "learning_rate": 6.701662532667127e-07,
      "logits/chosen": -0.44921875,
      "logits/rejected": 0.1962890625,
      "logps/chosen": -476.0,
      "logps/rejected": -536.0,
      "loss": 0.0544,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.25,
      "rewards/margins": 9.8125,
      "rewards/rejected": -29.0,
      "step": 14250
    },
    {
      "epoch": 1.822480669691354,
      "grad_norm": 4.684754417089129,
      "learning_rate": 6.699312309016855e-07,
      "logits/chosen": -0.458984375,
      "logits/rejected": 0.07373046875,
      "logps/chosen": -492.0,
      "logps/rejected": -540.0,
      "loss": 0.0765,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.375,
      "rewards/margins": 9.1875,
      "rewards/rejected": -30.625,
      "step": 14260
    },
    {
      "epoch": 1.8237587066266214,
      "grad_norm": 6.943006994531986,
      "learning_rate": 6.696964556251918e-07,
      "logits/chosen": -0.40625,
      "logits/rejected": 0.291015625,
      "logps/chosen": -468.0,
      "logps/rejected": -532.0,
      "loss": 0.0467,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.125,
      "rewards/margins": 10.75,
      "rewards/rejected": -30.875,
      "step": 14270
    },
    {
      "epoch": 1.8250367435618888,
      "grad_norm": 6.60999652840603,
      "learning_rate": 6.694619270045784e-07,
      "logits/chosen": -0.36328125,
      "logits/rejected": 0.29296875,
      "logps/chosen": -500.0,
      "logps/rejected": -548.0,
      "loss": 0.0601,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.125,
      "rewards/margins": 10.4375,
      "rewards/rejected": -31.5,
      "step": 14280
    },
    {
      "epoch": 1.8263147804971562,
      "grad_norm": 3.9962135255740154,
      "learning_rate": 6.69227644608251e-07,
      "logits/chosen": -0.29296875,
      "logits/rejected": 0.42578125,
      "logps/chosen": -466.0,
      "logps/rejected": -532.0,
      "loss": 0.0787,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.375,
      "rewards/margins": 10.1875,
      "rewards/rejected": -30.625,
      "step": 14290
    },
    {
      "epoch": 1.8275928174324236,
      "grad_norm": 7.489910669860267,
      "learning_rate": 6.689936080056727e-07,
      "logits/chosen": -0.1474609375,
      "logits/rejected": 0.388671875,
      "logps/chosen": -464.0,
      "logps/rejected": -556.0,
      "loss": 0.0481,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.0,
      "rewards/margins": 10.375,
      "rewards/rejected": -31.375,
      "step": 14300
    },
    {
      "epoch": 1.8288708543676913,
      "grad_norm": 4.437793734995448,
      "learning_rate": 6.687598167673588e-07,
      "logits/chosen": -0.263671875,
      "logits/rejected": 0.25390625,
      "logps/chosen": -460.0,
      "logps/rejected": -532.0,
      "loss": 0.0608,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.0,
      "rewards/margins": 10.5,
      "rewards/rejected": -30.5,
      "step": 14310
    },
    {
      "epoch": 1.8301488913029587,
      "grad_norm": 6.502742627929175,
      "learning_rate": 6.685262704648755e-07,
      "logits/chosen": -0.365234375,
      "logits/rejected": 0.3125,
      "logps/chosen": -468.0,
      "logps/rejected": -536.0,
      "loss": 0.0607,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.375,
      "rewards/margins": 10.125,
      "rewards/rejected": -30.625,
      "step": 14320
    },
    {
      "epoch": 1.831426928238226,
      "grad_norm": 4.52435995604078,
      "learning_rate": 6.682929686708352e-07,
      "logits/chosen": -0.3515625,
      "logits/rejected": 0.1845703125,
      "logps/chosen": -446.0,
      "logps/rejected": -524.0,
      "loss": 0.0697,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.625,
      "rewards/margins": 10.375,
      "rewards/rejected": -29.0,
      "step": 14330
    },
    {
      "epoch": 1.8327049651734935,
      "grad_norm": 5.412485905351788,
      "learning_rate": 6.680599109588932e-07,
      "logits/chosen": -0.34375,
      "logits/rejected": 0.2490234375,
      "logps/chosen": -472.0,
      "logps/rejected": -532.0,
      "loss": 0.0559,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.0,
      "rewards/margins": 10.125,
      "rewards/rejected": -30.125,
      "step": 14340
    },
    {
      "epoch": 1.833983002108761,
      "grad_norm": 5.868785950760559,
      "learning_rate": 6.678270969037457e-07,
      "logits/chosen": -0.31640625,
      "logits/rejected": 0.263671875,
      "logps/chosen": -464.0,
      "logps/rejected": -540.0,
      "loss": 0.0468,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.875,
      "rewards/margins": 10.625,
      "rewards/rejected": -30.5,
      "step": 14350
    },
    {
      "epoch": 1.8352610390440285,
      "grad_norm": 4.84641638599339,
      "learning_rate": 6.67594526081125e-07,
      "logits/chosen": -0.3203125,
      "logits/rejected": 0.333984375,
      "logps/chosen": -466.0,
      "logps/rejected": -504.0,
      "loss": 0.0554,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -19.125,
      "rewards/margins": 9.0625,
      "rewards/rejected": -28.125,
      "step": 14360
    },
    {
      "epoch": 1.8365390759792959,
      "grad_norm": 5.515342225958698,
      "learning_rate": 6.673621980677971e-07,
      "logits/chosen": -0.322265625,
      "logits/rejected": 0.35546875,
      "logps/chosen": -448.0,
      "logps/rejected": -502.0,
      "loss": 0.0431,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.0,
      "rewards/margins": 10.0,
      "rewards/rejected": -28.0,
      "step": 14370
    },
    {
      "epoch": 1.8378171129145633,
      "grad_norm": 1.4592821029666723,
      "learning_rate": 6.671301124415585e-07,
      "logits/chosen": -0.482421875,
      "logits/rejected": 0.1650390625,
      "logps/chosen": -466.0,
      "logps/rejected": -496.0,
      "loss": 0.0566,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.625,
      "rewards/margins": 9.5,
      "rewards/rejected": -27.125,
      "step": 14380
    },
    {
      "epoch": 1.8390951498498307,
      "grad_norm": 4.7003908716508604,
      "learning_rate": 6.668982687812326e-07,
      "logits/chosen": -0.408203125,
      "logits/rejected": 0.28515625,
      "logps/chosen": -452.0,
      "logps/rejected": -510.0,
      "loss": 0.0534,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.375,
      "rewards/margins": 9.8125,
      "rewards/rejected": -28.25,
      "step": 14390
    },
    {
      "epoch": 1.840373186785098,
      "grad_norm": 2.3946552677716477,
      "learning_rate": 6.666666666666666e-07,
      "logits/chosen": -0.328125,
      "logits/rejected": 0.294921875,
      "logps/chosen": -480.0,
      "logps/rejected": -532.0,
      "loss": 0.0743,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -20.5,
      "rewards/margins": 10.4375,
      "rewards/rejected": -31.0,
      "step": 14400
    },
    {
      "epoch": 1.8416512237203655,
      "grad_norm": 6.21534401376548,
      "learning_rate": 6.664353056787287e-07,
      "logits/chosen": -0.349609375,
      "logits/rejected": 0.328125,
      "logps/chosen": -496.0,
      "logps/rejected": -556.0,
      "loss": 0.0634,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -21.625,
      "rewards/margins": 10.0625,
      "rewards/rejected": -31.625,
      "step": 14410
    },
    {
      "epoch": 1.842929260655633,
      "grad_norm": 7.825242015297215,
      "learning_rate": 6.662041853993043e-07,
      "logits/chosen": -0.357421875,
      "logits/rejected": 0.201171875,
      "logps/chosen": -498.0,
      "logps/rejected": -580.0,
      "loss": 0.067,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -20.75,
      "rewards/margins": 10.4375,
      "rewards/rejected": -31.125,
      "step": 14420
    },
    {
      "epoch": 1.8442072975909003,
      "grad_norm": 5.034115914366234,
      "learning_rate": 6.659733054112932e-07,
      "logits/chosen": -0.171875,
      "logits/rejected": 0.396484375,
      "logps/chosen": -484.0,
      "logps/rejected": -528.0,
      "loss": 0.0558,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -21.25,
      "rewards/margins": 9.5625,
      "rewards/rejected": -30.875,
      "step": 14430
    },
    {
      "epoch": 1.8454853345261677,
      "grad_norm": 3.9141749954505487,
      "learning_rate": 6.657426652986061e-07,
      "logits/chosen": -0.23046875,
      "logits/rejected": 0.345703125,
      "logps/chosen": -466.0,
      "logps/rejected": -524.0,
      "loss": 0.0588,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.875,
      "rewards/margins": 9.375,
      "rewards/rejected": -30.25,
      "step": 14440
    },
    {
      "epoch": 1.846763371461435,
      "grad_norm": 3.396626627151028,
      "learning_rate": 6.655122646461624e-07,
      "logits/chosen": -0.228515625,
      "logits/rejected": 0.380859375,
      "logps/chosen": -490.0,
      "logps/rejected": -552.0,
      "loss": 0.0459,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.25,
      "rewards/margins": 10.625,
      "rewards/rejected": -32.0,
      "step": 14450
    },
    {
      "epoch": 1.8480414083967025,
      "grad_norm": 13.55312653891372,
      "learning_rate": 6.652821030398855e-07,
      "logits/chosen": -0.357421875,
      "logits/rejected": 0.33203125,
      "logps/chosen": -488.0,
      "logps/rejected": -560.0,
      "loss": 0.0729,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.375,
      "rewards/margins": 10.875,
      "rewards/rejected": -32.25,
      "step": 14460
    },
    {
      "epoch": 1.8493194453319701,
      "grad_norm": 6.317894295144871,
      "learning_rate": 6.650521800667013e-07,
      "logits/chosen": -0.291015625,
      "logits/rejected": 0.34765625,
      "logps/chosen": -490.0,
      "logps/rejected": -548.0,
      "loss": 0.0695,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -22.125,
      "rewards/margins": 9.8125,
      "rewards/rejected": -31.875,
      "step": 14470
    },
    {
      "epoch": 1.8505974822672375,
      "grad_norm": 8.2841995452199,
      "learning_rate": 6.648224953145336e-07,
      "logits/chosen": -0.265625,
      "logits/rejected": 0.39453125,
      "logps/chosen": -506.0,
      "logps/rejected": -552.0,
      "loss": 0.0798,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -22.125,
      "rewards/margins": 10.75,
      "rewards/rejected": -33.0,
      "step": 14480
    },
    {
      "epoch": 1.851875519202505,
      "grad_norm": 9.276467825673807,
      "learning_rate": 6.645930483723025e-07,
      "logits/chosen": -0.3046875,
      "logits/rejected": 0.251953125,
      "logps/chosen": -488.0,
      "logps/rejected": -560.0,
      "loss": 0.0643,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -22.0,
      "rewards/margins": 10.0,
      "rewards/rejected": -31.875,
      "step": 14490
    },
    {
      "epoch": 1.8531535561377723,
      "grad_norm": 2.5323657996349205,
      "learning_rate": 6.643638388299198e-07,
      "logits/chosen": -0.296875,
      "logits/rejected": 0.396484375,
      "logps/chosen": -480.0,
      "logps/rejected": -544.0,
      "loss": 0.056,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -21.375,
      "rewards/margins": 10.5625,
      "rewards/rejected": -32.0,
      "step": 14500
    },
    {
      "epoch": 1.85443159307304,
      "grad_norm": 7.295307998718311,
      "learning_rate": 6.64134866278287e-07,
      "logits/chosen": -0.2392578125,
      "logits/rejected": 0.46484375,
      "logps/chosen": -478.0,
      "logps/rejected": -532.0,
      "loss": 0.0503,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -21.125,
      "rewards/margins": 9.125,
      "rewards/rejected": -30.375,
      "step": 14510
    },
    {
      "epoch": 1.8557096300083074,
      "grad_norm": 5.452384277232883,
      "learning_rate": 6.639061303092922e-07,
      "logits/chosen": -0.32421875,
      "logits/rejected": 0.392578125,
      "logps/chosen": -516.0,
      "logps/rejected": -576.0,
      "loss": 0.0497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -22.0,
      "rewards/margins": 10.625,
      "rewards/rejected": -32.5,
      "step": 14520
    },
    {
      "epoch": 1.8569876669435748,
      "grad_norm": 7.740699803303673,
      "learning_rate": 6.636776305158062e-07,
      "logits/chosen": -0.2890625,
      "logits/rejected": 0.234375,
      "logps/chosen": -478.0,
      "logps/rejected": -560.0,
      "loss": 0.0671,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.75,
      "rewards/margins": 10.1875,
      "rewards/rejected": -31.0,
      "step": 14530
    },
    {
      "epoch": 1.8582657038788422,
      "grad_norm": 2.255586991913487,
      "learning_rate": 6.634493664916802e-07,
      "logits/chosen": -0.244140625,
      "logits/rejected": 0.3046875,
      "logps/chosen": -484.0,
      "logps/rejected": -540.0,
      "loss": 0.049,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -21.375,
      "rewards/margins": 9.0625,
      "rewards/rejected": -30.5,
      "step": 14540
    },
    {
      "epoch": 1.8595437408141096,
      "grad_norm": 1.6607847285940949,
      "learning_rate": 6.632213378317426e-07,
      "logits/chosen": -0.1796875,
      "logits/rejected": 0.46875,
      "logps/chosen": -492.0,
      "logps/rejected": -544.0,
      "loss": 0.0458,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -22.125,
      "rewards/margins": 10.1875,
      "rewards/rejected": -32.25,
      "step": 14550
    },
    {
      "epoch": 1.860821777749377,
      "grad_norm": 2.7400576640088863,
      "learning_rate": 6.629935441317959e-07,
      "logits/chosen": -0.1865234375,
      "logits/rejected": 0.35546875,
      "logps/chosen": -470.0,
      "logps/rejected": -532.0,
      "loss": 0.078,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -21.375,
      "rewards/margins": 9.1875,
      "rewards/rejected": -30.5,
      "step": 14560
    },
    {
      "epoch": 1.8620998146846444,
      "grad_norm": 8.104983107122214,
      "learning_rate": 6.627659849886136e-07,
      "logits/chosen": -0.3203125,
      "logits/rejected": 0.44921875,
      "logps/chosen": -492.0,
      "logps/rejected": -564.0,
      "loss": 0.0415,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -22.25,
      "rewards/margins": 10.4375,
      "rewards/rejected": -32.75,
      "step": 14570
    },
    {
      "epoch": 1.8633778516199118,
      "grad_norm": 10.414065040147147,
      "learning_rate": 6.625386599999376e-07,
      "logits/chosen": -0.30078125,
      "logits/rejected": 0.400390625,
      "logps/chosen": -482.0,
      "logps/rejected": -536.0,
      "loss": 0.0627,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.5,
      "rewards/margins": 10.375,
      "rewards/rejected": -31.875,
      "step": 14580
    },
    {
      "epoch": 1.8646558885551792,
      "grad_norm": 5.96909195114153,
      "learning_rate": 6.623115687644749e-07,
      "logits/chosen": -0.34765625,
      "logits/rejected": 0.353515625,
      "logps/chosen": -500.0,
      "logps/rejected": -552.0,
      "loss": 0.0466,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -22.5,
      "rewards/margins": 10.5,
      "rewards/rejected": -33.0,
      "step": 14590
    },
    {
      "epoch": 1.8659339254904466,
      "grad_norm": 8.942256773330856,
      "learning_rate": 6.620847108818943e-07,
      "logits/chosen": -0.443359375,
      "logits/rejected": 0.296875,
      "logps/chosen": -506.0,
      "logps/rejected": -580.0,
      "loss": 0.0529,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -22.25,
      "rewards/margins": 11.375,
      "rewards/rejected": -33.5,
      "step": 14600
    },
    {
      "epoch": 1.867211962425714,
      "grad_norm": 6.995215480522469,
      "learning_rate": 6.618580859528244e-07,
      "logits/chosen": -0.275390625,
      "logits/rejected": 0.4296875,
      "logps/chosen": -494.0,
      "logps/rejected": -548.0,
      "loss": 0.0768,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -22.0,
      "rewards/margins": 9.9375,
      "rewards/rejected": -32.0,
      "step": 14610
    },
    {
      "epoch": 1.8684899993609814,
      "grad_norm": 4.951954614169423,
      "learning_rate": 6.616316935788494e-07,
      "logits/chosen": -0.240234375,
      "logits/rejected": 0.32421875,
      "logps/chosen": -486.0,
      "logps/rejected": -548.0,
      "loss": 0.0665,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -22.5,
      "rewards/margins": 9.625,
      "rewards/rejected": -32.25,
      "step": 14620
    },
    {
      "epoch": 1.869768036296249,
      "grad_norm": 1.1177496700682252,
      "learning_rate": 6.614055333625071e-07,
      "logits/chosen": -0.25390625,
      "logits/rejected": 0.240234375,
      "logps/chosen": -476.0,
      "logps/rejected": -540.0,
      "loss": 0.0536,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.25,
      "rewards/margins": 9.25,
      "rewards/rejected": -30.625,
      "step": 14630
    },
    {
      "epoch": 1.8710460732315164,
      "grad_norm": 3.0199228230355155,
      "learning_rate": 6.611796049072861e-07,
      "logits/chosen": -0.421875,
      "logits/rejected": 0.279296875,
      "logps/chosen": -528.0,
      "logps/rejected": -552.0,
      "loss": 0.0598,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -21.875,
      "rewards/margins": 9.75,
      "rewards/rejected": -31.625,
      "step": 14640
    },
    {
      "epoch": 1.8723241101667838,
      "grad_norm": 7.818084703047404,
      "learning_rate": 6.60953907817622e-07,
      "logits/chosen": -0.42578125,
      "logits/rejected": 0.353515625,
      "logps/chosen": -446.0,
      "logps/rejected": -498.0,
      "loss": 0.0582,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.125,
      "rewards/margins": 10.0625,
      "rewards/rejected": -29.125,
      "step": 14650
    },
    {
      "epoch": 1.8736021471020512,
      "grad_norm": 3.366387610882266,
      "learning_rate": 6.607284416988948e-07,
      "logits/chosen": -0.298828125,
      "logits/rejected": 0.2314453125,
      "logps/chosen": -456.0,
      "logps/rejected": -528.0,
      "loss": 0.0498,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.0,
      "rewards/margins": 9.0,
      "rewards/rejected": -29.0,
      "step": 14660
    },
    {
      "epoch": 1.8748801840373188,
      "grad_norm": 3.9565010428327083,
      "learning_rate": 6.605032061574266e-07,
      "logits/chosen": -0.400390625,
      "logits/rejected": 0.166015625,
      "logps/chosen": -480.0,
      "logps/rejected": -532.0,
      "loss": 0.0597,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.875,
      "rewards/margins": 10.0625,
      "rewards/rejected": -29.875,
      "step": 14670
    },
    {
      "epoch": 1.8761582209725862,
      "grad_norm": 5.550606107431804,
      "learning_rate": 6.602782008004778e-07,
      "logits/chosen": -0.359375,
      "logits/rejected": 0.34375,
      "logps/chosen": -468.0,
      "logps/rejected": -512.0,
      "loss": 0.0567,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.875,
      "rewards/margins": 10.375,
      "rewards/rejected": -30.25,
      "step": 14680
    },
    {
      "epoch": 1.8774362579078536,
      "grad_norm": 5.973825242201665,
      "learning_rate": 6.600534252362451e-07,
      "logits/chosen": -0.423828125,
      "logits/rejected": 0.220703125,
      "logps/chosen": -450.0,
      "logps/rejected": -516.0,
      "loss": 0.0426,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.75,
      "rewards/margins": 10.3125,
      "rewards/rejected": -29.0,
      "step": 14690
    },
    {
      "epoch": 1.878714294843121,
      "grad_norm": 0.94930258712671,
      "learning_rate": 6.59828879073858e-07,
      "logits/chosen": -0.46484375,
      "logits/rejected": 0.357421875,
      "logps/chosen": -494.0,
      "logps/rejected": -536.0,
      "loss": 0.0644,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -21.25,
      "rewards/margins": 10.25,
      "rewards/rejected": -31.5,
      "step": 14700
    },
    {
      "epoch": 1.8799923317783884,
      "grad_norm": 11.124178301150023,
      "learning_rate": 6.596045619233764e-07,
      "logits/chosen": -0.5234375,
      "logits/rejected": 0.21484375,
      "logps/chosen": -516.0,
      "logps/rejected": -560.0,
      "loss": 0.065,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.75,
      "rewards/margins": 11.375,
      "rewards/rejected": -32.0,
      "step": 14710
    },
    {
      "epoch": 1.8812703687136558,
      "grad_norm": 4.941861806219977,
      "learning_rate": 6.59380473395787e-07,
      "logits/chosen": -0.39453125,
      "logits/rejected": 0.306640625,
      "logps/chosen": -472.0,
      "logps/rejected": -524.0,
      "loss": 0.0632,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -20.25,
      "rewards/margins": 9.6875,
      "rewards/rejected": -30.0,
      "step": 14720
    },
    {
      "epoch": 1.8825484056489232,
      "grad_norm": 4.68453864858164,
      "learning_rate": 6.591566131030017e-07,
      "logits/chosen": -0.466796875,
      "logits/rejected": 0.1591796875,
      "logps/chosen": -482.0,
      "logps/rejected": -548.0,
      "loss": 0.0433,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.875,
      "rewards/margins": 10.25,
      "rewards/rejected": -30.125,
      "step": 14730
    },
    {
      "epoch": 1.8838264425841906,
      "grad_norm": 4.009003400734236,
      "learning_rate": 6.589329806578535e-07,
      "logits/chosen": -0.453125,
      "logits/rejected": 0.1982421875,
      "logps/chosen": -456.0,
      "logps/rejected": -520.0,
      "loss": 0.0419,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.125,
      "rewards/margins": 9.3125,
      "rewards/rejected": -28.5,
      "step": 14740
    },
    {
      "epoch": 1.885104479519458,
      "grad_norm": 4.768821555445601,
      "learning_rate": 6.587095756740946e-07,
      "logits/chosen": -0.5390625,
      "logits/rejected": 0.107421875,
      "logps/chosen": -466.0,
      "logps/rejected": -532.0,
      "loss": 0.0553,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.125,
      "rewards/margins": 10.3125,
      "rewards/rejected": -29.5,
      "step": 14750
    },
    {
      "epoch": 1.8863825164547254,
      "grad_norm": 3.8970028012706788,
      "learning_rate": 6.58486397766393e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": 0.189453125,
      "logps/chosen": -468.0,
      "logps/rejected": -516.0,
      "loss": 0.0421,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.125,
      "rewards/margins": 10.0625,
      "rewards/rejected": -28.25,
      "step": 14760
    },
    {
      "epoch": 1.8876605533899928,
      "grad_norm": 7.436303526015631,
      "learning_rate": 6.582634465503303e-07,
      "logits/chosen": -0.52734375,
      "logits/rejected": 0.142578125,
      "logps/chosen": -444.0,
      "logps/rejected": -520.0,
      "loss": 0.053,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.5,
      "rewards/margins": 10.25,
      "rewards/rejected": -28.75,
      "step": 14770
    },
    {
      "epoch": 1.8889385903252602,
      "grad_norm": 3.7241983478045944,
      "learning_rate": 6.580407216423983e-07,
      "logits/chosen": -0.31640625,
      "logits/rejected": 0.1708984375,
      "logps/chosen": -426.0,
      "logps/rejected": -524.0,
      "loss": 0.0639,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -18.0,
      "rewards/margins": 11.5,
      "rewards/rejected": -29.5,
      "step": 14780
    },
    {
      "epoch": 1.8902166272605279,
      "grad_norm": 5.319109267536875,
      "learning_rate": 6.578182226599962e-07,
      "logits/chosen": -0.458984375,
      "logits/rejected": 0.2265625,
      "logps/chosen": -486.0,
      "logps/rejected": -556.0,
      "loss": 0.0566,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.625,
      "rewards/margins": 10.8125,
      "rewards/rejected": -30.5,
      "step": 14790
    },
    {
      "epoch": 1.8914946641957953,
      "grad_norm": 5.062449260438405,
      "learning_rate": 6.575959492214291e-07,
      "logits/chosen": -0.390625,
      "logits/rejected": 0.30859375,
      "logps/chosen": -480.0,
      "logps/rejected": -556.0,
      "loss": 0.0454,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.625,
      "rewards/margins": 10.625,
      "rewards/rejected": -31.25,
      "step": 14800
    },
    {
      "epoch": 1.8927727011310627,
      "grad_norm": 8.042538847229746,
      "learning_rate": 6.573739009459035e-07,
      "logits/chosen": -0.5,
      "logits/rejected": 0.140625,
      "logps/chosen": -520.0,
      "logps/rejected": -568.0,
      "loss": 0.0377,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -21.875,
      "rewards/margins": 10.5625,
      "rewards/rejected": -32.5,
      "step": 14810
    },
    {
      "epoch": 1.89405073806633,
      "grad_norm": 6.740410232113967,
      "learning_rate": 6.571520774535256e-07,
      "logits/chosen": -0.26953125,
      "logits/rejected": 0.359375,
      "logps/chosen": -468.0,
      "logps/rejected": -548.0,
      "loss": 0.0606,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -21.875,
      "rewards/margins": 10.625,
      "rewards/rejected": -32.5,
      "step": 14820
    },
    {
      "epoch": 1.8953287750015977,
      "grad_norm": 5.355313274737181,
      "learning_rate": 6.569304783652981e-07,
      "logits/chosen": -0.447265625,
      "logits/rejected": 0.259765625,
      "logps/chosen": -480.0,
      "logps/rejected": -540.0,
      "loss": 0.0511,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.25,
      "rewards/margins": 10.3125,
      "rewards/rejected": -30.625,
      "step": 14830
    },
    {
      "epoch": 1.896606811936865,
      "grad_norm": 4.1617165318588825,
      "learning_rate": 6.567091033031185e-07,
      "logits/chosen": -0.326171875,
      "logits/rejected": 0.26953125,
      "logps/chosen": -468.0,
      "logps/rejected": -532.0,
      "loss": 0.0665,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.125,
      "rewards/margins": 10.375,
      "rewards/rejected": -30.5,
      "step": 14840
    },
    {
      "epoch": 1.8978848488721325,
      "grad_norm": 5.897667465928775,
      "learning_rate": 6.564879518897744e-07,
      "logits/chosen": -0.345703125,
      "logits/rejected": 0.2333984375,
      "logps/chosen": -480.0,
      "logps/rejected": -552.0,
      "loss": 0.0547,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.75,
      "rewards/margins": 9.9375,
      "rewards/rejected": -31.625,
      "step": 14850
    },
    {
      "epoch": 1.8991628858074,
      "grad_norm": 10.297686195963305,
      "learning_rate": 6.562670237489432e-07,
      "logits/chosen": -0.3671875,
      "logits/rejected": 0.259765625,
      "logps/chosen": -506.0,
      "logps/rejected": -564.0,
      "loss": 0.0559,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -22.5,
      "rewards/margins": 10.4375,
      "rewards/rejected": -33.0,
      "step": 14860
    },
    {
      "epoch": 1.9004409227426673,
      "grad_norm": 2.2124544144021416,
      "learning_rate": 6.560463185051874e-07,
      "logits/chosen": -0.1875,
      "logits/rejected": 0.369140625,
      "logps/chosen": -478.0,
      "logps/rejected": -560.0,
      "loss": 0.0515,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -22.875,
      "rewards/margins": 10.0625,
      "rewards/rejected": -33.0,
      "step": 14870
    },
    {
      "epoch": 1.9017189596779347,
      "grad_norm": 5.774498979583734,
      "learning_rate": 6.558258357839529e-07,
      "logits/chosen": -0.353515625,
      "logits/rejected": 0.310546875,
      "logps/chosen": -482.0,
      "logps/rejected": -556.0,
      "loss": 0.0482,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.625,
      "rewards/margins": 10.0625,
      "rewards/rejected": -31.75,
      "step": 14880
    },
    {
      "epoch": 1.902996996613202,
      "grad_norm": 7.850652380289027,
      "learning_rate": 6.556055752115664e-07,
      "logits/chosen": -0.419921875,
      "logits/rejected": 0.2392578125,
      "logps/chosen": -482.0,
      "logps/rejected": -564.0,
      "loss": 0.0443,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.625,
      "rewards/margins": 10.4375,
      "rewards/rejected": -32.0,
      "step": 14890
    },
    {
      "epoch": 1.9042750335484695,
      "grad_norm": 6.994872561386165,
      "learning_rate": 6.553855364152324e-07,
      "logits/chosen": -0.4609375,
      "logits/rejected": 0.20703125,
      "logps/chosen": -480.0,
      "logps/rejected": -552.0,
      "loss": 0.0608,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.625,
      "rewards/margins": 10.4375,
      "rewards/rejected": -31.125,
      "step": 14900
    },
    {
      "epoch": 1.905553070483737,
      "grad_norm": 7.155489214128642,
      "learning_rate": 6.551657190230304e-07,
      "logits/chosen": -0.29296875,
      "logits/rejected": 0.265625,
      "logps/chosen": -462.0,
      "logps/rejected": -564.0,
      "loss": 0.0704,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.75,
      "rewards/margins": 12.625,
      "rewards/rejected": -32.5,
      "step": 14910
    },
    {
      "epoch": 1.9068311074190043,
      "grad_norm": 3.501502769355034,
      "learning_rate": 6.549461226639129e-07,
      "logits/chosen": -0.375,
      "logits/rejected": 0.1865234375,
      "logps/chosen": -516.0,
      "logps/rejected": -576.0,
      "loss": 0.0566,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -22.0,
      "rewards/margins": 11.375,
      "rewards/rejected": -33.25,
      "step": 14920
    },
    {
      "epoch": 1.9081091443542717,
      "grad_norm": 8.510751292400093,
      "learning_rate": 6.547267469677022e-07,
      "logits/chosen": -0.37109375,
      "logits/rejected": 0.328125,
      "logps/chosen": -468.0,
      "logps/rejected": -532.0,
      "loss": 0.0483,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.25,
      "rewards/margins": 9.5625,
      "rewards/rejected": -30.875,
      "step": 14930
    },
    {
      "epoch": 1.909387181289539,
      "grad_norm": 6.490563109583087,
      "learning_rate": 6.545075915650879e-07,
      "logits/chosen": -0.52734375,
      "logits/rejected": 0.01220703125,
      "logps/chosen": -470.0,
      "logps/rejected": -536.0,
      "loss": 0.0571,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.75,
      "rewards/margins": 10.0,
      "rewards/rejected": -29.75,
      "step": 14940
    },
    {
      "epoch": 1.9106652182248067,
      "grad_norm": 2.9453589782027336,
      "learning_rate": 6.542886560876246e-07,
      "logits/chosen": -0.41015625,
      "logits/rejected": 0.1572265625,
      "logps/chosen": -476.0,
      "logps/rejected": -548.0,
      "loss": 0.0422,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.75,
      "rewards/margins": 10.0625,
      "rewards/rejected": -30.875,
      "step": 14950
    },
    {
      "epoch": 1.9119432551600741,
      "grad_norm": 11.087869419986532,
      "learning_rate": 6.540699401677286e-07,
      "logits/chosen": -0.4921875,
      "logits/rejected": 0.2353515625,
      "logps/chosen": -474.0,
      "logps/rejected": -532.0,
      "loss": 0.0419,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.375,
      "rewards/margins": 10.4375,
      "rewards/rejected": -30.75,
      "step": 14960
    },
    {
      "epoch": 1.9132212920953415,
      "grad_norm": 11.25427423922307,
      "learning_rate": 6.538514434386762e-07,
      "logits/chosen": -0.4609375,
      "logits/rejected": 0.177734375,
      "logps/chosen": -464.0,
      "logps/rejected": -544.0,
      "loss": 0.0658,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.875,
      "rewards/margins": 10.3125,
      "rewards/rejected": -29.125,
      "step": 14970
    },
    {
      "epoch": 1.914499329030609,
      "grad_norm": 3.36427949166528,
      "learning_rate": 6.536331655346004e-07,
      "logits/chosen": -0.515625,
      "logits/rejected": 0.011962890625,
      "logps/chosen": -452.0,
      "logps/rejected": -532.0,
      "loss": 0.0646,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.375,
      "rewards/margins": 10.8125,
      "rewards/rejected": -29.125,
      "step": 14980
    },
    {
      "epoch": 1.9157773659658766,
      "grad_norm": 10.152600711559208,
      "learning_rate": 6.534151060904888e-07,
      "logits/chosen": -0.357421875,
      "logits/rejected": 0.035400390625,
      "logps/chosen": -464.0,
      "logps/rejected": -556.0,
      "loss": 0.0516,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.25,
      "rewards/margins": 9.6875,
      "rewards/rejected": -28.875,
      "step": 14990
    },
    {
      "epoch": 1.917055402901144,
      "grad_norm": 6.248191002258739,
      "learning_rate": 6.531972647421808e-07,
      "logits/chosen": -0.34375,
      "logits/rejected": 0.2099609375,
      "logps/chosen": -470.0,
      "logps/rejected": -524.0,
      "loss": 0.0637,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.625,
      "rewards/margins": 9.6875,
      "rewards/rejected": -28.25,
      "step": 15000
    },
    {
      "epoch": 1.917055402901144,
      "eval_logits/chosen": -0.42578125,
      "eval_logits/rejected": 0.1162109375,
      "eval_logps/chosen": -466.0,
      "eval_logps/rejected": -504.0,
      "eval_loss": 0.2718987464904785,
      "eval_rewards/accuracies": 0.898329496383667,
      "eval_rewards/chosen": -20.0,
      "eval_rewards/margins": 7.375,
      "eval_rewards/rejected": -27.375,
      "eval_runtime": 1500.386,
      "eval_samples_per_second": 36.994,
      "eval_steps_per_second": 0.579,
      "step": 15000
    },
    {
      "epoch": 1.9183334398364114,
      "grad_norm": 6.408287621991318,
      "learning_rate": 6.529796411263649e-07,
      "logits/chosen": -0.44921875,
      "logits/rejected": 0.1298828125,
      "logps/chosen": -436.0,
      "logps/rejected": -506.0,
      "loss": 0.0484,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.0,
      "rewards/margins": 9.9375,
      "rewards/rejected": -28.875,
      "step": 15010
    },
    {
      "epoch": 1.9196114767716788,
      "grad_norm": 6.068050162281977,
      "learning_rate": 6.527622348805765e-07,
      "logits/chosen": -0.388671875,
      "logits/rejected": 0.234375,
      "logps/chosen": -474.0,
      "logps/rejected": -544.0,
      "loss": 0.0398,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.5,
      "rewards/margins": 10.1875,
      "rewards/rejected": -30.75,
      "step": 15020
    },
    {
      "epoch": 1.9208895137069462,
      "grad_norm": 9.628506064552667,
      "learning_rate": 6.525450456431951e-07,
      "logits/chosen": -0.375,
      "logits/rejected": 0.27734375,
      "logps/chosen": -484.0,
      "logps/rejected": -536.0,
      "loss": 0.053,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -21.5,
      "rewards/margins": 9.8125,
      "rewards/rejected": -31.25,
      "step": 15030
    },
    {
      "epoch": 1.9221675506422136,
      "grad_norm": 1.6158054335745968,
      "learning_rate": 6.523280730534421e-07,
      "logits/chosen": -0.4140625,
      "logits/rejected": 0.09130859375,
      "logps/chosen": -484.0,
      "logps/rejected": -560.0,
      "loss": 0.0631,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -21.25,
      "rewards/margins": 10.0625,
      "rewards/rejected": -31.375,
      "step": 15040
    },
    {
      "epoch": 1.923445587577481,
      "grad_norm": 7.23198589002131,
      "learning_rate": 6.521113167513779e-07,
      "logits/chosen": -0.4140625,
      "logits/rejected": 0.333984375,
      "logps/chosen": -474.0,
      "logps/rejected": -544.0,
      "loss": 0.0492,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -21.375,
      "rewards/margins": 10.375,
      "rewards/rejected": -31.75,
      "step": 15050
    },
    {
      "epoch": 1.9247236245127484,
      "grad_norm": 3.505365077974964,
      "learning_rate": 6.518947763778994e-07,
      "logits/chosen": -0.453125,
      "logits/rejected": 0.2275390625,
      "logps/chosen": -478.0,
      "logps/rejected": -552.0,
      "loss": 0.0543,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -20.75,
      "rewards/margins": 10.0625,
      "rewards/rejected": -30.75,
      "step": 15060
    },
    {
      "epoch": 1.9260016614480158,
      "grad_norm": 5.379953928392647,
      "learning_rate": 6.516784515747379e-07,
      "logits/chosen": -0.43359375,
      "logits/rejected": 0.06884765625,
      "logps/chosen": -480.0,
      "logps/rejected": -544.0,
      "loss": 0.0574,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -20.125,
      "rewards/margins": 10.625,
      "rewards/rejected": -30.75,
      "step": 15070
    },
    {
      "epoch": 1.9272796983832832,
      "grad_norm": 3.0749114525232457,
      "learning_rate": 6.514623419844562e-07,
      "logits/chosen": -0.373046875,
      "logits/rejected": 0.2119140625,
      "logps/chosen": -492.0,
      "logps/rejected": -556.0,
      "loss": 0.0547,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -21.25,
      "rewards/margins": 10.0,
      "rewards/rejected": -31.25,
      "step": 15080
    },
    {
      "epoch": 1.9285577353185506,
      "grad_norm": 5.157164384803509,
      "learning_rate": 6.512464472504465e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": 0.23046875,
      "logps/chosen": -484.0,
      "logps/rejected": -532.0,
      "loss": 0.051,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.75,
      "rewards/margins": 9.5625,
      "rewards/rejected": -30.25,
      "step": 15090
    },
    {
      "epoch": 1.929835772253818,
      "grad_norm": 3.3116809287756155,
      "learning_rate": 6.510307670169275e-07,
      "logits/chosen": -0.419921875,
      "logits/rejected": 0.1953125,
      "logps/chosen": -480.0,
      "logps/rejected": -536.0,
      "loss": 0.04,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -20.5,
      "rewards/margins": 10.0,
      "rewards/rejected": -30.5,
      "step": 15100
    },
    {
      "epoch": 1.9311138091890856,
      "grad_norm": 6.012155396659911,
      "learning_rate": 6.508153009289422e-07,
      "logits/chosen": -0.376953125,
      "logits/rejected": 0.208984375,
      "logps/chosen": -504.0,
      "logps/rejected": -548.0,
      "loss": 0.057,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.375,
      "rewards/margins": 10.0,
      "rewards/rejected": -31.375,
      "step": 15110
    },
    {
      "epoch": 1.932391846124353,
      "grad_norm": 6.389379410471804,
      "learning_rate": 6.506000486323554e-07,
      "logits/chosen": -0.416015625,
      "logits/rejected": 0.1650390625,
      "logps/chosen": -476.0,
      "logps/rejected": -544.0,
      "loss": 0.0473,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.75,
      "rewards/margins": 9.75,
      "rewards/rejected": -30.5,
      "step": 15120
    },
    {
      "epoch": 1.9336698830596204,
      "grad_norm": 6.153903951156255,
      "learning_rate": 6.503850097738512e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": 0.08642578125,
      "logps/chosen": -468.0,
      "logps/rejected": -532.0,
      "loss": 0.0661,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.5,
      "rewards/margins": 10.0625,
      "rewards/rejected": -28.625,
      "step": 15130
    },
    {
      "epoch": 1.9349479199948878,
      "grad_norm": 4.887871163629688,
      "learning_rate": 6.50170184000931e-07,
      "logits/chosen": -0.453125,
      "logits/rejected": 0.2021484375,
      "logps/chosen": -468.0,
      "logps/rejected": -536.0,
      "loss": 0.0525,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.25,
      "rewards/margins": 9.9375,
      "rewards/rejected": -30.25,
      "step": 15140
    },
    {
      "epoch": 1.9362259569301554,
      "grad_norm": 6.194003688135744,
      "learning_rate": 6.4995557096191e-07,
      "logits/chosen": -0.44140625,
      "logits/rejected": 0.1923828125,
      "logps/chosen": -476.0,
      "logps/rejected": -544.0,
      "loss": 0.0693,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -20.5,
      "rewards/margins": 10.4375,
      "rewards/rejected": -31.0,
      "step": 15150
    },
    {
      "epoch": 1.9375039938654228,
      "grad_norm": 1.4242517113233717,
      "learning_rate": 6.49741170305916e-07,
      "logits/chosen": -0.515625,
      "logits/rejected": 0.09912109375,
      "logps/chosen": -486.0,
      "logps/rejected": -556.0,
      "loss": 0.0475,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -21.0,
      "rewards/margins": 10.3125,
      "rewards/rejected": -31.25,
      "step": 15160
    },
    {
      "epoch": 1.9387820308006902,
      "grad_norm": 5.599475771501328,
      "learning_rate": 6.495269816828862e-07,
      "logits/chosen": -0.5390625,
      "logits/rejected": 0.1220703125,
      "logps/chosen": -516.0,
      "logps/rejected": -552.0,
      "loss": 0.0664,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -21.625,
      "rewards/margins": 10.1875,
      "rewards/rejected": -31.75,
      "step": 15170
    },
    {
      "epoch": 1.9400600677359576,
      "grad_norm": 8.63213332731896,
      "learning_rate": 6.493130047435654e-07,
      "logits/chosen": -0.359375,
      "logits/rejected": 0.181640625,
      "logps/chosen": -474.0,
      "logps/rejected": -548.0,
      "loss": 0.0454,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -20.25,
      "rewards/margins": 10.875,
      "rewards/rejected": -31.125,
      "step": 15180
    },
    {
      "epoch": 1.941338104671225,
      "grad_norm": 5.662290399974009,
      "learning_rate": 6.490992391395026e-07,
      "logits/chosen": -0.4453125,
      "logits/rejected": 0.1875,
      "logps/chosen": -484.0,
      "logps/rejected": -532.0,
      "loss": 0.035,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -20.125,
      "rewards/margins": 10.25,
      "rewards/rejected": -30.375,
      "step": 15190
    },
    {
      "epoch": 1.9426161416064924,
      "grad_norm": 7.225481356281021,
      "learning_rate": 6.488856845230501e-07,
      "logits/chosen": -0.43359375,
      "logits/rejected": 0.1455078125,
      "logps/chosen": -468.0,
      "logps/rejected": -540.0,
      "loss": 0.0576,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -20.0,
      "rewards/margins": 10.75,
      "rewards/rejected": -30.75,
      "step": 15200
    },
    {
      "epoch": 1.9438941785417598,
      "grad_norm": 4.8192286421975155,
      "learning_rate": 6.486723405473599e-07,
      "logits/chosen": -0.359375,
      "logits/rejected": 0.22265625,
      "logps/chosen": -488.0,
      "logps/rejected": -548.0,
      "loss": 0.0434,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -21.75,
      "rewards/margins": 9.875,
      "rewards/rejected": -31.625,
      "step": 15210
    },
    {
      "epoch": 1.9451722154770272,
      "grad_norm": 8.32253447357115,
      "learning_rate": 6.484592068663816e-07,
      "logits/chosen": -0.431640625,
      "logits/rejected": 0.07177734375,
      "logps/chosen": -500.0,
      "logps/rejected": -572.0,
      "loss": 0.0626,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -22.0,
      "rewards/margins": 9.9375,
      "rewards/rejected": -31.875,
      "step": 15220
    },
    {
      "epoch": 1.9464502524122946,
      "grad_norm": 4.51880101050914,
      "learning_rate": 6.482462831348603e-07,
      "logits/chosen": -0.275390625,
      "logits/rejected": 0.2060546875,
      "logps/chosen": -454.0,
      "logps/rejected": -564.0,
      "loss": 0.0491,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.5,
      "rewards/margins": 10.5625,
      "rewards/rejected": -31.125,
      "step": 15230
    },
    {
      "epoch": 1.947728289347562,
      "grad_norm": 6.209318411346991,
      "learning_rate": 6.480335690083345e-07,
      "logits/chosen": -0.337890625,
      "logits/rejected": 0.169921875,
      "logps/chosen": -470.0,
      "logps/rejected": -544.0,
      "loss": 0.0478,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -20.25,
      "rewards/margins": 9.3125,
      "rewards/rejected": -29.5,
      "step": 15240
    },
    {
      "epoch": 1.9490063262828294,
      "grad_norm": 7.72250065338728,
      "learning_rate": 6.478210641431327e-07,
      "logits/chosen": -0.419921875,
      "logits/rejected": 0.1640625,
      "logps/chosen": -502.0,
      "logps/rejected": -564.0,
      "loss": 0.0487,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.75,
      "rewards/margins": 10.125,
      "rewards/rejected": -30.875,
      "step": 15250
    },
    {
      "epoch": 1.9502843632180968,
      "grad_norm": 10.444871220323723,
      "learning_rate": 6.476087681963725e-07,
      "logits/chosen": -0.53515625,
      "logits/rejected": 0.1865234375,
      "logps/chosen": -492.0,
      "logps/rejected": -560.0,
      "loss": 0.0471,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -20.875,
      "rewards/margins": 11.1875,
      "rewards/rejected": -32.0,
      "step": 15260
    },
    {
      "epoch": 1.9515624001533645,
      "grad_norm": 3.754805618902749,
      "learning_rate": 6.473966808259572e-07,
      "logits/chosen": -0.40234375,
      "logits/rejected": 0.2314453125,
      "logps/chosen": -464.0,
      "logps/rejected": -548.0,
      "loss": 0.0821,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -21.25,
      "rewards/margins": 10.625,
      "rewards/rejected": -31.875,
      "step": 15270
    },
    {
      "epoch": 1.9528404370886319,
      "grad_norm": 5.176488760383505,
      "learning_rate": 6.471848016905735e-07,
      "logits/chosen": -0.3046875,
      "logits/rejected": 0.251953125,
      "logps/chosen": -464.0,
      "logps/rejected": -536.0,
      "loss": 0.064,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -20.875,
      "rewards/margins": 9.8125,
      "rewards/rejected": -30.75,
      "step": 15280
    },
    {
      "epoch": 1.9541184740238993,
      "grad_norm": 6.429934733155299,
      "learning_rate": 6.469731304496901e-07,
      "logits/chosen": -0.37890625,
      "logits/rejected": 0.08740234375,
      "logps/chosen": -466.0,
      "logps/rejected": -544.0,
      "loss": 0.0771,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.75,
      "rewards/margins": 11.625,
      "rewards/rejected": -31.375,
      "step": 15290
    },
    {
      "epoch": 1.9553965109591667,
      "grad_norm": 5.471190605697601,
      "learning_rate": 6.467616667635546e-07,
      "logits/chosen": -0.369140625,
      "logits/rejected": 0.296875,
      "logps/chosen": -466.0,
      "logps/rejected": -552.0,
      "loss": 0.0545,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -20.875,
      "rewards/margins": 10.875,
      "rewards/rejected": -31.625,
      "step": 15300
    },
    {
      "epoch": 1.9566745478944343,
      "grad_norm": 4.0099746770104,
      "learning_rate": 6.465504102931912e-07,
      "logits/chosen": -0.2734375,
      "logits/rejected": 0.439453125,
      "logps/chosen": -460.0,
      "logps/rejected": -510.0,
      "loss": 0.0509,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.125,
      "rewards/margins": 10.0625,
      "rewards/rejected": -29.125,
      "step": 15310
    },
    {
      "epoch": 1.9579525848297017,
      "grad_norm": 12.259154154949098,
      "learning_rate": 6.463393607003991e-07,
      "logits/chosen": -0.400390625,
      "logits/rejected": 0.216796875,
      "logps/chosen": -460.0,
      "logps/rejected": -516.0,
      "loss": 0.0686,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.625,
      "rewards/margins": 9.625,
      "rewards/rejected": -29.25,
      "step": 15320
    },
    {
      "epoch": 1.959230621764969,
      "grad_norm": 3.009243749382414,
      "learning_rate": 6.461285176477491e-07,
      "logits/chosen": -0.49609375,
      "logits/rejected": 0.1201171875,
      "logps/chosen": -476.0,
      "logps/rejected": -552.0,
      "loss": 0.0556,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.25,
      "rewards/margins": 10.0625,
      "rewards/rejected": -30.25,
      "step": 15330
    },
    {
      "epoch": 1.9605086587002365,
      "grad_norm": 9.169548766650433,
      "learning_rate": 6.45917880798583e-07,
      "logits/chosen": -0.3828125,
      "logits/rejected": 0.1787109375,
      "logps/chosen": -486.0,
      "logps/rejected": -560.0,
      "loss": 0.0516,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.875,
      "rewards/margins": 10.125,
      "rewards/rejected": -31.0,
      "step": 15340
    },
    {
      "epoch": 1.961786695635504,
      "grad_norm": 3.5424080154653983,
      "learning_rate": 6.457074498170094e-07,
      "logits/chosen": -0.328125,
      "logits/rejected": 0.275390625,
      "logps/chosen": -504.0,
      "logps/rejected": -548.0,
      "loss": 0.0473,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -22.125,
      "rewards/margins": 9.9375,
      "rewards/rejected": -32.0,
      "step": 15350
    },
    {
      "epoch": 1.9630647325707713,
      "grad_norm": 7.554801653930557,
      "learning_rate": 6.454972243679028e-07,
      "logits/chosen": -0.451171875,
      "logits/rejected": 0.33984375,
      "logps/chosen": -476.0,
      "logps/rejected": -524.0,
      "loss": 0.0626,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.0,
      "rewards/margins": 10.125,
      "rewards/rejected": -31.125,
      "step": 15360
    },
    {
      "epoch": 1.9643427695060387,
      "grad_norm": 10.896468171111087,
      "learning_rate": 6.452872041169011e-07,
      "logits/chosen": -0.419921875,
      "logits/rejected": 0.19140625,
      "logps/chosen": -496.0,
      "logps/rejected": -584.0,
      "loss": 0.0795,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -22.375,
      "rewards/margins": 10.3125,
      "rewards/rejected": -32.75,
      "step": 15370
    },
    {
      "epoch": 1.965620806441306,
      "grad_norm": 6.3604747436218085,
      "learning_rate": 6.450773887304029e-07,
      "logits/chosen": -0.4375,
      "logits/rejected": 0.169921875,
      "logps/chosen": -472.0,
      "logps/rejected": -552.0,
      "loss": 0.0756,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.375,
      "rewards/margins": 10.5625,
      "rewards/rejected": -31.0,
      "step": 15380
    },
    {
      "epoch": 1.9668988433765735,
      "grad_norm": 11.813137883604053,
      "learning_rate": 6.448677778755658e-07,
      "logits/chosen": -0.376953125,
      "logits/rejected": 0.2001953125,
      "logps/chosen": -490.0,
      "logps/rejected": -544.0,
      "loss": 0.0838,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -21.5,
      "rewards/margins": 9.75,
      "rewards/rejected": -31.25,
      "step": 15390
    },
    {
      "epoch": 1.968176880311841,
      "grad_norm": 2.1323334426785396,
      "learning_rate": 6.446583712203042e-07,
      "logits/chosen": -0.3515625,
      "logits/rejected": 0.25,
      "logps/chosen": -472.0,
      "logps/rejected": -516.0,
      "loss": 0.0439,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.625,
      "rewards/margins": 10.0625,
      "rewards/rejected": -29.625,
      "step": 15400
    },
    {
      "epoch": 1.9694549172471083,
      "grad_norm": 7.217609076497343,
      "learning_rate": 6.444491684332863e-07,
      "logits/chosen": -0.341796875,
      "logits/rejected": 0.2734375,
      "logps/chosen": -494.0,
      "logps/rejected": -548.0,
      "loss": 0.066,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -21.25,
      "rewards/margins": 10.125,
      "rewards/rejected": -31.375,
      "step": 15410
    },
    {
      "epoch": 1.9707329541823757,
      "grad_norm": 7.252799840108293,
      "learning_rate": 6.442401691839331e-07,
      "logits/chosen": -0.263671875,
      "logits/rejected": 0.37109375,
      "logps/chosen": -468.0,
      "logps/rejected": -536.0,
      "loss": 0.0614,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -21.125,
      "rewards/margins": 10.1875,
      "rewards/rejected": -31.375,
      "step": 15420
    },
    {
      "epoch": 1.9720109911176433,
      "grad_norm": 9.065211187110087,
      "learning_rate": 6.440313731424148e-07,
      "logits/chosen": -0.259765625,
      "logits/rejected": 0.2734375,
      "logps/chosen": -472.0,
      "logps/rejected": -540.0,
      "loss": 0.0547,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -20.25,
      "rewards/margins": 10.0625,
      "rewards/rejected": -30.375,
      "step": 15430
    },
    {
      "epoch": 1.9732890280529107,
      "grad_norm": 3.4853267946430355,
      "learning_rate": 6.438227799796504e-07,
      "logits/chosen": -0.279296875,
      "logits/rejected": 0.30078125,
      "logps/chosen": -476.0,
      "logps/rejected": -552.0,
      "loss": 0.0535,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -22.375,
      "rewards/margins": 10.0625,
      "rewards/rejected": -32.5,
      "step": 15440
    },
    {
      "epoch": 1.9745670649881781,
      "grad_norm": 3.3879616659036595,
      "learning_rate": 6.436143893673037e-07,
      "logits/chosen": -0.32421875,
      "logits/rejected": 0.2373046875,
      "logps/chosen": -478.0,
      "logps/rejected": -548.0,
      "loss": 0.053,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -20.875,
      "rewards/margins": 10.125,
      "rewards/rejected": -31.0,
      "step": 15450
    },
    {
      "epoch": 1.9758451019234455,
      "grad_norm": 3.531417517444132,
      "learning_rate": 6.434062009777823e-07,
      "logits/chosen": -0.37109375,
      "logits/rejected": 0.28125,
      "logps/chosen": -496.0,
      "logps/rejected": -576.0,
      "loss": 0.0467,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -21.625,
      "rewards/margins": 10.875,
      "rewards/rejected": -32.5,
      "step": 15460
    },
    {
      "epoch": 1.9771231388587132,
      "grad_norm": 3.5751802841817106,
      "learning_rate": 6.431982144842349e-07,
      "logits/chosen": -0.390625,
      "logits/rejected": 0.228515625,
      "logps/chosen": -488.0,
      "logps/rejected": -544.0,
      "loss": 0.0583,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.125,
      "rewards/margins": 10.25,
      "rewards/rejected": -31.375,
      "step": 15470
    },
    {
      "epoch": 1.9784011757939806,
      "grad_norm": 2.459531098622172,
      "learning_rate": 6.429904295605495e-07,
      "logits/chosen": -0.349609375,
      "logits/rejected": 0.2421875,
      "logps/chosen": -490.0,
      "logps/rejected": -540.0,
      "loss": 0.0709,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.875,
      "rewards/margins": 10.3125,
      "rewards/rejected": -31.125,
      "step": 15480
    },
    {
      "epoch": 1.979679212729248,
      "grad_norm": 1.9183079596621238,
      "learning_rate": 6.42782845881351e-07,
      "logits/chosen": -0.32421875,
      "logits/rejected": 0.27734375,
      "logps/chosen": -474.0,
      "logps/rejected": -552.0,
      "loss": 0.0519,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -21.125,
      "rewards/margins": 10.6875,
      "rewards/rejected": -31.75,
      "step": 15490
    },
    {
      "epoch": 1.9809572496645154,
      "grad_norm": 3.449522617409642,
      "learning_rate": 6.42575463121999e-07,
      "logits/chosen": -0.26953125,
      "logits/rejected": 0.283203125,
      "logps/chosen": -508.0,
      "logps/rejected": -564.0,
      "loss": 0.0627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -22.375,
      "rewards/margins": 10.625,
      "rewards/rejected": -33.0,
      "step": 15500
    },
    {
      "epoch": 1.9822352865997828,
      "grad_norm": 6.765638437166043,
      "learning_rate": 6.423682809585863e-07,
      "logits/chosen": -0.37109375,
      "logits/rejected": 0.271484375,
      "logps/chosen": -504.0,
      "logps/rejected": -572.0,
      "loss": 0.0651,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -21.875,
      "rewards/margins": 10.875,
      "rewards/rejected": -32.75,
      "step": 15510
    },
    {
      "epoch": 1.9835133235350502,
      "grad_norm": 5.1633799324969,
      "learning_rate": 6.421612990679356e-07,
      "logits/chosen": -0.447265625,
      "logits/rejected": 0.201171875,
      "logps/chosen": -476.0,
      "logps/rejected": -552.0,
      "loss": 0.0811,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -21.0,
      "rewards/margins": 10.0625,
      "rewards/rejected": -31.0,
      "step": 15520
    },
    {
      "epoch": 1.9847913604703176,
      "grad_norm": 4.134620353097269,
      "learning_rate": 6.419545171275983e-07,
      "logits/chosen": -0.177734375,
      "logits/rejected": 0.369140625,
      "logps/chosen": -468.0,
      "logps/rejected": -540.0,
      "loss": 0.0721,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.125,
      "rewards/margins": 10.0625,
      "rewards/rejected": -30.125,
      "step": 15530
    },
    {
      "epoch": 1.986069397405585,
      "grad_norm": 6.450942820258016,
      "learning_rate": 6.417479348158526e-07,
      "logits/chosen": -0.41015625,
      "logits/rejected": 0.064453125,
      "logps/chosen": -504.0,
      "logps/rejected": -556.0,
      "loss": 0.0657,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -20.5,
      "rewards/margins": 9.5,
      "rewards/rejected": -30.0,
      "step": 15540
    },
    {
      "epoch": 1.9873474343408524,
      "grad_norm": 5.693753986558539,
      "learning_rate": 6.415415518117003e-07,
      "logits/chosen": -0.291015625,
      "logits/rejected": 0.2333984375,
      "logps/chosen": -490.0,
      "logps/rejected": -528.0,
      "loss": 0.0666,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -20.0,
      "rewards/margins": 10.0,
      "rewards/rejected": -30.0,
      "step": 15550
    },
    {
      "epoch": 1.9886254712761198,
      "grad_norm": 9.421301144011721,
      "learning_rate": 6.413353677948659e-07,
      "logits/chosen": -0.51171875,
      "logits/rejected": 0.234375,
      "logps/chosen": -472.0,
      "logps/rejected": -576.0,
      "loss": 0.0565,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.75,
      "rewards/margins": 10.9375,
      "rewards/rejected": -30.75,
      "step": 15560
    },
    {
      "epoch": 1.9899035082113872,
      "grad_norm": 5.0622826774473,
      "learning_rate": 6.411293824457933e-07,
      "logits/chosen": -0.498046875,
      "logits/rejected": 0.2392578125,
      "logps/chosen": -468.0,
      "logps/rejected": -512.0,
      "loss": 0.0523,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.25,
      "rewards/margins": 9.8125,
      "rewards/rejected": -29.0,
      "step": 15570
    },
    {
      "epoch": 1.9911815451466546,
      "grad_norm": 5.8504652820816645,
      "learning_rate": 6.409235954456451e-07,
      "logits/chosen": -0.396484375,
      "logits/rejected": 0.166015625,
      "logps/chosen": -470.0,
      "logps/rejected": -520.0,
      "loss": 0.0689,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.0,
      "rewards/margins": 9.9375,
      "rewards/rejected": -30.0,
      "step": 15580
    },
    {
      "epoch": 1.9924595820819222,
      "grad_norm": 9.617499149226367,
      "learning_rate": 6.407180064762995e-07,
      "logits/chosen": -0.3984375,
      "logits/rejected": 0.1767578125,
      "logps/chosen": -456.0,
      "logps/rejected": -536.0,
      "loss": 0.0589,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.5,
      "rewards/margins": 10.0625,
      "rewards/rejected": -30.5,
      "step": 15590
    },
    {
      "epoch": 1.9937376190171896,
      "grad_norm": 6.201215947565185,
      "learning_rate": 6.405126152203485e-07,
      "logits/chosen": -0.203125,
      "logits/rejected": 0.373046875,
      "logps/chosen": -478.0,
      "logps/rejected": -556.0,
      "loss": 0.054,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -22.625,
      "rewards/margins": 10.25,
      "rewards/rejected": -32.75,
      "step": 15600
    },
    {
      "epoch": 1.995015655952457,
      "grad_norm": 3.7426765330676477,
      "learning_rate": 6.403074213610959e-07,
      "logits/chosen": -0.328125,
      "logits/rejected": 0.330078125,
      "logps/chosen": -520.0,
      "logps/rejected": -580.0,
      "loss": 0.0609,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -23.625,
      "rewards/margins": 10.5625,
      "rewards/rejected": -34.25,
      "step": 15610
    },
    {
      "epoch": 1.9962936928877244,
      "grad_norm": 3.2572037541927967,
      "learning_rate": 6.401024245825555e-07,
      "logits/chosen": -0.34375,
      "logits/rejected": 0.2578125,
      "logps/chosen": -516.0,
      "logps/rejected": -584.0,
      "loss": 0.0446,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -24.125,
      "rewards/margins": 11.125,
      "rewards/rejected": -35.25,
      "step": 15620
    },
    {
      "epoch": 1.997571729822992,
      "grad_norm": 6.89123136077629,
      "learning_rate": 6.398976245694481e-07,
      "logits/chosen": -0.404296875,
      "logits/rejected": 0.32421875,
      "logps/chosen": -496.0,
      "logps/rejected": -560.0,
      "loss": 0.0478,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -23.125,
      "rewards/margins": 10.5625,
      "rewards/rejected": -33.75,
      "step": 15630
    },
    {
      "epoch": 1.9988497667582594,
      "grad_norm": 7.647043695596885,
      "learning_rate": 6.396930210072012e-07,
      "logits/chosen": -0.419921875,
      "logits/rejected": 0.11376953125,
      "logps/chosen": -496.0,
      "logps/rejected": -580.0,
      "loss": 0.0616,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -22.25,
      "rewards/margins": 11.1875,
      "rewards/rejected": -33.5,
      "step": 15640
    },
    {
      "epoch": 1.9998721963064732,
      "step": 15648,
      "total_flos": 0.0,
      "train_loss": 0.0,
      "train_runtime": 23.3629,
      "train_samples_per_second": 42867.703,
      "train_steps_per_second": 334.889
    }
  ],
  "logging_steps": 10,
  "max_steps": 7824,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 5000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}