{
  "best_metric": 0.22941964864730835,
  "best_model_checkpoint": "models/qwen2.5-3b-orpo-mini-fp/checkpoint-5000",
  "epoch": 0.9999360981532366,
  "eval_steps": 5000,
  "global_step": 7824,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.00012780369352674293,
      "grad_norm": 17.79294632846499,
      "learning_rate": 8e-08,
      "logits/chosen": -1.6875,
      "logits/rejected": -1.484375,
      "logps/chosen": -186.0,
      "logps/rejected": -200.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0012780369352674292,
      "grad_norm": 19.157016992578,
      "learning_rate": 8e-07,
      "logits/chosen": -1.9453125,
      "logits/rejected": -1.4765625,
      "logps/chosen": -212.0,
      "logps/rejected": -182.0,
      "loss": 0.6925,
      "rewards/accuracies": 0.2083333283662796,
      "rewards/chosen": 0.004180908203125,
      "rewards/margins": 0.001068115234375,
      "rewards/rejected": 0.0031280517578125,
      "step": 10
    },
    {
      "epoch": 0.0025560738705348585,
      "grad_norm": 15.16100498420179,
      "learning_rate": 1.6e-06,
      "logits/chosen": -2.015625,
      "logits/rejected": -1.6640625,
      "logps/chosen": -224.0,
      "logps/rejected": -186.0,
      "loss": 0.6179,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": 0.0400390625,
      "rewards/margins": 0.1953125,
      "rewards/rejected": -0.1552734375,
      "step": 20
    },
    {
      "epoch": 0.0038341108058022877,
      "grad_norm": 14.514374788683362,
      "learning_rate": 2.4e-06,
      "logits/chosen": -2.125,
      "logits/rejected": -1.8046875,
      "logps/chosen": -232.0,
      "logps/rejected": -206.0,
      "loss": 0.5059,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.42578125,
      "rewards/margins": 0.8671875,
      "rewards/rejected": -1.296875,
      "step": 30
    },
    {
      "epoch": 0.005112147741069717,
      "grad_norm": 15.141785262237963,
      "learning_rate": 3.2e-06,
      "logits/chosen": -2.0,
      "logits/rejected": -1.8046875,
      "logps/chosen": -210.0,
      "logps/rejected": -198.0,
      "loss": 0.4729,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.37109375,
      "rewards/margins": 1.375,
      "rewards/rejected": -1.75,
      "step": 40
    },
    {
      "epoch": 0.006390184676337146,
      "grad_norm": 13.196147001717241,
      "learning_rate": 4e-06,
      "logits/chosen": -1.9609375,
      "logits/rejected": -1.640625,
      "logps/chosen": -229.0,
      "logps/rejected": -209.0,
      "loss": 0.4255,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.765625,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -2.53125,
      "step": 50
    },
    {
      "epoch": 0.007668221611604575,
      "grad_norm": 10.824664501964739,
      "learning_rate": 4.8e-06,
      "logits/chosen": -1.8515625,
      "logits/rejected": -1.53125,
      "logps/chosen": -234.0,
      "logps/rejected": -216.0,
      "loss": 0.419,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.609375,
      "rewards/margins": 1.6328125,
      "rewards/rejected": -4.25,
      "step": 60
    },
    {
      "epoch": 0.008946258546872005,
      "grad_norm": 13.387280930116003,
      "learning_rate": 5.6e-06,
      "logits/chosen": -1.890625,
      "logits/rejected": -1.59375,
      "logps/chosen": -242.0,
      "logps/rejected": -230.0,
      "loss": 0.3932,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.421875,
      "rewards/margins": 1.75,
      "rewards/rejected": -4.15625,
      "step": 70
    },
    {
      "epoch": 0.010224295482139434,
      "grad_norm": 12.469898753244802,
      "learning_rate": 6.4e-06,
      "logits/chosen": -1.9140625,
      "logits/rejected": -1.5625,
      "logps/chosen": -245.0,
      "logps/rejected": -229.0,
      "loss": 0.3623,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.484375,
      "rewards/margins": 1.8359375,
      "rewards/rejected": -4.34375,
      "step": 80
    },
    {
      "epoch": 0.011502332417406863,
      "grad_norm": 12.861662288653752,
      "learning_rate": 7.2e-06,
      "logits/chosen": -2.0,
      "logits/rejected": -1.6484375,
      "logps/chosen": -239.0,
      "logps/rejected": -244.0,
      "loss": 0.3818,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 2.171875,
      "rewards/rejected": -5.6875,
      "step": 90
    },
    {
      "epoch": 0.012780369352674292,
      "grad_norm": 11.065773265550133,
      "learning_rate": 8e-06,
      "logits/chosen": -1.796875,
      "logits/rejected": -1.359375,
      "logps/chosen": -253.0,
      "logps/rejected": -249.0,
      "loss": 0.3528,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.921875,
      "rewards/margins": 2.390625,
      "rewards/rejected": -6.3125,
      "step": 100
    },
    {
      "epoch": 0.014058406287941722,
      "grad_norm": 11.175931675292244,
      "learning_rate": 7.627700713964738e-06,
      "logits/chosen": -2.078125,
      "logits/rejected": -1.65625,
      "logps/chosen": -268.0,
      "logps/rejected": -272.0,
      "loss": 0.3401,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.25,
      "rewards/rejected": -8.9375,
      "step": 110
    },
    {
      "epoch": 0.01533644322320915,
      "grad_norm": 11.576569565074632,
      "learning_rate": 7.3029674334022146e-06,
      "logits/chosen": -1.859375,
      "logits/rejected": -1.4375,
      "logps/chosen": -264.0,
      "logps/rejected": -274.0,
      "loss": 0.3178,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.78125,
      "rewards/margins": 3.125,
      "rewards/rejected": -8.875,
      "step": 120
    },
    {
      "epoch": 0.01661448015847658,
      "grad_norm": 11.049695523890255,
      "learning_rate": 7.016464154456233e-06,
      "logits/chosen": -1.703125,
      "logits/rejected": -1.3046875,
      "logps/chosen": -244.0,
      "logps/rejected": -280.0,
      "loss": 0.3579,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -4.65625,
      "rewards/margins": 2.625,
      "rewards/rejected": -7.28125,
      "step": 130
    },
    {
      "epoch": 0.01789251709374401,
      "grad_norm": 22.185328155457004,
      "learning_rate": 6.7612340378281325e-06,
      "logits/chosen": -1.8203125,
      "logits/rejected": -1.3515625,
      "logps/chosen": -262.0,
      "logps/rejected": -260.0,
      "loss": 0.3328,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -4.375,
      "rewards/margins": 2.53125,
      "rewards/rejected": -6.90625,
      "step": 140
    },
    {
      "epoch": 0.01917055402901144,
      "grad_norm": 17.28703916612154,
      "learning_rate": 6.531972647421809e-06,
      "logits/chosen": -1.8515625,
      "logits/rejected": -1.578125,
      "logps/chosen": -268.0,
      "logps/rejected": -276.0,
      "loss": 0.3136,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -5.625,
      "rewards/margins": 3.640625,
      "rewards/rejected": -9.3125,
      "step": 150
    },
    {
      "epoch": 0.020448590964278868,
      "grad_norm": 8.964686735274466,
      "learning_rate": 6.3245553203367575e-06,
      "logits/chosen": -1.6328125,
      "logits/rejected": -1.25,
      "logps/chosen": -280.0,
      "logps/rejected": -286.0,
      "loss": 0.3425,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.28125,
      "rewards/rejected": -8.625,
      "step": 160
    },
    {
      "epoch": 0.021726627899546297,
      "grad_norm": 9.864423945583349,
      "learning_rate": 6.135719910778963e-06,
      "logits/chosen": -1.796875,
      "logits/rejected": -1.296875,
      "logps/chosen": -278.0,
      "logps/rejected": -298.0,
      "loss": 0.2862,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -6.21875,
      "rewards/margins": 3.734375,
      "rewards/rejected": -9.9375,
      "step": 170
    },
    {
      "epoch": 0.023004664834813726,
      "grad_norm": 70.27629195144868,
      "learning_rate": 5.962847939999439e-06,
      "logits/chosen": -1.734375,
      "logits/rejected": -1.296875,
      "logps/chosen": -278.0,
      "logps/rejected": -290.0,
      "loss": 0.3416,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -6.375,
      "rewards/margins": 3.546875,
      "rewards/rejected": -9.875,
      "step": 180
    },
    {
      "epoch": 0.024282701770081155,
      "grad_norm": 13.874329510771819,
      "learning_rate": 5.803810000880094e-06,
      "logits/chosen": -1.6875,
      "logits/rejected": -1.4453125,
      "logps/chosen": -282.0,
      "logps/rejected": -282.0,
      "loss": 0.2947,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -5.65625,
      "rewards/margins": 3.015625,
      "rewards/rejected": -8.6875,
      "step": 190
    },
    {
      "epoch": 0.025560738705348585,
      "grad_norm": 8.125044719458415,
      "learning_rate": 5.65685424949238e-06,
      "logits/chosen": -1.6640625,
      "logits/rejected": -1.3359375,
      "logps/chosen": -256.0,
      "logps/rejected": -270.0,
      "loss": 0.2684,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -7.0625,
      "rewards/margins": 3.40625,
      "rewards/rejected": -10.4375,
      "step": 200
    },
    {
      "epoch": 0.026838775640616014,
      "grad_norm": 12.378410623228271,
      "learning_rate": 5.5205244747388325e-06,
      "logits/chosen": -1.8046875,
      "logits/rejected": -1.390625,
      "logps/chosen": -292.0,
      "logps/rejected": -308.0,
      "loss": 0.3271,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -8.1875,
      "rewards/margins": 3.96875,
      "rewards/rejected": -12.125,
      "step": 210
    },
    {
      "epoch": 0.028116812575883443,
      "grad_norm": 11.605914067555702,
      "learning_rate": 5.393598899705936e-06,
      "logits/chosen": -1.609375,
      "logits/rejected": -1.234375,
      "logps/chosen": -286.0,
      "logps/rejected": -304.0,
      "loss": 0.3309,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -6.21875,
      "rewards/margins": 3.1875,
      "rewards/rejected": -9.375,
      "step": 220
    },
    {
      "epoch": 0.029394849511150872,
      "grad_norm": 10.98243489780871,
      "learning_rate": 5.275043787166296e-06,
      "logits/chosen": -1.6484375,
      "logits/rejected": -1.234375,
      "logps/chosen": -276.0,
      "logps/rejected": -288.0,
      "loss": 0.282,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.3125,
      "rewards/margins": 3.3125,
      "rewards/rejected": -9.625,
      "step": 230
    },
    {
      "epoch": 0.0306728864464183,
      "grad_norm": 12.146537231707752,
      "learning_rate": 5.163977794943223e-06,
      "logits/chosen": -1.7109375,
      "logits/rejected": -1.3515625,
      "logps/chosen": -306.0,
      "logps/rejected": -310.0,
      "loss": 0.357,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -7.78125,
      "rewards/margins": 4.625,
      "rewards/rejected": -12.4375,
      "step": 240
    },
    {
      "epoch": 0.03195092338168573,
      "grad_norm": 10.52586401600269,
      "learning_rate": 5.059644256269407e-06,
      "logits/chosen": -1.7578125,
      "logits/rejected": -1.3359375,
      "logps/chosen": -280.0,
      "logps/rejected": -296.0,
      "loss": 0.326,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.84375,
      "rewards/margins": 3.578125,
      "rewards/rejected": -10.4375,
      "step": 250
    },
    {
      "epoch": 0.03322896031695316,
      "grad_norm": 9.97681998813517,
      "learning_rate": 4.961389383568338e-06,
      "logits/chosen": -1.6484375,
      "logits/rejected": -1.1796875,
      "logps/chosen": -276.0,
      "logps/rejected": -296.0,
      "loss": 0.2686,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -6.0625,
      "rewards/margins": 3.5,
      "rewards/rejected": -9.5625,
      "step": 260
    },
    {
      "epoch": 0.03450699725222059,
      "grad_norm": 11.613233025975493,
      "learning_rate": 4.8686449556014755e-06,
      "logits/chosen": -1.625,
      "logits/rejected": -1.109375,
      "logps/chosen": -290.0,
      "logps/rejected": -300.0,
      "loss": 0.2771,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -7.0625,
      "rewards/margins": 4.53125,
      "rewards/rejected": -11.625,
      "step": 270
    },
    {
      "epoch": 0.03578503418748802,
      "grad_norm": 9.745923970722576,
      "learning_rate": 4.780914437337574e-06,
      "logits/chosen": -1.4453125,
      "logits/rejected": -1.015625,
      "logps/chosen": -280.0,
      "logps/rejected": -306.0,
      "loss": 0.2908,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -7.28125,
      "rewards/margins": 4.15625,
      "rewards/rejected": -11.4375,
      "step": 280
    },
    {
      "epoch": 0.03706307112275545,
      "grad_norm": 31.165695435477424,
      "learning_rate": 4.697761756117627e-06,
      "logits/chosen": -1.5078125,
      "logits/rejected": -0.99609375,
      "logps/chosen": -288.0,
      "logps/rejected": -288.0,
      "loss": 0.3195,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -7.625,
      "rewards/margins": 3.328125,
      "rewards/rejected": -10.9375,
      "step": 290
    },
    {
      "epoch": 0.03834110805802288,
      "grad_norm": 8.224947268900738,
      "learning_rate": 4.618802153517006e-06,
      "logits/chosen": -1.578125,
      "logits/rejected": -1.0703125,
      "logps/chosen": -284.0,
      "logps/rejected": -294.0,
      "loss": 0.2811,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.78125,
      "rewards/margins": 3.6875,
      "rewards/rejected": -9.5,
      "step": 300
    },
    {
      "epoch": 0.039619144993290306,
      "grad_norm": 10.714304708458737,
      "learning_rate": 4.543694673976518e-06,
      "logits/chosen": -1.5078125,
      "logits/rejected": -1.078125,
      "logps/chosen": -278.0,
      "logps/rejected": -308.0,
      "loss": 0.2862,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -6.9375,
      "rewards/margins": 4.25,
      "rewards/rejected": -11.1875,
      "step": 310
    },
    {
      "epoch": 0.040897181928557735,
      "grad_norm": 12.213286736600438,
      "learning_rate": 4.472135954999579e-06,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.1328125,
      "logps/chosen": -276.0,
      "logps/rejected": -308.0,
      "loss": 0.2748,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -7.15625,
      "rewards/margins": 4.25,
      "rewards/rejected": -11.375,
      "step": 320
    },
    {
      "epoch": 0.042175218863825165,
      "grad_norm": 8.816716689927773,
      "learning_rate": 4.403855060505443e-06,
      "logits/chosen": -1.5703125,
      "logits/rejected": -1.1328125,
      "logps/chosen": -300.0,
      "logps/rejected": -320.0,
      "loss": 0.2611,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -8.0625,
      "rewards/margins": 4.21875,
      "rewards/rejected": -12.3125,
      "step": 330
    },
    {
      "epoch": 0.043453255799092594,
      "grad_norm": 7.5180022473301396,
      "learning_rate": 4.338609156373123e-06,
      "logits/chosen": -1.5234375,
      "logits/rejected": -1.015625,
      "logps/chosen": -308.0,
      "logps/rejected": -304.0,
      "loss": 0.284,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -8.0625,
      "rewards/margins": 4.09375,
      "rewards/rejected": -12.1875,
      "step": 340
    },
    {
      "epoch": 0.04473129273436002,
      "grad_norm": 8.837953203359074,
      "learning_rate": 4.27617987059879e-06,
      "logits/chosen": -1.5859375,
      "logits/rejected": -1.1328125,
      "logps/chosen": -278.0,
      "logps/rejected": -320.0,
      "loss": 0.2771,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -8.1875,
      "rewards/margins": 4.46875,
      "rewards/rejected": -12.625,
      "step": 350
    },
    {
      "epoch": 0.04600932966962745,
      "grad_norm": 12.418392338038274,
      "learning_rate": 4.216370213557839e-06,
      "logits/chosen": -1.625,
      "logits/rejected": -1.203125,
      "logps/chosen": -314.0,
      "logps/rejected": -314.0,
      "loss": 0.3432,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -9.125,
      "rewards/margins": 4.25,
      "rewards/rejected": -13.375,
      "step": 360
    },
    {
      "epoch": 0.04728736660489488,
      "grad_norm": 11.179079166019818,
      "learning_rate": 4.15900195928029e-06,
      "logits/chosen": -1.578125,
      "logits/rejected": -1.09375,
      "logps/chosen": -324.0,
      "logps/rejected": -340.0,
      "loss": 0.2558,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -9.9375,
      "rewards/margins": 4.90625,
      "rewards/rejected": -14.875,
      "step": 370
    },
    {
      "epoch": 0.04856540354016231,
      "grad_norm": 10.245281575173605,
      "learning_rate": 4.103913408340617e-06,
      "logits/chosen": -1.5,
      "logits/rejected": -0.99609375,
      "logps/chosen": -276.0,
      "logps/rejected": -314.0,
      "loss": 0.2193,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -8.4375,
      "rewards/margins": 4.40625,
      "rewards/rejected": -12.8125,
      "step": 380
    },
    {
      "epoch": 0.04984344047542974,
      "grad_norm": 11.694711875966762,
      "learning_rate": 4.050957468334666e-06,
      "logits/chosen": -1.4375,
      "logits/rejected": -1.0234375,
      "logps/chosen": -334.0,
      "logps/rejected": -318.0,
      "loss": 0.2731,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -10.625,
      "rewards/margins": 4.09375,
      "rewards/rejected": -14.75,
      "step": 390
    },
    {
      "epoch": 0.05112147741069717,
      "grad_norm": 9.12136191566598,
      "learning_rate": 4e-06,
      "logits/chosen": -1.484375,
      "logits/rejected": -1.1875,
      "logps/chosen": -310.0,
      "logps/rejected": -336.0,
      "loss": 0.2437,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -9.125,
      "rewards/margins": 3.90625,
      "rewards/rejected": -13.0,
      "step": 400
    },
    {
      "epoch": 0.0523995143459646,
      "grad_norm": 9.398967109872249,
      "learning_rate": 3.950918386598359e-06,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.0390625,
      "logps/chosen": -302.0,
      "logps/rejected": -332.0,
      "loss": 0.2542,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -8.6875,
      "rewards/margins": 4.78125,
      "rewards/rejected": -13.4375,
      "step": 410
    },
    {
      "epoch": 0.05367755128123203,
      "grad_norm": 10.947419724503769,
      "learning_rate": 3.903600291794132e-06,
      "logits/chosen": -1.4921875,
      "logits/rejected": -1.09375,
      "logps/chosen": -310.0,
      "logps/rejected": -328.0,
      "loss": 0.2464,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -9.375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -13.5,
      "step": 420
    },
    {
      "epoch": 0.05495558821649946,
      "grad_norm": 7.357391686206119,
      "learning_rate": 3.857942577363297e-06,
      "logits/chosen": -1.4375,
      "logits/rejected": -1.1171875,
      "logps/chosen": -318.0,
      "logps/rejected": -318.0,
      "loss": 0.2527,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -10.625,
      "rewards/margins": 3.890625,
      "rewards/rejected": -14.5,
      "step": 430
    },
    {
      "epoch": 0.056233625151766886,
      "grad_norm": 9.167491287146401,
      "learning_rate": 3.813850356982369e-06,
      "logits/chosen": -1.3671875,
      "logits/rejected": -0.98046875,
      "logps/chosen": -324.0,
      "logps/rejected": -354.0,
      "loss": 0.303,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -10.75,
      "rewards/margins": 4.65625,
      "rewards/rejected": -15.4375,
      "step": 440
    },
    {
      "epoch": 0.057511662087034315,
      "grad_norm": 11.92715381879063,
      "learning_rate": 3.7712361663282537e-06,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.0546875,
      "logps/chosen": -322.0,
      "logps/rejected": -342.0,
      "loss": 0.2442,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -10.875,
      "rewards/margins": 4.625,
      "rewards/rejected": -15.5,
      "step": 450
    },
    {
      "epoch": 0.058789699022301745,
      "grad_norm": 9.152175337110775,
      "learning_rate": 3.730019232961255e-06,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.9609375,
      "logps/chosen": -294.0,
      "logps/rejected": -332.0,
      "loss": 0.255,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -10.375,
      "rewards/margins": 4.5,
      "rewards/rejected": -14.875,
      "step": 460
    },
    {
      "epoch": 0.060067735957569174,
      "grad_norm": 8.479514710254218,
      "learning_rate": 3.6901248321155403e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -1.0234375,
      "logps/chosen": -314.0,
      "logps/rejected": -348.0,
      "loss": 0.2572,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -11.1875,
      "rewards/margins": 4.5625,
      "rewards/rejected": -15.8125,
      "step": 470
    },
    {
      "epoch": 0.0613457728928366,
      "grad_norm": 11.294433704433228,
      "learning_rate": 3.6514837167011073e-06,
      "logits/chosen": -1.4375,
      "logits/rejected": -0.97265625,
      "logps/chosen": -336.0,
      "logps/rejected": -354.0,
      "loss": 0.2726,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -10.9375,
      "rewards/margins": 4.71875,
      "rewards/rejected": -15.625,
      "step": 480
    },
    {
      "epoch": 0.06262380982810403,
      "grad_norm": 8.721780734979998,
      "learning_rate": 3.6140316116210052e-06,
      "logits/chosen": -1.4609375,
      "logits/rejected": -1.0078125,
      "logps/chosen": -314.0,
      "logps/rejected": -344.0,
      "loss": 0.2384,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -10.4375,
      "rewards/margins": 5.34375,
      "rewards/rejected": -15.75,
      "step": 490
    },
    {
      "epoch": 0.06390184676337146,
      "grad_norm": 6.681639670730452,
      "learning_rate": 3.5777087639996634e-06,
      "logits/chosen": -1.5546875,
      "logits/rejected": -1.0078125,
      "logps/chosen": -304.0,
      "logps/rejected": -332.0,
      "loss": 0.2762,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -10.0625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -14.4375,
      "step": 500
    },
    {
      "epoch": 0.06517988369863889,
      "grad_norm": 8.995120822345854,
      "learning_rate": 3.5424595421603814e-06,
      "logits/chosen": -1.421875,
      "logits/rejected": -0.99609375,
      "logps/chosen": -328.0,
      "logps/rejected": -338.0,
      "loss": 0.2889,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -11.5625,
      "rewards/margins": 4.5,
      "rewards/rejected": -16.0,
      "step": 510
    },
    {
      "epoch": 0.06645792063390632,
      "grad_norm": 10.770561253198553,
      "learning_rate": 3.5082320772281165e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.9921875,
      "logps/chosen": -308.0,
      "logps/rejected": -346.0,
      "loss": 0.252,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -11.5,
      "rewards/margins": 3.984375,
      "rewards/rejected": -15.4375,
      "step": 520
    },
    {
      "epoch": 0.06773595756917375,
      "grad_norm": 6.079296570005446,
      "learning_rate": 3.474977942104555e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.875,
      "logps/chosen": -324.0,
      "logps/rejected": -348.0,
      "loss": 0.3093,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -10.625,
      "rewards/margins": 4.6875,
      "rewards/rejected": -15.3125,
      "step": 530
    },
    {
      "epoch": 0.06901399450444118,
      "grad_norm": 12.62206416090569,
      "learning_rate": 3.442651863295481e-06,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.046875,
      "logps/chosen": -326.0,
      "logps/rejected": -356.0,
      "loss": 0.2329,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -10.4375,
      "rewards/margins": 5.03125,
      "rewards/rejected": -15.5,
      "step": 540
    },
    {
      "epoch": 0.07029203143970861,
      "grad_norm": 9.176567797536125,
      "learning_rate": 3.4112114616897665e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.9296875,
      "logps/chosen": -326.0,
      "logps/rejected": -362.0,
      "loss": 0.2517,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -12.125,
      "rewards/margins": 4.78125,
      "rewards/rejected": -16.875,
      "step": 550
    },
    {
      "epoch": 0.07157006837497604,
      "grad_norm": 8.74022454073919,
      "learning_rate": 3.3806170189140663e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -1.0859375,
      "logps/chosen": -362.0,
      "logps/rejected": -378.0,
      "loss": 0.2659,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -13.5625,
      "rewards/margins": 4.40625,
      "rewards/rejected": -18.0,
      "step": 560
    },
    {
      "epoch": 0.07284810531024347,
      "grad_norm": 10.773762894101269,
      "learning_rate": 3.350831266333564e-06,
      "logits/chosen": -1.4765625,
      "logits/rejected": -0.97265625,
      "logps/chosen": -332.0,
      "logps/rejected": -354.0,
      "loss": 0.2567,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -13.0625,
      "rewards/margins": 4.6875,
      "rewards/rejected": -17.75,
      "step": 570
    },
    {
      "epoch": 0.0741261422455109,
      "grad_norm": 8.112499776796634,
      "learning_rate": 3.3218191941495984e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -1.0078125,
      "logps/chosen": -326.0,
      "logps/rejected": -338.0,
      "loss": 0.2485,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -11.25,
      "rewards/margins": 4.40625,
      "rewards/rejected": -15.625,
      "step": 580
    },
    {
      "epoch": 0.07540417918077832,
      "grad_norm": 6.874604479059108,
      "learning_rate": 3.293547878370473e-06,
      "logits/chosen": -1.3984375,
      "logits/rejected": -0.9140625,
      "logps/chosen": -338.0,
      "logps/rejected": -362.0,
      "loss": 0.2821,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -11.4375,
      "rewards/margins": 4.28125,
      "rewards/rejected": -15.75,
      "step": 590
    },
    {
      "epoch": 0.07668221611604575,
      "grad_norm": 6.0927407879064335,
      "learning_rate": 3.2659863237109044e-06,
      "logits/chosen": -1.4609375,
      "logits/rejected": -1.046875,
      "logps/chosen": -346.0,
      "logps/rejected": -374.0,
      "loss": 0.2364,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -11.625,
      "rewards/margins": 4.6875,
      "rewards/rejected": -16.375,
      "step": 600
    },
    {
      "epoch": 0.07796025305131318,
      "grad_norm": 9.236445366800993,
      "learning_rate": 3.239105320715664e-06,
      "logits/chosen": -1.375,
      "logits/rejected": -1.0703125,
      "logps/chosen": -342.0,
      "logps/rejected": -380.0,
      "loss": 0.2465,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -12.25,
      "rewards/margins": 5.09375,
      "rewards/rejected": -17.375,
      "step": 610
    },
    {
      "epoch": 0.07923828998658061,
      "grad_norm": 9.205637729690743,
      "learning_rate": 3.2128773156099956e-06,
      "logits/chosen": -1.4453125,
      "logits/rejected": -1.1171875,
      "logps/chosen": -348.0,
      "logps/rejected": -358.0,
      "loss": 0.2138,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -12.0,
      "rewards/margins": 4.78125,
      "rewards/rejected": -16.75,
      "step": 620
    },
    {
      "epoch": 0.08051632692184804,
      "grad_norm": 7.78576931618227,
      "learning_rate": 3.187276291558383e-06,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.078125,
      "logps/chosen": -322.0,
      "logps/rejected": -338.0,
      "loss": 0.2459,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -11.3125,
      "rewards/margins": 4.84375,
      "rewards/rejected": -16.125,
      "step": 630
    },
    {
      "epoch": 0.08179436385711547,
      "grad_norm": 6.241479026309425,
      "learning_rate": 3.1622776601683788e-06,
      "logits/chosen": -1.5078125,
      "logits/rejected": -1.109375,
      "logps/chosen": -340.0,
      "logps/rejected": -368.0,
      "loss": 0.2207,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -11.375,
      "rewards/margins": 4.90625,
      "rewards/rejected": -16.25,
      "step": 640
    },
    {
      "epoch": 0.0830724007923829,
      "grad_norm": 8.366680007747462,
      "learning_rate": 3.1378581622109444e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.98828125,
      "logps/chosen": -308.0,
      "logps/rejected": -362.0,
      "loss": 0.2124,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -10.75,
      "rewards/margins": 4.96875,
      "rewards/rejected": -15.6875,
      "step": 650
    },
    {
      "epoch": 0.08435043772765033,
      "grad_norm": 7.6573367267194214,
      "learning_rate": 3.113995776646092e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.94140625,
      "logps/chosen": -320.0,
      "logps/rejected": -344.0,
      "loss": 0.245,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -11.1875,
      "rewards/margins": 5.15625,
      "rewards/rejected": -16.375,
      "step": 660
    },
    {
      "epoch": 0.08562847466291776,
      "grad_norm": 8.419148525770197,
      "learning_rate": 3.090669637145023e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -1.0078125,
      "logps/chosen": -322.0,
      "logps/rejected": -360.0,
      "loss": 0.2788,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -11.125,
      "rewards/margins": 4.5625,
      "rewards/rejected": -15.6875,
      "step": 670
    },
    {
      "epoch": 0.08690651159818519,
      "grad_norm": 9.051815254003762,
      "learning_rate": 3.0678599553894814e-06,
      "logits/chosen": -1.4453125,
      "logits/rejected": -0.93359375,
      "logps/chosen": -336.0,
      "logps/rejected": -360.0,
      "loss": 0.2477,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -10.3125,
      "rewards/margins": 4.9375,
      "rewards/rejected": -15.25,
      "step": 680
    },
    {
      "epoch": 0.08818454853345262,
      "grad_norm": 8.004690186050613,
      "learning_rate": 3.0455479505075235e-06,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.0546875,
      "logps/chosen": -316.0,
      "logps/rejected": -342.0,
      "loss": 0.2206,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -10.5,
      "rewards/margins": 4.8125,
      "rewards/rejected": -15.3125,
      "step": 690
    },
    {
      "epoch": 0.08946258546872005,
      "grad_norm": 9.039388107114702,
      "learning_rate": 3.0237157840738173e-06,
      "logits/chosen": -1.5,
      "logits/rejected": -1.140625,
      "logps/chosen": -322.0,
      "logps/rejected": -354.0,
      "loss": 0.2307,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -11.5625,
      "rewards/margins": 4.8125,
      "rewards/rejected": -16.375,
      "step": 700
    },
    {
      "epoch": 0.09074062240398748,
      "grad_norm": 8.831137542819324,
      "learning_rate": 3.002346500163206e-06,
      "logits/chosen": -1.4609375,
      "logits/rejected": -1.0078125,
      "logps/chosen": -324.0,
      "logps/rejected": -348.0,
      "loss": 0.2564,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -11.5,
      "rewards/margins": 4.4375,
      "rewards/rejected": -15.9375,
      "step": 710
    },
    {
      "epoch": 0.0920186593392549,
      "grad_norm": 12.35639948281655,
      "learning_rate": 2.9814239699997195e-06,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.93359375,
      "logps/chosen": -328.0,
      "logps/rejected": -350.0,
      "loss": 0.2241,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -10.75,
      "rewards/margins": 5.125,
      "rewards/rejected": -15.875,
      "step": 720
    },
    {
      "epoch": 0.09329669627452233,
      "grad_norm": 11.906540090311804,
      "learning_rate": 2.9609328407904207e-06,
      "logits/chosen": -1.421875,
      "logits/rejected": -0.93359375,
      "logps/chosen": -314.0,
      "logps/rejected": -336.0,
      "loss": 0.2208,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -11.375,
      "rewards/margins": 5.59375,
      "rewards/rejected": -16.875,
      "step": 730
    },
    {
      "epoch": 0.09457473320978976,
      "grad_norm": 9.910361601015193,
      "learning_rate": 2.940858488375231e-06,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.0390625,
      "logps/chosen": -348.0,
      "logps/rejected": -366.0,
      "loss": 0.2578,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -11.625,
      "rewards/margins": 5.40625,
      "rewards/rejected": -17.0,
      "step": 740
    },
    {
      "epoch": 0.09585277014505719,
      "grad_norm": 7.664973746403084,
      "learning_rate": 2.9211869733608857e-06,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.03125,
      "logps/chosen": -308.0,
      "logps/rejected": -336.0,
      "loss": 0.2206,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -10.75,
      "rewards/margins": 5.09375,
      "rewards/rejected": -15.875,
      "step": 750
    },
    {
      "epoch": 0.09713080708032462,
      "grad_norm": 8.923513183014242,
      "learning_rate": 2.901905000440047e-06,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.015625,
      "logps/chosen": -332.0,
      "logps/rejected": -370.0,
      "loss": 0.2159,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -10.9375,
      "rewards/margins": 4.5,
      "rewards/rejected": -15.4375,
      "step": 760
    },
    {
      "epoch": 0.09840884401559205,
      "grad_norm": 5.936833041892835,
      "learning_rate": 2.8829998806257885e-06,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.015625,
      "logps/chosen": -322.0,
      "logps/rejected": -356.0,
      "loss": 0.2303,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -10.75,
      "rewards/margins": 4.875,
      "rewards/rejected": -15.625,
      "step": 770
    },
    {
      "epoch": 0.09968688095085948,
      "grad_norm": 8.565353722787313,
      "learning_rate": 2.8644594961577314e-06,
      "logits/chosen": -1.4609375,
      "logits/rejected": -1.0703125,
      "logps/chosen": -340.0,
      "logps/rejected": -376.0,
      "loss": 0.2265,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -12.1875,
      "rewards/margins": 5.96875,
      "rewards/rejected": -18.125,
      "step": 780
    },
    {
      "epoch": 0.10096491788612691,
      "grad_norm": 6.009507757496106,
      "learning_rate": 2.84627226785928e-06,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.125,
      "logps/chosen": -362.0,
      "logps/rejected": -380.0,
      "loss": 0.2396,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -14.4375,
      "rewards/margins": 4.5625,
      "rewards/rejected": -19.0,
      "step": 790
    },
    {
      "epoch": 0.10224295482139434,
      "grad_norm": 8.55104947400273,
      "learning_rate": 2.82842712474619e-06,
      "logits/chosen": -1.484375,
      "logits/rejected": -0.95703125,
      "logps/chosen": -352.0,
      "logps/rejected": -388.0,
      "loss": 0.2426,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.125,
      "rewards/margins": 5.21875,
      "rewards/rejected": -19.375,
      "step": 800
    },
    {
      "epoch": 0.10352099175666177,
      "grad_norm": 10.05070126379492,
      "learning_rate": 2.810913475705226e-06,
      "logits/chosen": -1.4609375,
      "logits/rejected": -1.109375,
      "logps/chosen": -346.0,
      "logps/rejected": -370.0,
      "loss": 0.2045,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -13.5,
      "rewards/margins": 5.59375,
      "rewards/rejected": -19.0,
      "step": 810
    },
    {
      "epoch": 0.1047990286919292,
      "grad_norm": 7.527595357248868,
      "learning_rate": 2.7937211830783128e-06,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.1171875,
      "logps/chosen": -338.0,
      "logps/rejected": -436.0,
      "loss": 0.2235,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.3125,
      "rewards/margins": 10.6875,
      "rewards/rejected": -24.0,
      "step": 820
    },
    {
      "epoch": 0.10607706562719663,
      "grad_norm": 8.530666074380585,
      "learning_rate": 2.776840538002493e-06,
      "logits/chosen": -1.453125,
      "logits/rejected": -0.9453125,
      "logps/chosen": -354.0,
      "logps/rejected": -384.0,
      "loss": 0.2654,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -14.125,
      "rewards/margins": 6.09375,
      "rewards/rejected": -20.25,
      "step": 830
    },
    {
      "epoch": 0.10735510256246406,
      "grad_norm": 10.787495247687803,
      "learning_rate": 2.7602622373694163e-06,
      "logits/chosen": -1.4765625,
      "logits/rejected": -0.9453125,
      "logps/chosen": -354.0,
      "logps/rejected": -388.0,
      "loss": 0.2232,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -13.0625,
      "rewards/margins": 5.46875,
      "rewards/rejected": -18.5,
      "step": 840
    },
    {
      "epoch": 0.10863313949773148,
      "grad_norm": 7.8359825960942295,
      "learning_rate": 2.743977362280141e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.96875,
      "logps/chosen": -346.0,
      "logps/rejected": -402.0,
      "loss": 0.2467,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -12.875,
      "rewards/margins": 5.40625,
      "rewards/rejected": -18.25,
      "step": 850
    },
    {
      "epoch": 0.10991117643299891,
      "grad_norm": 13.447577856534576,
      "learning_rate": 2.7279773578818937e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.8203125,
      "logps/chosen": -342.0,
      "logps/rejected": -388.0,
      "loss": 0.1986,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -13.25,
      "rewards/margins": 5.46875,
      "rewards/rejected": -18.75,
      "step": 860
    },
    {
      "epoch": 0.11118921336826634,
      "grad_norm": 7.041609505878959,
      "learning_rate": 2.7122540144832417e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.84375,
      "logps/chosen": -354.0,
      "logps/rejected": -386.0,
      "loss": 0.2369,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -15.0625,
      "rewards/margins": 5.21875,
      "rewards/rejected": -20.25,
      "step": 870
    },
    {
      "epoch": 0.11246725030353377,
      "grad_norm": 7.781276448490546,
      "learning_rate": 2.696799449852968e-06,
      "logits/chosen": -1.390625,
      "logits/rejected": -0.98828125,
      "logps/chosen": -360.0,
      "logps/rejected": -406.0,
      "loss": 0.2232,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.5,
      "rewards/margins": 5.375,
      "rewards/rejected": -19.875,
      "step": 880
    },
    {
      "epoch": 0.1137452872388012,
      "grad_norm": 9.348557172912107,
      "learning_rate": 2.6816060926159636e-06,
      "logits/chosen": -1.40625,
      "logits/rejected": -0.8984375,
      "logps/chosen": -388.0,
      "logps/rejected": -416.0,
      "loss": 0.2618,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -14.3125,
      "rewards/margins": 5.84375,
      "rewards/rejected": -20.125,
      "step": 890
    },
    {
      "epoch": 0.11502332417406863,
      "grad_norm": 5.835346764796168,
      "learning_rate": 2.6666666666666664e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.92578125,
      "logps/chosen": -342.0,
      "logps/rejected": -382.0,
      "loss": 0.2619,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -13.3125,
      "rewards/margins": 5.1875,
      "rewards/rejected": -18.5,
      "step": 900
    },
    {
      "epoch": 0.11630136110933606,
      "grad_norm": 7.688310083217483,
      "learning_rate": 2.6519741765271837e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.90234375,
      "logps/chosen": -350.0,
      "logps/rejected": -378.0,
      "loss": 0.2111,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -12.25,
      "rewards/margins": 5.0,
      "rewards/rejected": -17.25,
      "step": 910
    },
    {
      "epoch": 0.11757939804460349,
      "grad_norm": 9.903656681697754,
      "learning_rate": 2.637521893583148e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.8203125,
      "logps/chosen": -332.0,
      "logps/rejected": -352.0,
      "loss": 0.2695,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -12.5,
      "rewards/margins": 5.78125,
      "rewards/rejected": -18.375,
      "step": 920
    },
    {
      "epoch": 0.11885743497987092,
      "grad_norm": 6.854873808238501,
      "learning_rate": 2.6233033431358115e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.81640625,
      "logps/chosen": -344.0,
      "logps/rejected": -388.0,
      "loss": 0.216,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -13.3125,
      "rewards/margins": 5.0625,
      "rewards/rejected": -18.375,
      "step": 930
    },
    {
      "epoch": 0.12013547191513835,
      "grad_norm": 8.200143990076109,
      "learning_rate": 2.6093122922137685e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.86328125,
      "logps/chosen": -332.0,
      "logps/rejected": -370.0,
      "loss": 0.2064,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -11.875,
      "rewards/margins": 5.40625,
      "rewards/rejected": -17.25,
      "step": 940
    },
    {
      "epoch": 0.12141350885040578,
      "grad_norm": 10.801289654465965,
      "learning_rate": 2.5955427380922006e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.76953125,
      "logps/chosen": -340.0,
      "logps/rejected": -392.0,
      "loss": 0.2281,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -12.5,
      "rewards/margins": 5.625,
      "rewards/rejected": -18.125,
      "step": 950
    },
    {
      "epoch": 0.1226915457856732,
      "grad_norm": 7.4824512674466614,
      "learning_rate": 2.5819888974716113e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.90234375,
      "logps/chosen": -338.0,
      "logps/rejected": -370.0,
      "loss": 0.2642,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -13.75,
      "rewards/margins": 5.15625,
      "rewards/rejected": -18.875,
      "step": 960
    },
    {
      "epoch": 0.12396958272094064,
      "grad_norm": 6.19203289064979,
      "learning_rate": 2.5686451962717425e-06,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.90234375,
      "logps/chosen": -342.0,
      "logps/rejected": -384.0,
      "loss": 0.1832,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -13.5,
      "rewards/margins": 5.46875,
      "rewards/rejected": -19.0,
      "step": 970
    },
    {
      "epoch": 0.12524761965620806,
      "grad_norm": 10.382349426307343,
      "learning_rate": 2.5555062599997596e-06,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.8515625,
      "logps/chosen": -352.0,
      "logps/rejected": -372.0,
      "loss": 0.2144,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -12.625,
      "rewards/margins": 5.40625,
      "rewards/rejected": -18.0,
      "step": 980
    },
    {
      "epoch": 0.1265256565914755,
      "grad_norm": 7.587426630074104,
      "learning_rate": 2.5425669046549126e-06,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.765625,
      "logps/chosen": -308.0,
      "logps/rejected": -354.0,
      "loss": 0.1975,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -11.75,
      "rewards/margins": 5.40625,
      "rewards/rejected": -17.25,
      "step": 990
    },
    {
      "epoch": 0.12780369352674292,
      "grad_norm": 10.223393637584227,
      "learning_rate": 2.5298221281347034e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.82421875,
      "logps/chosen": -330.0,
      "logps/rejected": -364.0,
      "loss": 0.1985,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -11.625,
      "rewards/margins": 5.25,
      "rewards/rejected": -16.875,
      "step": 1000
    },
    {
      "epoch": 0.12908173046201035,
      "grad_norm": 10.553274672351115,
      "learning_rate": 2.5172671021102103e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.765625,
      "logps/chosen": -330.0,
      "logps/rejected": -346.0,
      "loss": 0.2281,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -11.9375,
      "rewards/margins": 5.34375,
      "rewards/rejected": -17.25,
      "step": 1010
    },
    {
      "epoch": 0.13035976739727778,
      "grad_norm": 8.746281719111414,
      "learning_rate": 2.504897164340598e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.890625,
      "logps/chosen": -314.0,
      "logps/rejected": -356.0,
      "loss": 0.2421,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -11.5625,
      "rewards/margins": 5.15625,
      "rewards/rejected": -16.75,
      "step": 1020
    },
    {
      "epoch": 0.1316378043325452,
      "grad_norm": 11.388130950550568,
      "learning_rate": 2.492707811399023e-06,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.9375,
      "logps/chosen": -320.0,
      "logps/rejected": -358.0,
      "loss": 0.217,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -11.8125,
      "rewards/margins": 4.875,
      "rewards/rejected": -16.75,
      "step": 1030
    },
    {
      "epoch": 0.13291584126781264,
      "grad_norm": 8.179183929917569,
      "learning_rate": 2.480694691784169e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.82421875,
      "logps/chosen": -326.0,
      "logps/rejected": -382.0,
      "loss": 0.1904,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -12.625,
      "rewards/margins": 5.15625,
      "rewards/rejected": -17.75,
      "step": 1040
    },
    {
      "epoch": 0.13419387820308007,
      "grad_norm": 8.452079931882126,
      "learning_rate": 2.4688535993934706e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.95703125,
      "logps/chosen": -360.0,
      "logps/rejected": -376.0,
      "loss": 0.2162,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -12.5625,
      "rewards/margins": 5.46875,
      "rewards/rejected": -18.0,
      "step": 1050
    },
    {
      "epoch": 0.1354719151383475,
      "grad_norm": 10.080003508360383,
      "learning_rate": 2.457180467335805e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.9140625,
      "logps/chosen": -348.0,
      "logps/rejected": -376.0,
      "loss": 0.2009,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -12.5625,
      "rewards/margins": 6.125,
      "rewards/rejected": -18.75,
      "step": 1060
    },
    {
      "epoch": 0.13674995207361493,
      "grad_norm": 4.930020871590482,
      "learning_rate": 2.4456713620629725e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.87890625,
      "logps/chosen": -354.0,
      "logps/rejected": -380.0,
      "loss": 0.1887,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -13.8125,
      "rewards/margins": 5.125,
      "rewards/rejected": -18.875,
      "step": 1070
    },
    {
      "epoch": 0.13802798900888236,
      "grad_norm": 12.328974970402871,
      "learning_rate": 2.4343224778007378e-06,
      "logits/chosen": -1.3671875,
      "logits/rejected": -0.90625,
      "logps/chosen": -326.0,
      "logps/rejected": -378.0,
      "loss": 0.1887,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -12.6875,
      "rewards/margins": 5.84375,
      "rewards/rejected": -18.5,
      "step": 1080
    },
    {
      "epoch": 0.13930602594414979,
      "grad_norm": 7.123976380223215,
      "learning_rate": 2.4231301312615306e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -1.0390625,
      "logps/chosen": -336.0,
      "logps/rejected": -368.0,
      "loss": 0.2072,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -11.9375,
      "rewards/margins": 5.5625,
      "rewards/rejected": -17.5,
      "step": 1090
    },
    {
      "epoch": 0.14058406287941722,
      "grad_norm": 10.348894955643111,
      "learning_rate": 2.412090756622109e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.91796875,
      "logps/chosen": -338.0,
      "logps/rejected": -366.0,
      "loss": 0.1881,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -12.375,
      "rewards/margins": 5.65625,
      "rewards/rejected": -18.0,
      "step": 1100
    },
    {
      "epoch": 0.14186209981468464,
      "grad_norm": 9.555252360705285,
      "learning_rate": 2.401200900750657e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.84375,
      "logps/chosen": -324.0,
      "logps/rejected": -360.0,
      "loss": 0.2139,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -12.6875,
      "rewards/margins": 5.96875,
      "rewards/rejected": -18.625,
      "step": 1110
    },
    {
      "epoch": 0.14314013674995207,
      "grad_norm": 7.775056482706318,
      "learning_rate": 2.390457218668787e-06,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.8203125,
      "logps/chosen": -350.0,
      "logps/rejected": -370.0,
      "loss": 0.2201,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -13.875,
      "rewards/margins": 4.5625,
      "rewards/rejected": -18.375,
      "step": 1120
    },
    {
      "epoch": 0.1444181736852195,
      "grad_norm": 7.781531804026228,
      "learning_rate": 2.379856469234918e-06,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.70703125,
      "logps/chosen": -368.0,
      "logps/rejected": -376.0,
      "loss": 0.2176,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -14.375,
      "rewards/margins": 5.0625,
      "rewards/rejected": -19.375,
      "step": 1130
    },
    {
      "epoch": 0.14569621062048693,
      "grad_norm": 7.959265653617074,
      "learning_rate": 2.369395511036369e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -1.0,
      "logps/chosen": -352.0,
      "logps/rejected": -376.0,
      "loss": 0.2298,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -13.625,
      "rewards/margins": 5.21875,
      "rewards/rejected": -18.875,
      "step": 1140
    },
    {
      "epoch": 0.14697424755575436,
      "grad_norm": 7.77791723985759,
      "learning_rate": 2.359071298478354e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.90625,
      "logps/chosen": -348.0,
      "logps/rejected": -378.0,
      "loss": 0.2135,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -12.9375,
      "rewards/margins": 5.28125,
      "rewards/rejected": -18.25,
      "step": 1150
    },
    {
      "epoch": 0.1482522844910218,
      "grad_norm": 8.223034988982452,
      "learning_rate": 2.3488808780588137e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.8125,
      "logps/chosen": -338.0,
      "logps/rejected": -384.0,
      "loss": 0.1996,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -12.8125,
      "rewards/margins": 5.75,
      "rewards/rejected": -18.625,
      "step": 1160
    },
    {
      "epoch": 0.14953032142628922,
      "grad_norm": 5.7848293271122255,
      "learning_rate": 2.3388213848187446e-06,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.0,
      "logps/chosen": -334.0,
      "logps/rejected": -370.0,
      "loss": 0.1698,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -13.1875,
      "rewards/margins": 5.40625,
      "rewards/rejected": -18.625,
      "step": 1170
    },
    {
      "epoch": 0.15080835836155665,
      "grad_norm": 8.164688622869864,
      "learning_rate": 2.328890038958328e-06,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.9140625,
      "logps/chosen": -348.0,
      "logps/rejected": -386.0,
      "loss": 0.1921,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -13.3125,
      "rewards/margins": 5.96875,
      "rewards/rejected": -19.25,
      "step": 1180
    },
    {
      "epoch": 0.15208639529682408,
      "grad_norm": 8.04337169941588,
      "learning_rate": 2.3190841426097937e-06,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.171875,
      "logps/chosen": -338.0,
      "logps/rejected": -384.0,
      "loss": 0.2115,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -14.0,
      "rewards/margins": 5.53125,
      "rewards/rejected": -19.5,
      "step": 1190
    },
    {
      "epoch": 0.1533644322320915,
      "grad_norm": 9.703077665095819,
      "learning_rate": 2.309401076758503e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.8203125,
      "logps/chosen": -348.0,
      "logps/rejected": -368.0,
      "loss": 0.2356,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -12.6875,
      "rewards/margins": 5.09375,
      "rewards/rejected": -17.75,
      "step": 1200
    },
    {
      "epoch": 0.15464246916735894,
      "grad_norm": 8.91817264676939,
      "learning_rate": 2.299838298304276e-06,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.93359375,
      "logps/chosen": -324.0,
      "logps/rejected": -374.0,
      "loss": 0.1952,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -12.0,
      "rewards/margins": 5.625,
      "rewards/rejected": -17.625,
      "step": 1210
    },
    {
      "epoch": 0.15592050610262637,
      "grad_norm": 8.09612595486752,
      "learning_rate": 2.2903933372554728e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.83984375,
      "logps/chosen": -334.0,
      "logps/rejected": -384.0,
      "loss": 0.2129,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -13.4375,
      "rewards/margins": 4.96875,
      "rewards/rejected": -18.375,
      "step": 1220
    },
    {
      "epoch": 0.1571985430378938,
      "grad_norm": 8.40687465517894,
      "learning_rate": 2.281063794048804e-06,
      "logits/chosen": -1.375,
      "logits/rejected": -0.9609375,
      "logps/chosen": -356.0,
      "logps/rejected": -406.0,
      "loss": 0.1936,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.0,
      "rewards/margins": 5.03125,
      "rewards/rejected": -19.0,
      "step": 1230
    },
    {
      "epoch": 0.15847657997316122,
      "grad_norm": 38.143447949912485,
      "learning_rate": 2.271847336988259e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.96875,
      "logps/chosen": -346.0,
      "logps/rejected": -378.0,
      "loss": 0.1814,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -13.5,
      "rewards/margins": 5.84375,
      "rewards/rejected": -19.25,
      "step": 1240
    },
    {
      "epoch": 0.15975461690842865,
      "grad_norm": 9.0012252642816,
      "learning_rate": 2.262741699796952e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.82421875,
      "logps/chosen": -346.0,
      "logps/rejected": -382.0,
      "loss": 0.1959,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -12.5625,
      "rewards/margins": 6.03125,
      "rewards/rejected": -18.625,
      "step": 1250
    },
    {
      "epoch": 0.16103265384369608,
      "grad_norm": 7.7085887098229575,
      "learning_rate": 2.253744679276044e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.96484375,
      "logps/chosen": -336.0,
      "logps/rejected": -368.0,
      "loss": 0.2488,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -12.625,
      "rewards/margins": 5.3125,
      "rewards/rejected": -18.0,
      "step": 1260
    },
    {
      "epoch": 0.1623106907789635,
      "grad_norm": 8.516064849377404,
      "learning_rate": 2.244854133065255e-06,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.8671875,
      "logps/chosen": -360.0,
      "logps/rejected": -386.0,
      "loss": 0.2082,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -13.0,
      "rewards/margins": 4.8125,
      "rewards/rejected": -17.875,
      "step": 1270
    },
    {
      "epoch": 0.16358872771423094,
      "grad_norm": 8.259370651534303,
      "learning_rate": 2.2360679774997895e-06,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.83984375,
      "logps/chosen": -352.0,
      "logps/rejected": -370.0,
      "loss": 0.2143,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -14.375,
      "rewards/margins": 4.6875,
      "rewards/rejected": -19.125,
      "step": 1280
    },
    {
      "epoch": 0.16486676464949837,
      "grad_norm": 9.548931080612926,
      "learning_rate": 2.2273841855588183e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.94140625,
      "logps/chosen": -358.0,
      "logps/rejected": -382.0,
      "loss": 0.2117,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.25,
      "rewards/margins": 5.5,
      "rewards/rejected": -19.75,
      "step": 1290
    },
    {
      "epoch": 0.1661448015847658,
      "grad_norm": 8.611906621487027,
      "learning_rate": 2.2188007849009167e-06,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.83203125,
      "logps/chosen": -350.0,
      "logps/rejected": -394.0,
      "loss": 0.1978,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -13.6875,
      "rewards/margins": 5.75,
      "rewards/rejected": -19.5,
      "step": 1300
    },
    {
      "epoch": 0.16742283852003323,
      "grad_norm": 8.954643944245037,
      "learning_rate": 2.2103158559821502e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.96484375,
      "logps/chosen": -358.0,
      "logps/rejected": -396.0,
      "loss": 0.1751,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.0625,
      "rewards/margins": 5.8125,
      "rewards/rejected": -19.875,
      "step": 1310
    },
    {
      "epoch": 0.16870087545530066,
      "grad_norm": 8.530762142871044,
      "learning_rate": 2.2019275302527213e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.85546875,
      "logps/chosen": -334.0,
      "logps/rejected": -382.0,
      "loss": 0.1949,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.125,
      "rewards/margins": 5.28125,
      "rewards/rejected": -19.5,
      "step": 1320
    },
    {
      "epoch": 0.1699789123905681,
      "grad_norm": 7.935100904860878,
      "learning_rate": 2.193633988428327e-06,
      "logits/chosen": -1.3828125,
      "logits/rejected": -0.92578125,
      "logps/chosen": -348.0,
      "logps/rejected": -390.0,
      "loss": 0.2028,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.0,
      "rewards/margins": 7.34375,
      "rewards/rejected": -21.375,
      "step": 1330
    },
    {
      "epoch": 0.17125694932583552,
      "grad_norm": 19.10854859862156,
      "learning_rate": 2.185433458832612e-06,
      "logits/chosen": -1.453125,
      "logits/rejected": -1.03125,
      "logps/chosen": -364.0,
      "logps/rejected": -402.0,
      "loss": 0.1786,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.4375,
      "rewards/margins": 5.90625,
      "rewards/rejected": -20.375,
      "step": 1340
    },
    {
      "epoch": 0.17253498626110295,
      "grad_norm": 6.167399644936153,
      "learning_rate": 2.177324215807269e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.87109375,
      "logps/chosen": -374.0,
      "logps/rejected": -396.0,
      "loss": 0.2088,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -14.625,
      "rewards/margins": 5.0,
      "rewards/rejected": -19.625,
      "step": 1350
    },
    {
      "epoch": 0.17381302319637038,
      "grad_norm": 8.958518377708664,
      "learning_rate": 2.1693045781865616e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.87109375,
      "logps/chosen": -378.0,
      "logps/rejected": -410.0,
      "loss": 0.2075,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -13.875,
      "rewards/margins": 6.5,
      "rewards/rejected": -20.375,
      "step": 1360
    },
    {
      "epoch": 0.1750910601316378,
      "grad_norm": 6.390424788669095,
      "learning_rate": 2.1613729078331965e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.78125,
      "logps/chosen": -344.0,
      "logps/rejected": -374.0,
      "loss": 0.1837,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.125,
      "rewards/margins": 5.09375,
      "rewards/rejected": -19.25,
      "step": 1370
    },
    {
      "epoch": 0.17636909706690523,
      "grad_norm": 5.515284743724044,
      "learning_rate": 2.1535276082326617e-06,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.890625,
      "logps/chosen": -346.0,
      "logps/rejected": -380.0,
      "loss": 0.1943,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.625,
      "rewards/margins": 5.375,
      "rewards/rejected": -19.0,
      "step": 1380
    },
    {
      "epoch": 0.17764713400217266,
      "grad_norm": 8.4570921005926,
      "learning_rate": 2.14576712314328e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.73046875,
      "logps/chosen": -344.0,
      "logps/rejected": -376.0,
      "loss": 0.1883,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.4375,
      "rewards/margins": 5.6875,
      "rewards/rejected": -19.125,
      "step": 1390
    },
    {
      "epoch": 0.1789251709374401,
      "grad_norm": 9.549613268445338,
      "learning_rate": 2.138089935299395e-06,
      "logits/chosen": -1.4375,
      "logits/rejected": -1.015625,
      "logps/chosen": -376.0,
      "logps/rejected": -408.0,
      "loss": 0.1875,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.1875,
      "rewards/margins": 6.40625,
      "rewards/rejected": -20.625,
      "step": 1400
    },
    {
      "epoch": 0.18020320787270752,
      "grad_norm": 7.380828136210876,
      "learning_rate": 2.1304945651652297e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.83203125,
      "logps/chosen": -370.0,
      "logps/rejected": -422.0,
      "loss": 0.1704,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -14.9375,
      "rewards/margins": 6.09375,
      "rewards/rejected": -21.0,
      "step": 1410
    },
    {
      "epoch": 0.18148124480797495,
      "grad_norm": 6.263374950188119,
      "learning_rate": 2.122979569737101e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.8125,
      "logps/chosen": -366.0,
      "logps/rejected": -408.0,
      "loss": 0.1829,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -15.125,
      "rewards/margins": 6.09375,
      "rewards/rejected": -21.25,
      "step": 1420
    },
    {
      "epoch": 0.18275928174324238,
      "grad_norm": 5.833976849391102,
      "learning_rate": 2.11554354139178e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.87890625,
      "logps/chosen": -346.0,
      "logps/rejected": -392.0,
      "loss": 0.2099,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -14.625,
      "rewards/margins": 6.28125,
      "rewards/rejected": -20.875,
      "step": 1430
    },
    {
      "epoch": 0.1840373186785098,
      "grad_norm": 9.495848476346712,
      "learning_rate": 2.1081851067789196e-06,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.0234375,
      "logps/chosen": -378.0,
      "logps/rejected": -418.0,
      "loss": 0.2122,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -15.1875,
      "rewards/margins": 6.25,
      "rewards/rejected": -21.5,
      "step": 1440
    },
    {
      "epoch": 0.18531535561377724,
      "grad_norm": 9.911111591218397,
      "learning_rate": 2.1009029257555606e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.921875,
      "logps/chosen": -352.0,
      "logps/rejected": -392.0,
      "loss": 0.2132,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -13.5625,
      "rewards/margins": 5.25,
      "rewards/rejected": -18.75,
      "step": 1450
    },
    {
      "epoch": 0.18659339254904467,
      "grad_norm": 8.722955061254407,
      "learning_rate": 2.0936956903608545e-06,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.9375,
      "logps/chosen": -334.0,
      "logps/rejected": -362.0,
      "loss": 0.2117,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -12.625,
      "rewards/margins": 5.3125,
      "rewards/rejected": -18.0,
      "step": 1460
    },
    {
      "epoch": 0.1878714294843121,
      "grad_norm": 11.653053012562472,
      "learning_rate": 2.0865621238292046e-06,
      "logits/chosen": -1.4296875,
      "logits/rejected": -1.0546875,
      "logps/chosen": -358.0,
      "logps/rejected": -390.0,
      "loss": 0.2219,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -12.625,
      "rewards/margins": 5.5,
      "rewards/rejected": -18.125,
      "step": 1470
    },
    {
      "epoch": 0.18914946641957953,
      "grad_norm": 9.702295528815855,
      "learning_rate": 2.079500979640145e-06,
      "logits/chosen": -1.3828125,
      "logits/rejected": -0.93359375,
      "logps/chosen": -362.0,
      "logps/rejected": -392.0,
      "loss": 0.1891,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -13.625,
      "rewards/margins": 5.5625,
      "rewards/rejected": -19.25,
      "step": 1480
    },
    {
      "epoch": 0.19042750335484696,
      "grad_norm": 7.596019203915662,
      "learning_rate": 2.072511040603359e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.7890625,
      "logps/chosen": -358.0,
      "logps/rejected": -384.0,
      "loss": 0.1981,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -13.375,
      "rewards/margins": 5.90625,
      "rewards/rejected": -19.25,
      "step": 1490
    },
    {
      "epoch": 0.19170554029011438,
      "grad_norm": 17.55835999545242,
      "learning_rate": 2.065591117977289e-06,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.84765625,
      "logps/chosen": -346.0,
      "logps/rejected": -380.0,
      "loss": 0.1927,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -13.6875,
      "rewards/margins": 6.09375,
      "rewards/rejected": -19.75,
      "step": 1500
    },
    {
      "epoch": 0.1929835772253818,
      "grad_norm": 10.066204082582184,
      "learning_rate": 2.058740050619915e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.73046875,
      "logps/chosen": -352.0,
      "logps/rejected": -366.0,
      "loss": 0.1902,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -13.875,
      "rewards/margins": 6.03125,
      "rewards/rejected": -19.875,
      "step": 1510
    },
    {
      "epoch": 0.19426161416064924,
      "grad_norm": 9.063845876423743,
      "learning_rate": 2.0519567041703083e-06,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.83203125,
      "logps/chosen": -352.0,
      "logps/rejected": -400.0,
      "loss": 0.1805,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -13.5,
      "rewards/margins": 5.59375,
      "rewards/rejected": -19.125,
      "step": 1520
    },
    {
      "epoch": 0.19553965109591667,
      "grad_norm": 31.381712203748485,
      "learning_rate": 2.0452399702596544e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.77734375,
      "logps/chosen": -342.0,
      "logps/rejected": -378.0,
      "loss": 0.19,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -13.8125,
      "rewards/margins": 5.03125,
      "rewards/rejected": -18.875,
      "step": 1530
    },
    {
      "epoch": 0.1968176880311841,
      "grad_norm": 10.403937902501944,
      "learning_rate": 2.0385887657505017e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.8046875,
      "logps/chosen": -366.0,
      "logps/rejected": -384.0,
      "loss": 0.1993,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -13.8125,
      "rewards/margins": 5.46875,
      "rewards/rejected": -19.25,
      "step": 1540
    },
    {
      "epoch": 0.19809572496645153,
      "grad_norm": 10.449478748361528,
      "learning_rate": 2.032002032003048e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.90234375,
      "logps/chosen": -372.0,
      "logps/rejected": -408.0,
      "loss": 0.1596,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.375,
      "rewards/margins": 5.65625,
      "rewards/rejected": -20.0,
      "step": 1550
    },
    {
      "epoch": 0.19937376190171896,
      "grad_norm": 10.678174088854613,
      "learning_rate": 2.025478734167333e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.796875,
      "logps/chosen": -362.0,
      "logps/rejected": -406.0,
      "loss": 0.1941,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.125,
      "rewards/margins": 6.46875,
      "rewards/rejected": -20.625,
      "step": 1560
    },
    {
      "epoch": 0.2006517988369864,
      "grad_norm": 9.290832142092876,
      "learning_rate": 2.0190178605002747e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.84375,
      "logps/chosen": -352.0,
      "logps/rejected": -394.0,
      "loss": 0.2298,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.125,
      "rewards/margins": 6.0,
      "rewards/rejected": -20.125,
      "step": 1570
    },
    {
      "epoch": 0.20192983577225382,
      "grad_norm": 9.006013263613145,
      "learning_rate": 2.0126184217065104e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.72265625,
      "logps/chosen": -356.0,
      "logps/rejected": -384.0,
      "loss": 0.2089,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.5,
      "rewards/margins": 5.09375,
      "rewards/rejected": -20.625,
      "step": 1580
    },
    {
      "epoch": 0.20320787270752125,
      "grad_norm": 9.545196000147508,
      "learning_rate": 2.0062794503020765e-06,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.77734375,
      "logps/chosen": -368.0,
      "logps/rejected": -394.0,
      "loss": 0.1906,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -14.875,
      "rewards/margins": 5.15625,
      "rewards/rejected": -20.0,
      "step": 1590
    },
    {
      "epoch": 0.20448590964278868,
      "grad_norm": 14.017014588015606,
      "learning_rate": 2e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.8203125,
      "logps/chosen": -358.0,
      "logps/rejected": -402.0,
      "loss": 0.1768,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.375,
      "rewards/margins": 6.34375,
      "rewards/rejected": -21.75,
      "step": 1600
    },
    {
      "epoch": 0.2057639465780561,
      "grad_norm": 12.143680516585514,
      "learning_rate": 1.993779145116907e-06,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.7734375,
      "logps/chosen": -356.0,
      "logps/rejected": -412.0,
      "loss": 0.1801,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.0625,
      "rewards/margins": 6.875,
      "rewards/rejected": -22.0,
      "step": 1610
    },
    {
      "epoch": 0.20704198351332354,
      "grad_norm": 17.788019561478457,
      "learning_rate": 1.987615979999813e-06,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.76171875,
      "logps/chosen": -368.0,
      "logps/rejected": -446.0,
      "loss": 0.2137,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.375,
      "rewards/margins": 6.40625,
      "rewards/rejected": -21.75,
      "step": 1620
    },
    {
      "epoch": 0.20832002044859096,
      "grad_norm": 8.638126700789615,
      "learning_rate": 1.9815096184722797e-06,
      "logits/chosen": -1.125,
      "logits/rejected": -0.76171875,
      "logps/chosen": -376.0,
      "logps/rejected": -402.0,
      "loss": 0.1823,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -16.125,
      "rewards/margins": 5.59375,
      "rewards/rejected": -21.75,
      "step": 1630
    },
    {
      "epoch": 0.2095980573838584,
      "grad_norm": 4.332363462943313,
      "learning_rate": 1.9754591932991793e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.796875,
      "logps/chosen": -362.0,
      "logps/rejected": -410.0,
      "loss": 0.161,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.0625,
      "rewards/margins": 6.5,
      "rewards/rejected": -20.5,
      "step": 1640
    },
    {
      "epoch": 0.21087609431912582,
      "grad_norm": 15.57112713107406,
      "learning_rate": 1.9694638556693235e-06,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.78515625,
      "logps/chosen": -358.0,
      "logps/rejected": -396.0,
      "loss": 0.1749,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.8125,
      "rewards/margins": 7.1875,
      "rewards/rejected": -22.0,
      "step": 1650
    },
    {
      "epoch": 0.21215413125439325,
      "grad_norm": 377.7920454927129,
      "learning_rate": 1.963522774695264e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.8515625,
      "logps/chosen": -380.0,
      "logps/rejected": -418.0,
      "loss": 0.1955,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.75,
      "rewards/margins": 6.46875,
      "rewards/rejected": -22.25,
      "step": 1660
    },
    {
      "epoch": 0.21343216818966068,
      "grad_norm": 8.019294941904924,
      "learning_rate": 1.9576351369295853e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.8046875,
      "logps/chosen": -394.0,
      "logps/rejected": -434.0,
      "loss": 0.1668,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -16.25,
      "rewards/margins": 5.65625,
      "rewards/rejected": -21.875,
      "step": 1670
    },
    {
      "epoch": 0.2147102051249281,
      "grad_norm": 8.504402521384709,
      "learning_rate": 1.951800145897066e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.76953125,
      "logps/chosen": -354.0,
      "logps/rejected": -376.0,
      "loss": 0.2052,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -16.0,
      "rewards/margins": 5.21875,
      "rewards/rejected": -21.25,
      "step": 1680
    },
    {
      "epoch": 0.21598824206019554,
      "grad_norm": 9.142850942392126,
      "learning_rate": 1.9460170216420797e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.8828125,
      "logps/chosen": -386.0,
      "logps/rejected": -400.0,
      "loss": 0.1704,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -16.25,
      "rewards/margins": 5.3125,
      "rewards/rejected": -21.5,
      "step": 1690
    },
    {
      "epoch": 0.21726627899546297,
      "grad_norm": 14.28325315069547,
      "learning_rate": 1.9402850002906637e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.828125,
      "logps/chosen": -368.0,
      "logps/rejected": -400.0,
      "loss": 0.1995,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -15.875,
      "rewards/margins": 5.65625,
      "rewards/rejected": -21.5,
      "step": 1700
    },
    {
      "epoch": 0.2185443159307304,
      "grad_norm": 2.8177020230992915,
      "learning_rate": 1.9346033336266974e-06,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.8203125,
      "logps/chosen": -372.0,
      "logps/rejected": -402.0,
      "loss": 0.1802,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -15.375,
      "rewards/margins": 5.25,
      "rewards/rejected": -20.625,
      "step": 1710
    },
    {
      "epoch": 0.21982235286599783,
      "grad_norm": 7.067056996819025,
      "learning_rate": 1.9289712886816486e-06,
      "logits/chosen": -1.3515625,
      "logits/rejected": -0.81640625,
      "logps/chosen": -356.0,
      "logps/rejected": -396.0,
      "loss": 0.1691,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -14.0625,
      "rewards/margins": 6.34375,
      "rewards/rejected": -20.375,
      "step": 1720
    },
    {
      "epoch": 0.22110038980126526,
      "grad_norm": 78.04958948240446,
      "learning_rate": 1.92338814733738e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.83984375,
      "logps/chosen": -340.0,
      "logps/rejected": -390.0,
      "loss": 0.1905,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -13.5625,
      "rewards/margins": 6.4375,
      "rewards/rejected": -20.0,
      "step": 1730
    },
    {
      "epoch": 0.22237842673653269,
      "grad_norm": 12.184941512990717,
      "learning_rate": 1.9178532059415367e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.86328125,
      "logps/chosen": -366.0,
      "logps/rejected": -402.0,
      "loss": 0.1743,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -14.5625,
      "rewards/margins": 6.03125,
      "rewards/rejected": -20.625,
      "step": 1740
    },
    {
      "epoch": 0.22365646367180012,
      "grad_norm": 9.125673745600459,
      "learning_rate": 1.9123657749350298e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.9140625,
      "logps/chosen": -354.0,
      "logps/rejected": -378.0,
      "loss": 0.2051,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.375,
      "rewards/margins": 6.0625,
      "rewards/rejected": -20.5,
      "step": 1750
    },
    {
      "epoch": 0.22493450060706754,
      "grad_norm": 8.373944216513223,
      "learning_rate": 1.9069251784911844e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -1.015625,
      "logps/chosen": -384.0,
      "logps/rejected": -404.0,
      "loss": 0.205,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -15.875,
      "rewards/margins": 5.09375,
      "rewards/rejected": -21.0,
      "step": 1760
    },
    {
      "epoch": 0.22621253754233497,
      "grad_norm": 5.843200952238089,
      "learning_rate": 1.9015307541661132e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.83984375,
      "logps/chosen": -388.0,
      "logps/rejected": -416.0,
      "loss": 0.1761,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -16.0,
      "rewards/margins": 5.15625,
      "rewards/rejected": -21.125,
      "step": 1770
    },
    {
      "epoch": 0.2274905744776024,
      "grad_norm": 5.7890586821904915,
      "learning_rate": 1.8961818525599089e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.90234375,
      "logps/chosen": -366.0,
      "logps/rejected": -422.0,
      "loss": 0.1877,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.6875,
      "rewards/margins": 5.90625,
      "rewards/rejected": -21.625,
      "step": 1780
    },
    {
      "epoch": 0.22876861141286983,
      "grad_norm": 7.842327664831985,
      "learning_rate": 1.890877836988262e-06,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.859375,
      "logps/chosen": -354.0,
      "logps/rejected": -416.0,
      "loss": 0.1585,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.6875,
      "rewards/margins": 7.3125,
      "rewards/rejected": -22.0,
      "step": 1790
    },
    {
      "epoch": 0.23004664834813726,
      "grad_norm": 7.2870112608491455,
      "learning_rate": 1.8856180831641269e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.91796875,
      "logps/chosen": -370.0,
      "logps/rejected": -390.0,
      "loss": 0.1765,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.125,
      "rewards/margins": 5.78125,
      "rewards/rejected": -19.875,
      "step": 1800
    },
    {
      "epoch": 0.2313246852834047,
      "grad_norm": 4.52644645364935,
      "learning_rate": 1.8804019788890737e-06,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.78125,
      "logps/chosen": -356.0,
      "logps/rejected": -382.0,
      "loss": 0.1766,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -13.9375,
      "rewards/margins": 5.75,
      "rewards/rejected": -19.75,
      "step": 1810
    },
    {
      "epoch": 0.23260272221867212,
      "grad_norm": 8.308765595580109,
      "learning_rate": 1.8752289237539816e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -1.0234375,
      "logps/chosen": -350.0,
      "logps/rejected": -402.0,
      "loss": 0.1697,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.5625,
      "rewards/margins": 5.875,
      "rewards/rejected": -20.375,
      "step": 1820
    },
    {
      "epoch": 0.23388075915393955,
      "grad_norm": 12.408549898540661,
      "learning_rate": 1.8700983288487376e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.78515625,
      "logps/chosen": -362.0,
      "logps/rejected": -416.0,
      "loss": 0.2051,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -15.0,
      "rewards/margins": 6.1875,
      "rewards/rejected": -21.25,
      "step": 1830
    },
    {
      "epoch": 0.23515879608920698,
      "grad_norm": 9.45902705544609,
      "learning_rate": 1.8650096164806275e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.78515625,
      "logps/chosen": -358.0,
      "logps/rejected": -398.0,
      "loss": 0.1798,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -21.125,
      "step": 1840
    },
    {
      "epoch": 0.2364368330244744,
      "grad_norm": 7.21742966060711,
      "learning_rate": 1.8599622199011084e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.78125,
      "logps/chosen": -368.0,
      "logps/rejected": -400.0,
      "loss": 0.1824,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.8125,
      "rewards/margins": 6.25,
      "rewards/rejected": -21.0,
      "step": 1850
    },
    {
      "epoch": 0.23771486995974184,
      "grad_norm": 15.986007562763428,
      "learning_rate": 1.854955583040673e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.94921875,
      "logps/chosen": -376.0,
      "logps/rejected": -384.0,
      "loss": 0.1956,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -14.5625,
      "rewards/margins": 5.53125,
      "rewards/rejected": -20.125,
      "step": 1860
    },
    {
      "epoch": 0.23899290689500927,
      "grad_norm": 9.000142861883766,
      "learning_rate": 1.849989160251521e-06,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.76171875,
      "logps/chosen": -360.0,
      "logps/rejected": -420.0,
      "loss": 0.1738,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.6875,
      "rewards/margins": 5.5,
      "rewards/rejected": -20.25,
      "step": 1870
    },
    {
      "epoch": 0.2402709438302767,
      "grad_norm": 5.232158513035901,
      "learning_rate": 1.8450624160577701e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.8359375,
      "logps/chosen": -378.0,
      "logps/rejected": -424.0,
      "loss": 0.1694,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -15.875,
      "rewards/margins": 6.875,
      "rewards/rejected": -22.75,
      "step": 1880
    },
    {
      "epoch": 0.24154898076554412,
      "grad_norm": 6.061017046339365,
      "learning_rate": 1.8401748249129445e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.6953125,
      "logps/chosen": -358.0,
      "logps/rejected": -392.0,
      "loss": 0.1705,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.25,
      "rewards/margins": 6.78125,
      "rewards/rejected": -22.0,
      "step": 1890
    },
    {
      "epoch": 0.24282701770081155,
      "grad_norm": 8.8928126131724,
      "learning_rate": 1.8353258709644938e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.83984375,
      "logps/chosen": -358.0,
      "logps/rejected": -388.0,
      "loss": 0.2131,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.0625,
      "rewards/margins": 6.34375,
      "rewards/rejected": -21.375,
      "step": 1900
    },
    {
      "epoch": 0.24410505463607898,
      "grad_norm": 8.0790568068676,
      "learning_rate": 1.830515047825102e-06,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.71484375,
      "logps/chosen": -370.0,
      "logps/rejected": -404.0,
      "loss": 0.201,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.9375,
      "rewards/margins": 5.71875,
      "rewards/rejected": -20.75,
      "step": 1910
    },
    {
      "epoch": 0.2453830915713464,
      "grad_norm": 8.41286507829882,
      "learning_rate": 1.8257418583505536e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.75,
      "logps/chosen": -376.0,
      "logps/rejected": -388.0,
      "loss": 0.1992,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -14.5,
      "rewards/margins": 4.9375,
      "rewards/rejected": -19.375,
      "step": 1920
    },
    {
      "epoch": 0.24666112850661384,
      "grad_norm": 9.175885823550692,
      "learning_rate": 1.8210058144239416e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.8125,
      "logps/chosen": -370.0,
      "logps/rejected": -386.0,
      "loss": 0.1927,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -15.0,
      "rewards/margins": 5.09375,
      "rewards/rejected": -20.125,
      "step": 1930
    },
    {
      "epoch": 0.24793916544188127,
      "grad_norm": 6.397145920826923,
      "learning_rate": 1.816306436745999e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.9765625,
      "logps/chosen": -350.0,
      "logps/rejected": -396.0,
      "loss": 0.2024,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -14.1875,
      "rewards/margins": 5.6875,
      "rewards/rejected": -19.875,
      "step": 1940
    },
    {
      "epoch": 0.2492172023771487,
      "grad_norm": 5.572438133900194,
      "learning_rate": 1.8116432546313529e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.8359375,
      "logps/chosen": -354.0,
      "logps/rejected": -396.0,
      "loss": 0.1735,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -13.75,
      "rewards/margins": 5.8125,
      "rewards/rejected": -19.5,
      "step": 1950
    },
    {
      "epoch": 0.25049523931241613,
      "grad_norm": 6.621464616473529,
      "learning_rate": 1.8070158058105026e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.76953125,
      "logps/chosen": -356.0,
      "logps/rejected": -378.0,
      "loss": 0.2021,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.8125,
      "rewards/margins": 4.90625,
      "rewards/rejected": -19.75,
      "step": 1960
    },
    {
      "epoch": 0.25177327624768353,
      "grad_norm": 6.616293991663503,
      "learning_rate": 1.8024236362373315e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.765625,
      "logps/chosen": -350.0,
      "logps/rejected": -366.0,
      "loss": 0.189,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.25,
      "rewards/margins": 5.0625,
      "rewards/rejected": -19.25,
      "step": 1970
    },
    {
      "epoch": 0.253051313182951,
      "grad_norm": 7.042079111166021,
      "learning_rate": 1.7978662999019787e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.90234375,
      "logps/chosen": -346.0,
      "logps/rejected": -402.0,
      "loss": 0.1701,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.6875,
      "rewards/margins": 5.59375,
      "rewards/rejected": -20.25,
      "step": 1980
    },
    {
      "epoch": 0.2543293501182184,
      "grad_norm": 10.514525806737336,
      "learning_rate": 1.793343358648881e-06,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.7890625,
      "logps/chosen": -358.0,
      "logps/rejected": -414.0,
      "loss": 0.1962,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.1875,
      "rewards/margins": 5.5,
      "rewards/rejected": -20.75,
      "step": 1990
    },
    {
      "epoch": 0.25560738705348585,
      "grad_norm": 8.633722682779158,
      "learning_rate": 1.7888543819998317e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.76171875,
      "logps/chosen": -362.0,
      "logps/rejected": -412.0,
      "loss": 0.1764,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -15.0,
      "rewards/margins": 6.25,
      "rewards/rejected": -21.25,
      "step": 2000
    },
    {
      "epoch": 0.25688542398875325,
      "grad_norm": 18.371332498606307,
      "learning_rate": 1.7843989469818819e-06,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.8671875,
      "logps/chosen": -364.0,
      "logps/rejected": -404.0,
      "loss": 0.1879,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.25,
      "rewards/margins": 6.59375,
      "rewards/rejected": -20.875,
      "step": 2010
    },
    {
      "epoch": 0.2581634609240207,
      "grad_norm": 9.132239411855826,
      "learning_rate": 1.779976637959939e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.875,
      "logps/chosen": -360.0,
      "logps/rejected": -394.0,
      "loss": 0.1553,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.125,
      "rewards/margins": 5.65625,
      "rewards/rejected": -19.75,
      "step": 2020
    },
    {
      "epoch": 0.2594414978592881,
      "grad_norm": 8.320021979050132,
      "learning_rate": 1.7755870464739012e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.7890625,
      "logps/chosen": -348.0,
      "logps/rejected": -398.0,
      "loss": 0.1745,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -14.5,
      "rewards/margins": 6.375,
      "rewards/rejected": -20.875,
      "step": 2030
    },
    {
      "epoch": 0.26071953479455556,
      "grad_norm": 6.351620079524678,
      "learning_rate": 1.7712297710801907e-06,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.7421875,
      "logps/chosen": -356.0,
      "logps/rejected": -400.0,
      "loss": 0.1964,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -15.3125,
      "rewards/margins": 5.59375,
      "rewards/rejected": -20.875,
      "step": 2040
    },
    {
      "epoch": 0.26199757172982296,
      "grad_norm": 7.980486087399472,
      "learning_rate": 1.7669044171975444e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.7890625,
      "logps/chosen": -390.0,
      "logps/rejected": -416.0,
      "loss": 0.1744,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.0,
      "rewards/margins": 5.78125,
      "rewards/rejected": -21.75,
      "step": 2050
    },
    {
      "epoch": 0.2632756086650904,
      "grad_norm": 6.326141505405479,
      "learning_rate": 1.7626105969569268e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.90234375,
      "logps/chosen": -382.0,
      "logps/rejected": -414.0,
      "loss": 0.2061,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.375,
      "rewards/margins": 5.90625,
      "rewards/rejected": -21.25,
      "step": 2060
    },
    {
      "epoch": 0.2645536456003578,
      "grad_norm": 5.719037242002649,
      "learning_rate": 1.758347929055432e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.890625,
      "logps/chosen": -354.0,
      "logps/rejected": -420.0,
      "loss": 0.1886,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -13.9375,
      "rewards/margins": 6.15625,
      "rewards/rejected": -20.125,
      "step": 2070
    },
    {
      "epoch": 0.2658316825356253,
      "grad_norm": 5.268723738243688,
      "learning_rate": 1.7541160386140582e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.80859375,
      "logps/chosen": -372.0,
      "logps/rejected": -372.0,
      "loss": 0.1885,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.4375,
      "rewards/margins": 5.28125,
      "rewards/rejected": -19.75,
      "step": 2080
    },
    {
      "epoch": 0.2671097194708927,
      "grad_norm": 10.735675382088875,
      "learning_rate": 1.7499145570392284e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.796875,
      "logps/chosen": -358.0,
      "logps/rejected": -382.0,
      "loss": 0.1832,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.5,
      "rewards/margins": 5.84375,
      "rewards/rejected": -20.375,
      "step": 2090
    },
    {
      "epoch": 0.26838775640616014,
      "grad_norm": 6.375756114324437,
      "learning_rate": 1.745743121887939e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.74609375,
      "logps/chosen": -352.0,
      "logps/rejected": -384.0,
      "loss": 0.1685,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.6875,
      "rewards/margins": 6.09375,
      "rewards/rejected": -19.875,
      "step": 2100
    },
    {
      "epoch": 0.26966579334142754,
      "grad_norm": 7.440944469589943,
      "learning_rate": 1.7416013767364324e-06,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.76171875,
      "logps/chosen": -362.0,
      "logps/rejected": -392.0,
      "loss": 0.1781,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.5625,
      "rewards/margins": 6.34375,
      "rewards/rejected": -20.875,
      "step": 2110
    },
    {
      "epoch": 0.270943830276695,
      "grad_norm": 9.25032063290401,
      "learning_rate": 1.7374889710522776e-06,
      "logits/chosen": -1.0546875,
      "logits/rejected": -0.59765625,
      "logps/chosen": -348.0,
      "logps/rejected": -414.0,
      "loss": 0.2085,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.0625,
      "rewards/margins": 6.75,
      "rewards/rejected": -20.75,
      "step": 2120
    },
    {
      "epoch": 0.2722218672119624,
      "grad_norm": 6.600058600287687,
      "learning_rate": 1.7334055600697579e-06,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.77734375,
      "logps/chosen": -382.0,
      "logps/rejected": -402.0,
      "loss": 0.1684,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.5625,
      "rewards/margins": 5.9375,
      "rewards/rejected": -21.5,
      "step": 2130
    },
    {
      "epoch": 0.27349990414722986,
      "grad_norm": 6.863890076493143,
      "learning_rate": 1.7293508046684678e-06,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.66015625,
      "logps/chosen": -350.0,
      "logps/rejected": -402.0,
      "loss": 0.1535,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.5,
      "rewards/margins": 5.6875,
      "rewards/rejected": -21.25,
      "step": 2140
    },
    {
      "epoch": 0.27477794108249726,
      "grad_norm": 8.419295670801706,
      "learning_rate": 1.7253243712550145e-06,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.73828125,
      "logps/chosen": -372.0,
      "logps/rejected": -426.0,
      "loss": 0.1739,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.875,
      "rewards/margins": 6.53125,
      "rewards/rejected": -22.375,
      "step": 2150
    },
    {
      "epoch": 0.2760559780177647,
      "grad_norm": 8.354124484051964,
      "learning_rate": 1.7213259316477406e-06,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.8125,
      "logps/chosen": -384.0,
      "logps/rejected": -436.0,
      "loss": 0.1649,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -16.125,
      "rewards/margins": 6.625,
      "rewards/rejected": -22.75,
      "step": 2160
    },
    {
      "epoch": 0.2773340149530321,
      "grad_norm": 7.851350208522757,
      "learning_rate": 1.7173551629643674e-06,
      "logits/chosen": -1.125,
      "logits/rejected": -0.703125,
      "logps/chosen": -368.0,
      "logps/rejected": -398.0,
      "loss": 0.1918,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.125,
      "rewards/margins": 5.59375,
      "rewards/rejected": -21.75,
      "step": 2170
    },
    {
      "epoch": 0.27861205188829957,
      "grad_norm": 8.571544162279022,
      "learning_rate": 1.713411747512477e-06,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.71484375,
      "logps/chosen": -384.0,
      "logps/rejected": -420.0,
      "loss": 0.1801,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.25,
      "rewards/margins": 6.03125,
      "rewards/rejected": -22.25,
      "step": 2180
    },
    {
      "epoch": 0.279890088823567,
      "grad_norm": 10.341956383788414,
      "learning_rate": 1.709495372682753e-06,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.81640625,
      "logps/chosen": -386.0,
      "logps/rejected": -424.0,
      "loss": 0.1764,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -17.125,
      "rewards/margins": 6.125,
      "rewards/rejected": -23.25,
      "step": 2190
    },
    {
      "epoch": 0.28116812575883443,
      "grad_norm": 6.518717291317195,
      "learning_rate": 1.7056057308448832e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.81640625,
      "logps/chosen": -368.0,
      "logps/rejected": -406.0,
      "loss": 0.1416,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.125,
      "rewards/margins": 6.375,
      "rewards/rejected": -21.5,
      "step": 2200
    },
    {
      "epoch": 0.28244616269410183,
      "grad_norm": 3.550327685087959,
      "learning_rate": 1.701742519246068e-06,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.8359375,
      "logps/chosen": -376.0,
      "logps/rejected": -424.0,
      "loss": 0.1389,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -15.625,
      "rewards/margins": 7.125,
      "rewards/rejected": -22.75,
      "step": 2210
    },
    {
      "epoch": 0.2837241996293693,
      "grad_norm": 4.549564280441093,
      "learning_rate": 1.6979054399120355e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.875,
      "logps/chosen": -374.0,
      "logps/rejected": -414.0,
      "loss": 0.1731,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.75,
      "rewards/margins": 6.71875,
      "rewards/rejected": -22.5,
      "step": 2220
    },
    {
      "epoch": 0.2850022365646367,
      "grad_norm": 8.173725909428468,
      "learning_rate": 1.6940941995505069e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.8125,
      "logps/chosen": -372.0,
      "logps/rejected": -402.0,
      "loss": 0.1755,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -15.5625,
      "rewards/margins": 6.125,
      "rewards/rejected": -21.625,
      "step": 2230
    },
    {
      "epoch": 0.28628027349990415,
      "grad_norm": 6.504685204854815,
      "learning_rate": 1.6903085094570331e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.8671875,
      "logps/chosen": -378.0,
      "logps/rejected": -426.0,
      "loss": 0.1824,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.875,
      "rewards/margins": 5.875,
      "rewards/rejected": -21.75,
      "step": 2240
    },
    {
      "epoch": 0.28755831043517155,
      "grad_norm": 8.492689434457757,
      "learning_rate": 1.6865480854231356e-06,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.7578125,
      "logps/chosen": -366.0,
      "logps/rejected": -398.0,
      "loss": 0.1563,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.9375,
      "rewards/margins": 6.28125,
      "rewards/rejected": -21.25,
      "step": 2250
    },
    {
      "epoch": 0.288836347370439,
      "grad_norm": 7.00501974809585,
      "learning_rate": 1.682812647646685e-06,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.89453125,
      "logps/chosen": -350.0,
      "logps/rejected": -394.0,
      "loss": 0.1495,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.9375,
      "rewards/margins": 6.09375,
      "rewards/rejected": -21.0,
      "step": 2260
    },
    {
      "epoch": 0.2901143843057064,
      "grad_norm": 7.441753386795718,
      "learning_rate": 1.6791019206444541e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.83984375,
      "logps/chosen": -372.0,
      "logps/rejected": -390.0,
      "loss": 0.1467,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.9375,
      "rewards/margins": 6.40625,
      "rewards/rejected": -21.375,
      "step": 2270
    },
    {
      "epoch": 0.29139242124097386,
      "grad_norm": 6.167243864062666,
      "learning_rate": 1.675415633166782e-06,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.84375,
      "logps/chosen": -358.0,
      "logps/rejected": -398.0,
      "loss": 0.1668,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.6875,
      "rewards/margins": 6.65625,
      "rewards/rejected": -21.375,
      "step": 2280
    },
    {
      "epoch": 0.29267045817624127,
      "grad_norm": 7.423572020249587,
      "learning_rate": 1.6717535181142914e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -1.0,
      "logps/chosen": -380.0,
      "logps/rejected": -426.0,
      "loss": 0.1795,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.75,
      "rewards/margins": 6.46875,
      "rewards/rejected": -21.25,
      "step": 2290
    },
    {
      "epoch": 0.2939484951115087,
      "grad_norm": 5.912951258940523,
      "learning_rate": 1.668115312456598e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -1.03125,
      "logps/chosen": -374.0,
      "logps/rejected": -418.0,
      "loss": 0.1346,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -15.125,
      "rewards/margins": 6.75,
      "rewards/rejected": -21.875,
      "step": 2300
    },
    {
      "epoch": 0.2952265320467761,
      "grad_norm": 9.012670015274143,
      "learning_rate": 1.6645007571529578e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.7890625,
      "logps/chosen": -354.0,
      "logps/rejected": -386.0,
      "loss": 0.1772,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -15.3125,
      "rewards/margins": 6.0625,
      "rewards/rejected": -21.375,
      "step": 2310
    },
    {
      "epoch": 0.2965045689820436,
      "grad_norm": 8.263141240275903,
      "learning_rate": 1.6609095970747992e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.8203125,
      "logps/chosen": -380.0,
      "logps/rejected": -400.0,
      "loss": 0.2187,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -15.75,
      "rewards/margins": 5.9375,
      "rewards/rejected": -21.75,
      "step": 2320
    },
    {
      "epoch": 0.297782605917311,
      "grad_norm": 4.279988417561156,
      "learning_rate": 1.6573415809300833e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.7265625,
      "logps/chosen": -354.0,
      "logps/rejected": -412.0,
      "loss": 0.1567,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.1875,
      "rewards/margins": 5.65625,
      "rewards/rejected": -20.875,
      "step": 2330
    },
    {
      "epoch": 0.29906064285257844,
      "grad_norm": 6.703162826003085,
      "learning_rate": 1.6537964611894462e-06,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.78515625,
      "logps/chosen": -364.0,
      "logps/rejected": -390.0,
      "loss": 0.1635,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.4375,
      "rewards/margins": 5.59375,
      "rewards/rejected": -20.0,
      "step": 2340
    },
    {
      "epoch": 0.30033867978784584,
      "grad_norm": 5.474679115348783,
      "learning_rate": 1.6502739940140692e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.8984375,
      "logps/chosen": -378.0,
      "logps/rejected": -418.0,
      "loss": 0.1907,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.1875,
      "rewards/margins": 6.4375,
      "rewards/rejected": -21.625,
      "step": 2350
    },
    {
      "epoch": 0.3016167167231133,
      "grad_norm": 8.440300163070988,
      "learning_rate": 1.6467739391852364e-06,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.765625,
      "logps/chosen": -368.0,
      "logps/rejected": -402.0,
      "loss": 0.2064,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -15.1875,
      "rewards/margins": 6.28125,
      "rewards/rejected": -21.5,
      "step": 2360
    },
    {
      "epoch": 0.3028947536583807,
      "grad_norm": 5.479599557746082,
      "learning_rate": 1.6432960600355221e-06,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.73046875,
      "logps/chosen": -370.0,
      "logps/rejected": -424.0,
      "loss": 0.1849,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -14.5,
      "rewards/margins": 6.03125,
      "rewards/rejected": -20.5,
      "step": 2370
    },
    {
      "epoch": 0.30417279059364816,
      "grad_norm": 6.835709879830589,
      "learning_rate": 1.6398401233815756e-06,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.78125,
      "logps/chosen": -356.0,
      "logps/rejected": -380.0,
      "loss": 0.1817,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.5625,
      "rewards/margins": 6.0625,
      "rewards/rejected": -20.625,
      "step": 2380
    },
    {
      "epoch": 0.30545082752891556,
      "grad_norm": 4.344420911042229,
      "learning_rate": 1.6364058994584524e-06,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.81640625,
      "logps/chosen": -366.0,
      "logps/rejected": -398.0,
      "loss": 0.1866,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.25,
      "rewards/margins": 6.15625,
      "rewards/rejected": -21.5,
      "step": 2390
    },
    {
      "epoch": 0.306728864464183,
      "grad_norm": 4.871698301531515,
      "learning_rate": 1.6329931618554522e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.79296875,
      "logps/chosen": -350.0,
      "logps/rejected": -392.0,
      "loss": 0.1599,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -14.875,
      "rewards/margins": 6.28125,
      "rewards/rejected": -21.125,
      "step": 2400
    },
    {
      "epoch": 0.3080069013994504,
      "grad_norm": 6.4310150550293015,
      "learning_rate": 1.6296016874534209e-06,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.78125,
      "logps/chosen": -380.0,
      "logps/rejected": -384.0,
      "loss": 0.1785,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -14.8125,
      "rewards/margins": 6.375,
      "rewards/rejected": -21.125,
      "step": 2410
    },
    {
      "epoch": 0.3092849383347179,
      "grad_norm": 8.091350582560779,
      "learning_rate": 1.6262312563634835e-06,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.79296875,
      "logps/chosen": -340.0,
      "logps/rejected": -384.0,
      "loss": 0.1622,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.0625,
      "rewards/margins": 7.03125,
      "rewards/rejected": -20.125,
      "step": 2420
    },
    {
      "epoch": 0.3105629752699853,
      "grad_norm": 8.173408715010284,
      "learning_rate": 1.6228816518671587e-06,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.7109375,
      "logps/chosen": -352.0,
      "logps/rejected": -380.0,
      "loss": 0.145,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -13.5625,
      "rewards/margins": 6.09375,
      "rewards/rejected": -19.75,
      "step": 2430
    },
    {
      "epoch": 0.31184101220525273,
      "grad_norm": 8.307777929289866,
      "learning_rate": 1.619552660357832e-06,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.83984375,
      "logps/chosen": -372.0,
      "logps/rejected": -406.0,
      "loss": 0.1786,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -14.25,
      "rewards/margins": 5.71875,
      "rewards/rejected": -20.0,
      "step": 2440
    },
    {
      "epoch": 0.31311904914052013,
      "grad_norm": 8.037346416486134,
      "learning_rate": 1.616244071283537e-06,
      "logits/chosen": -1.0859375,
      "logits/rejected": -0.75,
      "logps/chosen": -348.0,
      "logps/rejected": -420.0,
      "loss": 0.1786,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.8125,
      "rewards/margins": 5.53125,
      "rewards/rejected": -20.375,
      "step": 2450
    },
    {
      "epoch": 0.3143970860757876,
      "grad_norm": 4.598772736488354,
      "learning_rate": 1.6129556770910235e-06,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.734375,
      "logps/chosen": -340.0,
      "logps/rejected": -384.0,
      "loss": 0.1516,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -14.8125,
      "rewards/margins": 5.90625,
      "rewards/rejected": -20.75,
      "step": 2460
    },
    {
      "epoch": 0.315675123011055,
      "grad_norm": 9.516098388693068,
      "learning_rate": 1.6096872731710673e-06,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.85546875,
      "logps/chosen": -380.0,
      "logps/rejected": -420.0,
      "loss": 0.1676,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.5,
      "rewards/margins": 6.96875,
      "rewards/rejected": -22.375,
      "step": 2470
    },
    {
      "epoch": 0.31695315994632245,
      "grad_norm": 4.631558058752617,
      "learning_rate": 1.6064386578049978e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.91015625,
      "logps/chosen": -376.0,
      "logps/rejected": -422.0,
      "loss": 0.1652,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.5,
      "rewards/margins": 6.40625,
      "rewards/rejected": -20.875,
      "step": 2480
    },
    {
      "epoch": 0.31823119688158985,
      "grad_norm": 8.54424101337341,
      "learning_rate": 1.6032096321124046e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.81640625,
      "logps/chosen": -352.0,
      "logps/rejected": -378.0,
      "loss": 0.1393,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -13.625,
      "rewards/margins": 6.34375,
      "rewards/rejected": -20.0,
      "step": 2490
    },
    {
      "epoch": 0.3195092338168573,
      "grad_norm": 7.797885991647695,
      "learning_rate": 1.6e-06,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.8515625,
      "logps/chosen": -372.0,
      "logps/rejected": -422.0,
      "loss": 0.2337,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.75,
      "rewards/margins": 6.53125,
      "rewards/rejected": -21.25,
      "step": 2500
    },
    {
      "epoch": 0.3207872707521247,
      "grad_norm": 7.412229924158176,
      "learning_rate": 1.5968095681115984e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.90234375,
      "logps/chosen": -362.0,
      "logps/rejected": -436.0,
      "loss": 0.1701,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.9375,
      "rewards/margins": 7.375,
      "rewards/rejected": -22.25,
      "step": 2510
    },
    {
      "epoch": 0.32206530768739217,
      "grad_norm": 6.782291579224652,
      "learning_rate": 1.5936381457791914e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.91796875,
      "logps/chosen": -364.0,
      "logps/rejected": -380.0,
      "loss": 0.1608,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -13.625,
      "rewards/margins": 6.4375,
      "rewards/rejected": -20.0,
      "step": 2520
    },
    {
      "epoch": 0.32334334462265957,
      "grad_norm": 4.9692010410188585,
      "learning_rate": 1.590485544975088e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.96875,
      "logps/chosen": -350.0,
      "logps/rejected": -388.0,
      "loss": 0.1568,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -13.5625,
      "rewards/margins": 6.40625,
      "rewards/rejected": -20.0,
      "step": 2530
    },
    {
      "epoch": 0.324621381557927,
      "grad_norm": 7.435679923432267,
      "learning_rate": 1.5873515802650901e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.76953125,
      "logps/chosen": -352.0,
      "logps/rejected": -378.0,
      "loss": 0.1564,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -13.0625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -19.5,
      "step": 2540
    },
    {
      "epoch": 0.3258994184931944,
      "grad_norm": 7.586791561859869,
      "learning_rate": 1.584236068762679e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.88671875,
      "logps/chosen": -360.0,
      "logps/rejected": -406.0,
      "loss": 0.1374,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -14.8125,
      "rewards/margins": 6.625,
      "rewards/rejected": -21.375,
      "step": 2550
    },
    {
      "epoch": 0.3271774554284619,
      "grad_norm": 6.8995652015140205,
      "learning_rate": 1.5811388300841894e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.9375,
      "logps/chosen": -396.0,
      "logps/rejected": -410.0,
      "loss": 0.1763,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -15.5625,
      "rewards/margins": 6.90625,
      "rewards/rejected": -22.5,
      "step": 2560
    },
    {
      "epoch": 0.3284554923637293,
      "grad_norm": 5.35312877031937,
      "learning_rate": 1.5780596863049431e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.8046875,
      "logps/chosen": -364.0,
      "logps/rejected": -406.0,
      "loss": 0.1474,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.4375,
      "rewards/margins": 6.375,
      "rewards/rejected": -21.875,
      "step": 2570
    },
    {
      "epoch": 0.32973352929899674,
      "grad_norm": 8.38143060871466,
      "learning_rate": 1.5749984619163156e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.96484375,
      "logps/chosen": -358.0,
      "logps/rejected": -398.0,
      "loss": 0.1616,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.4375,
      "rewards/margins": 7.3125,
      "rewards/rejected": -21.75,
      "step": 2580
    },
    {
      "epoch": 0.33101156623426414,
      "grad_norm": 11.190613106126323,
      "learning_rate": 1.5719549837837187e-06,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.86328125,
      "logps/chosen": -376.0,
      "logps/rejected": -396.0,
      "loss": 0.1623,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -14.4375,
      "rewards/margins": 6.5,
      "rewards/rejected": -20.875,
      "step": 2590
    },
    {
      "epoch": 0.3322896031695316,
      "grad_norm": 6.903427457268049,
      "learning_rate": 1.5689290811054722e-06,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.87109375,
      "logps/chosen": -368.0,
      "logps/rejected": -402.0,
      "loss": 0.1725,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -14.9375,
      "rewards/margins": 6.125,
      "rewards/rejected": -21.125,
      "step": 2600
    },
    {
      "epoch": 0.333567640104799,
      "grad_norm": 7.016850224823338,
      "learning_rate": 1.5659205853725426e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.92578125,
      "logps/chosen": -362.0,
      "logps/rejected": -414.0,
      "loss": 0.1819,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -14.6875,
      "rewards/margins": 5.75,
      "rewards/rejected": -20.375,
      "step": 2610
    },
    {
      "epoch": 0.33484567704006646,
      "grad_norm": 3.3592430991404374,
      "learning_rate": 1.562929330329127e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.80859375,
      "logps/chosen": -338.0,
      "logps/rejected": -388.0,
      "loss": 0.1548,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -14.5,
      "rewards/margins": 5.71875,
      "rewards/rejected": -20.25,
      "step": 2620
    },
    {
      "epoch": 0.33612371397533386,
      "grad_norm": 9.916737864551118,
      "learning_rate": 1.5599551519340636e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.80078125,
      "logps/chosen": -352.0,
      "logps/rejected": -404.0,
      "loss": 0.1764,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.4375,
      "rewards/margins": 6.53125,
      "rewards/rejected": -21.0,
      "step": 2630
    },
    {
      "epoch": 0.3374017509106013,
      "grad_norm": 8.784713053781411,
      "learning_rate": 1.556997888323046e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.87890625,
      "logps/chosen": -384.0,
      "logps/rejected": -414.0,
      "loss": 0.1884,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -15.9375,
      "rewards/margins": 6.09375,
      "rewards/rejected": -22.0,
      "step": 2640
    },
    {
      "epoch": 0.3386797878458687,
      "grad_norm": 5.88442385550547,
      "learning_rate": 1.5540573797716226e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.87890625,
      "logps/chosen": -354.0,
      "logps/rejected": -406.0,
      "loss": 0.1431,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.8125,
      "rewards/margins": 6.40625,
      "rewards/rejected": -21.25,
      "step": 2650
    },
    {
      "epoch": 0.3399578247811362,
      "grad_norm": 8.004724254093452,
      "learning_rate": 1.5511334686589623e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -1.03125,
      "logps/chosen": -390.0,
      "logps/rejected": -430.0,
      "loss": 0.1676,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -17.0,
      "rewards/margins": 6.1875,
      "rewards/rejected": -23.25,
      "step": 2660
    },
    {
      "epoch": 0.3412358617164036,
      "grad_norm": 10.014756278323372,
      "learning_rate": 1.548225999432367e-06,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.78515625,
      "logps/chosen": -384.0,
      "logps/rejected": -416.0,
      "loss": 0.1595,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -16.875,
      "rewards/margins": 5.5625,
      "rewards/rejected": -22.5,
      "step": 2670
    },
    {
      "epoch": 0.34251389865167103,
      "grad_norm": 5.952696177013581,
      "learning_rate": 1.5453348185725114e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.875,
      "logps/chosen": -376.0,
      "logps/rejected": -414.0,
      "loss": 0.1712,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -16.5,
      "rewards/margins": 6.40625,
      "rewards/rejected": -22.875,
      "step": 2680
    },
    {
      "epoch": 0.34379193558693844,
      "grad_norm": 9.637261698353962,
      "learning_rate": 1.542459774559398e-06,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.7578125,
      "logps/chosen": -372.0,
      "logps/rejected": -412.0,
      "loss": 0.1655,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.8125,
      "rewards/margins": 7.125,
      "rewards/rejected": -23.0,
      "step": 2690
    },
    {
      "epoch": 0.3450699725222059,
      "grad_norm": 6.680174552709829,
      "learning_rate": 1.539600717839002e-06,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.87109375,
      "logps/chosen": -368.0,
      "logps/rejected": -416.0,
      "loss": 0.1665,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -15.875,
      "rewards/margins": 6.75,
      "rewards/rejected": -22.625,
      "step": 2700
    },
    {
      "epoch": 0.3463480094574733,
      "grad_norm": 8.235809092583775,
      "learning_rate": 1.536757500790597e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.74609375,
      "logps/chosen": -370.0,
      "logps/rejected": -398.0,
      "loss": 0.1805,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -16.125,
      "rewards/margins": 6.15625,
      "rewards/rejected": -22.25,
      "step": 2710
    },
    {
      "epoch": 0.34762604639274075,
      "grad_norm": 7.416167848623114,
      "learning_rate": 1.5339299776947407e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.921875,
      "logps/chosen": -408.0,
      "logps/rejected": -432.0,
      "loss": 0.1785,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.125,
      "rewards/margins": 6.375,
      "rewards/rejected": -23.5,
      "step": 2720
    },
    {
      "epoch": 0.34890408332800815,
      "grad_norm": 4.754198191009482,
      "learning_rate": 1.5311180047019054e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.77734375,
      "logps/chosen": -366.0,
      "logps/rejected": -408.0,
      "loss": 0.1531,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.375,
      "rewards/margins": 6.59375,
      "rewards/rejected": -23.0,
      "step": 2730
    },
    {
      "epoch": 0.3501821202632756,
      "grad_norm": 10.342217584069093,
      "learning_rate": 1.5283214398017402e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.83203125,
      "logps/chosen": -354.0,
      "logps/rejected": -402.0,
      "loss": 0.1538,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -16.125,
      "rewards/margins": 6.15625,
      "rewards/rejected": -22.25,
      "step": 2740
    },
    {
      "epoch": 0.351460157198543,
      "grad_norm": 10.957791790476678,
      "learning_rate": 1.5255401427929477e-06,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.7265625,
      "logps/chosen": -352.0,
      "logps/rejected": -416.0,
      "loss": 0.1647,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.0,
      "rewards/margins": 7.21875,
      "rewards/rejected": -22.25,
      "step": 2750
    },
    {
      "epoch": 0.35273819413381047,
      "grad_norm": 5.412848387495793,
      "learning_rate": 1.5227739752537617e-06,
      "logits/chosen": -1.1015625,
      "logits/rejected": -0.67578125,
      "logps/chosen": -380.0,
      "logps/rejected": -414.0,
      "loss": 0.1731,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -15.625,
      "rewards/margins": 6.59375,
      "rewards/rejected": -22.25,
      "step": 2760
    },
    {
      "epoch": 0.35401623106907787,
      "grad_norm": 5.358366197634935,
      "learning_rate": 1.5200228005130127e-06,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.765625,
      "logps/chosen": -376.0,
      "logps/rejected": -406.0,
      "loss": 0.1427,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -15.75,
      "rewards/margins": 5.96875,
      "rewards/rejected": -21.75,
      "step": 2770
    },
    {
      "epoch": 0.3552942680043453,
      "grad_norm": 7.5367918947665675,
      "learning_rate": 1.5172864836217631e-06,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.8203125,
      "logps/chosen": -382.0,
      "logps/rejected": -448.0,
      "loss": 0.1352,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -16.875,
      "rewards/margins": 6.375,
      "rewards/rejected": -23.25,
      "step": 2780
    },
    {
      "epoch": 0.3565723049396127,
      "grad_norm": 8.235192447266364,
      "learning_rate": 1.514564891325506e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.78125,
      "logps/chosen": -396.0,
      "logps/rejected": -426.0,
      "loss": 0.1554,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -17.25,
      "rewards/margins": 6.09375,
      "rewards/rejected": -23.375,
      "step": 2790
    },
    {
      "epoch": 0.3578503418748802,
      "grad_norm": 9.314267573220476,
      "learning_rate": 1.5118578920369086e-06,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.81640625,
      "logps/chosen": -406.0,
      "logps/rejected": -438.0,
      "loss": 0.1458,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.09375,
      "rewards/rejected": -23.625,
      "step": 2800
    },
    {
      "epoch": 0.3591283788101476,
      "grad_norm": 8.334808455115347,
      "learning_rate": 1.5091653558090898e-06,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.73046875,
      "logps/chosen": -362.0,
      "logps/rejected": -418.0,
      "loss": 0.1947,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.25,
      "rewards/margins": 6.25,
      "rewards/rejected": -22.5,
      "step": 2810
    },
    {
      "epoch": 0.36040641574541504,
      "grad_norm": 9.017869279372748,
      "learning_rate": 1.506487154309419e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.671875,
      "logps/chosen": -406.0,
      "logps/rejected": -434.0,
      "loss": 0.1607,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.625,
      "rewards/margins": 6.96875,
      "rewards/rejected": -23.625,
      "step": 2820
    },
    {
      "epoch": 0.36168445268068244,
      "grad_norm": 8.398489278507387,
      "learning_rate": 1.5038231607938247e-06,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.70703125,
      "logps/chosen": -388.0,
      "logps/rejected": -428.0,
      "loss": 0.1474,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -17.0,
      "rewards/margins": 6.8125,
      "rewards/rejected": -23.75,
      "step": 2830
    },
    {
      "epoch": 0.3629624896159499,
      "grad_norm": 8.98056329040152,
      "learning_rate": 1.501173250081603e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.953125,
      "logps/chosen": -400.0,
      "logps/rejected": -440.0,
      "loss": 0.1832,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -17.625,
      "rewards/margins": 7.15625,
      "rewards/rejected": -24.75,
      "step": 2840
    },
    {
      "epoch": 0.3642405265512173,
      "grad_norm": 3.631405415493462,
      "learning_rate": 1.4985372985307103e-06,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.78125,
      "logps/chosen": -404.0,
      "logps/rejected": -458.0,
      "loss": 0.1503,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.0625,
      "rewards/rejected": -23.75,
      "step": 2850
    },
    {
      "epoch": 0.36551856348648476,
      "grad_norm": 4.819913331659097,
      "learning_rate": 1.4959151840135313e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.93359375,
      "logps/chosen": -392.0,
      "logps/rejected": -446.0,
      "loss": 0.1181,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.34375,
      "rewards/rejected": -24.0,
      "step": 2860
    },
    {
      "epoch": 0.36679660042175216,
      "grad_norm": 6.012224391924082,
      "learning_rate": 1.4933067858931148e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.73046875,
      "logps/chosen": -342.0,
      "logps/rejected": -428.0,
      "loss": 0.1285,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -15.9375,
      "rewards/margins": 7.0625,
      "rewards/rejected": -23.0,
      "step": 2870
    },
    {
      "epoch": 0.3680746373570196,
      "grad_norm": 8.149056814970246,
      "learning_rate": 1.4907119849998597e-06,
      "logits/chosen": -1.15625,
      "logits/rejected": -0.81640625,
      "logps/chosen": -364.0,
      "logps/rejected": -416.0,
      "loss": 0.1304,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.75,
      "rewards/margins": 6.75,
      "rewards/rejected": -22.5,
      "step": 2880
    },
    {
      "epoch": 0.369352674292287,
      "grad_norm": 7.399935517875265,
      "learning_rate": 1.488130663608649e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.8125,
      "logps/chosen": -356.0,
      "logps/rejected": -420.0,
      "loss": 0.1187,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.625,
      "rewards/margins": 6.875,
      "rewards/rejected": -22.5,
      "step": 2890
    },
    {
      "epoch": 0.3706307112275545,
      "grad_norm": 5.092194237451721,
      "learning_rate": 1.4855627054164149e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.734375,
      "logps/chosen": -374.0,
      "logps/rejected": -438.0,
      "loss": 0.1471,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.125,
      "rewards/margins": 6.65625,
      "rewards/rejected": -23.75,
      "step": 2900
    },
    {
      "epoch": 0.3719087481628219,
      "grad_norm": 7.26327821963915,
      "learning_rate": 1.4830079955201294e-06,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.69140625,
      "logps/chosen": -390.0,
      "logps/rejected": -438.0,
      "loss": 0.1556,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.125,
      "rewards/margins": 8.125,
      "rewards/rejected": -24.25,
      "step": 2910
    },
    {
      "epoch": 0.37318678509808934,
      "grad_norm": 14.906869619296195,
      "learning_rate": 1.4804664203952103e-06,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.58203125,
      "logps/chosen": -368.0,
      "logps/rejected": -440.0,
      "loss": 0.1635,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.0,
      "rewards/margins": 7.09375,
      "rewards/rejected": -24.125,
      "step": 2920
    },
    {
      "epoch": 0.37446482203335674,
      "grad_norm": 6.831024418447815,
      "learning_rate": 1.4779378678743327e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.7734375,
      "logps/chosen": -400.0,
      "logps/rejected": -436.0,
      "loss": 0.1375,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -18.375,
      "rewards/margins": 6.6875,
      "rewards/rejected": -25.125,
      "step": 2930
    },
    {
      "epoch": 0.3757428589686242,
      "grad_norm": 11.87740582575529,
      "learning_rate": 1.4754222271266348e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.86328125,
      "logps/chosen": -404.0,
      "logps/rejected": -436.0,
      "loss": 0.1505,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -24.375,
      "step": 2940
    },
    {
      "epoch": 0.3770208959038916,
      "grad_norm": 4.897986183732004,
      "learning_rate": 1.4729193886373175e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.8203125,
      "logps/chosen": -388.0,
      "logps/rejected": -444.0,
      "loss": 0.1251,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.375,
      "rewards/margins": 6.78125,
      "rewards/rejected": -24.25,
      "step": 2950
    },
    {
      "epoch": 0.37829893283915905,
      "grad_norm": 9.07537762107968,
      "learning_rate": 1.4704292441876156e-06,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.83984375,
      "logps/chosen": -406.0,
      "logps/rejected": -436.0,
      "loss": 0.1579,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.25,
      "rewards/margins": 7.78125,
      "rewards/rejected": -25.0,
      "step": 2960
    },
    {
      "epoch": 0.37957696977442645,
      "grad_norm": 8.605454163095152,
      "learning_rate": 1.4679516868351474e-06,
      "logits/chosen": -1.1171875,
      "logits/rejected": -0.75,
      "logps/chosen": -378.0,
      "logps/rejected": -432.0,
      "loss": 0.1902,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -18.125,
      "rewards/margins": 6.3125,
      "rewards/rejected": -24.375,
      "step": 2970
    },
    {
      "epoch": 0.3808550067096939,
      "grad_norm": 7.3354029924134165,
      "learning_rate": 1.4654866108946234e-06,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.6953125,
      "logps/chosen": -390.0,
      "logps/rejected": -442.0,
      "loss": 0.1608,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -17.875,
      "rewards/margins": 6.875,
      "rewards/rejected": -24.75,
      "step": 2980
    },
    {
      "epoch": 0.3821330436449613,
      "grad_norm": 6.155418860212319,
      "learning_rate": 1.4630339119189101e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.78515625,
      "logps/chosen": -392.0,
      "logps/rejected": -430.0,
      "loss": 0.1353,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.125,
      "rewards/margins": 5.75,
      "rewards/rejected": -23.875,
      "step": 2990
    },
    {
      "epoch": 0.38341108058022877,
      "grad_norm": 5.403666679517565,
      "learning_rate": 1.4605934866804429e-06,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.83203125,
      "logps/chosen": -390.0,
      "logps/rejected": -446.0,
      "loss": 0.1469,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -19.0,
      "rewards/margins": 5.65625,
      "rewards/rejected": -24.625,
      "step": 3000
    },
    {
      "epoch": 0.38468911751549617,
      "grad_norm": 6.389340594085818,
      "learning_rate": 1.4581652331529784e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.8828125,
      "logps/chosen": -400.0,
      "logps/rejected": -444.0,
      "loss": 0.1595,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.0,
      "rewards/margins": 6.71875,
      "rewards/rejected": -24.625,
      "step": 3010
    },
    {
      "epoch": 0.3859671544507636,
      "grad_norm": 6.834943049888752,
      "learning_rate": 1.4557490504936778e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.8203125,
      "logps/chosen": -398.0,
      "logps/rejected": -446.0,
      "loss": 0.1624,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.625,
      "rewards/margins": 6.90625,
      "rewards/rejected": -24.5,
      "step": 3020
    },
    {
      "epoch": 0.38724519138603103,
      "grad_norm": 9.868830042691824,
      "learning_rate": 1.453344839025519e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.8046875,
      "logps/chosen": -412.0,
      "logps/rejected": -464.0,
      "loss": 0.1585,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.25,
      "rewards/margins": 6.71875,
      "rewards/rejected": -25.0,
      "step": 3030
    },
    {
      "epoch": 0.3885232283212985,
      "grad_norm": 4.815120173317151,
      "learning_rate": 1.4509525002200234e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.99609375,
      "logps/chosen": -404.0,
      "logps/rejected": -424.0,
      "loss": 0.1657,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -17.625,
      "rewards/margins": 6.25,
      "rewards/rejected": -23.875,
      "step": 3040
    },
    {
      "epoch": 0.3898012652565659,
      "grad_norm": 5.908075246079269,
      "learning_rate": 1.4485719366802965e-06,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.85546875,
      "logps/chosen": -396.0,
      "logps/rejected": -444.0,
      "loss": 0.1417,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -17.25,
      "rewards/margins": 6.8125,
      "rewards/rejected": -24.0,
      "step": 3050
    },
    {
      "epoch": 0.39107930219183334,
      "grad_norm": 5.599288022876796,
      "learning_rate": 1.4462030521243742e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.80078125,
      "logps/chosen": -402.0,
      "logps/rejected": -428.0,
      "loss": 0.1171,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.03125,
      "rewards/rejected": -23.75,
      "step": 3060
    },
    {
      "epoch": 0.39235733912710075,
      "grad_norm": 8.265629169805178,
      "learning_rate": 1.443845751368867e-06,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.84375,
      "logps/chosen": -386.0,
      "logps/rejected": -438.0,
      "loss": 0.1653,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -16.375,
      "rewards/margins": 6.96875,
      "rewards/rejected": -23.375,
      "step": 3070
    },
    {
      "epoch": 0.3936353760623682,
      "grad_norm": 4.139917186421324,
      "learning_rate": 1.4414999403128943e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.83984375,
      "logps/chosen": -362.0,
      "logps/rejected": -412.0,
      "loss": 0.1625,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -15.4375,
      "rewards/margins": 6.90625,
      "rewards/rejected": -22.25,
      "step": 3080
    },
    {
      "epoch": 0.3949134129976356,
      "grad_norm": 5.942930998801207,
      "learning_rate": 1.439165525922309e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.796875,
      "logps/chosen": -378.0,
      "logps/rejected": -422.0,
      "loss": 0.145,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.5,
      "rewards/margins": 6.28125,
      "rewards/rejected": -22.75,
      "step": 3090
    },
    {
      "epoch": 0.39619144993290306,
      "grad_norm": 4.319772742437636,
      "learning_rate": 1.4368424162141992e-06,
      "logits/chosen": -1.3671875,
      "logits/rejected": -0.96484375,
      "logps/chosen": -376.0,
      "logps/rejected": -416.0,
      "loss": 0.1813,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.25,
      "rewards/margins": 6.375,
      "rewards/rejected": -22.625,
      "step": 3100
    },
    {
      "epoch": 0.39746948686817046,
      "grad_norm": 5.130278794235308,
      "learning_rate": 1.434530520241665e-06,
      "logits/chosen": -1.3671875,
      "logits/rejected": -0.87109375,
      "logps/chosen": -364.0,
      "logps/rejected": -432.0,
      "loss": 0.1335,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.9375,
      "rewards/margins": 6.4375,
      "rewards/rejected": -22.375,
      "step": 3110
    },
    {
      "epoch": 0.3987475238034379,
      "grad_norm": 9.325617973309617,
      "learning_rate": 1.4322297480788657e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.94140625,
      "logps/chosen": -360.0,
      "logps/rejected": -406.0,
      "loss": 0.1545,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -14.875,
      "rewards/margins": 6.71875,
      "rewards/rejected": -21.625,
      "step": 3120
    },
    {
      "epoch": 0.4000255607387053,
      "grad_norm": 8.780450151065313,
      "learning_rate": 1.4299400108063247e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.90234375,
      "logps/chosen": -380.0,
      "logps/rejected": -424.0,
      "loss": 0.1473,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -16.0,
      "rewards/margins": 7.09375,
      "rewards/rejected": -23.125,
      "step": 3130
    },
    {
      "epoch": 0.4013035976739728,
      "grad_norm": 10.2040261805842,
      "learning_rate": 1.4276612204964992e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.8984375,
      "logps/chosen": -376.0,
      "logps/rejected": -430.0,
      "loss": 0.1199,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.09375,
      "rewards/rejected": -23.25,
      "step": 3140
    },
    {
      "epoch": 0.4025816346092402,
      "grad_norm": 7.057410129328684,
      "learning_rate": 1.4253932901995967e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.80078125,
      "logps/chosen": -396.0,
      "logps/rejected": -454.0,
      "loss": 0.1645,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.375,
      "rewards/rejected": -23.875,
      "step": 3150
    },
    {
      "epoch": 0.40385967154450764,
      "grad_norm": 9.19615316476253,
      "learning_rate": 1.42313613392964e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.94921875,
      "logps/chosen": -398.0,
      "logps/rejected": -436.0,
      "loss": 0.1507,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.0,
      "rewards/margins": 6.28125,
      "rewards/rejected": -24.25,
      "step": 3160
    },
    {
      "epoch": 0.40513770847977504,
      "grad_norm": 5.529158499367323,
      "learning_rate": 1.4208896666507756e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.87890625,
      "logps/chosen": -390.0,
      "logps/rejected": -432.0,
      "loss": 0.1376,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.25,
      "rewards/margins": 7.28125,
      "rewards/rejected": -24.625,
      "step": 3170
    },
    {
      "epoch": 0.4064157454150425,
      "grad_norm": 8.839300613924307,
      "learning_rate": 1.4186538042638173e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.90625,
      "logps/chosen": -398.0,
      "logps/rejected": -428.0,
      "loss": 0.1418,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -17.75,
      "rewards/margins": 6.4375,
      "rewards/rejected": -24.125,
      "step": 3180
    },
    {
      "epoch": 0.4076937823503099,
      "grad_norm": 9.807154546126684,
      "learning_rate": 1.416428463593022e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.8828125,
      "logps/chosen": -374.0,
      "logps/rejected": -428.0,
      "loss": 0.1429,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.25,
      "rewards/margins": 6.90625,
      "rewards/rejected": -24.125,
      "step": 3190
    },
    {
      "epoch": 0.40897181928557735,
      "grad_norm": 6.209826291258967,
      "learning_rate": 1.414213562373095e-06,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.8828125,
      "logps/chosen": -382.0,
      "logps/rejected": -430.0,
      "loss": 0.1688,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -17.625,
      "rewards/margins": 7.21875,
      "rewards/rejected": -24.875,
      "step": 3200
    },
    {
      "epoch": 0.41024985622084476,
      "grad_norm": 6.7685423285524,
      "learning_rate": 1.4120090192364154e-06,
      "logits/chosen": -1.3515625,
      "logits/rejected": -0.890625,
      "logps/chosen": -378.0,
      "logps/rejected": -432.0,
      "loss": 0.1296,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.875,
      "rewards/margins": 7.375,
      "rewards/rejected": -24.25,
      "step": 3210
    },
    {
      "epoch": 0.4115278931561122,
      "grad_norm": 8.116545814283745,
      "learning_rate": 1.4098147537004828e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.8203125,
      "logps/chosen": -384.0,
      "logps/rejected": -430.0,
      "loss": 0.1744,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -16.75,
      "rewards/margins": 6.75,
      "rewards/rejected": -23.5,
      "step": 3220
    },
    {
      "epoch": 0.4128059300913796,
      "grad_norm": 7.781823219134506,
      "learning_rate": 1.4076306861555735e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -1.0078125,
      "logps/chosen": -384.0,
      "logps/rejected": -422.0,
      "loss": 0.1359,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -17.0,
      "rewards/margins": 6.5,
      "rewards/rejected": -23.5,
      "step": 3230
    },
    {
      "epoch": 0.41408396702664707,
      "grad_norm": 8.946415474640869,
      "learning_rate": 1.405456737852613e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.88671875,
      "logps/chosen": -382.0,
      "logps/rejected": -438.0,
      "loss": 0.1446,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -16.625,
      "rewards/margins": 7.1875,
      "rewards/rejected": -23.875,
      "step": 3240
    },
    {
      "epoch": 0.41536200396191447,
      "grad_norm": 4.764559880498014,
      "learning_rate": 1.4032928308912468e-06,
      "logits/chosen": -1.375,
      "logits/rejected": -0.96875,
      "logps/chosen": -400.0,
      "logps/rejected": -450.0,
      "loss": 0.145,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.125,
      "rewards/margins": 6.53125,
      "rewards/rejected": -23.625,
      "step": 3250
    },
    {
      "epoch": 0.41664004089718193,
      "grad_norm": 4.3114908803784315,
      "learning_rate": 1.4011388882081175e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.88671875,
      "logps/chosen": -366.0,
      "logps/rejected": -404.0,
      "loss": 0.1509,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.75,
      "rewards/margins": 6.375,
      "rewards/rejected": -23.125,
      "step": 3260
    },
    {
      "epoch": 0.41791807783244933,
      "grad_norm": 3.855559530088154,
      "learning_rate": 1.3989948335653378e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.90234375,
      "logps/chosen": -386.0,
      "logps/rejected": -418.0,
      "loss": 0.1312,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -16.875,
      "rewards/margins": 6.625,
      "rewards/rejected": -23.5,
      "step": 3270
    },
    {
      "epoch": 0.4191961147677168,
      "grad_norm": 7.950890171041907,
      "learning_rate": 1.3968605915391564e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.83203125,
      "logps/chosen": -380.0,
      "logps/rejected": -422.0,
      "loss": 0.1369,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.375,
      "rewards/margins": 7.15625,
      "rewards/rejected": -24.5,
      "step": 3280
    },
    {
      "epoch": 0.4204741517029842,
      "grad_norm": 6.158235945972364,
      "learning_rate": 1.3947360875088132e-06,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.953125,
      "logps/chosen": -386.0,
      "logps/rejected": -434.0,
      "loss": 0.147,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.875,
      "rewards/margins": 7.03125,
      "rewards/rejected": -24.0,
      "step": 3290
    },
    {
      "epoch": 0.42175218863825165,
      "grad_norm": 6.627835436865485,
      "learning_rate": 1.3926212476455828e-06,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.85546875,
      "logps/chosen": -378.0,
      "logps/rejected": -440.0,
      "loss": 0.1448,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -18.125,
      "rewards/margins": 6.40625,
      "rewards/rejected": -24.5,
      "step": 3300
    },
    {
      "epoch": 0.42303022557351905,
      "grad_norm": 6.60348599001986,
      "learning_rate": 1.3905159989019964e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.74609375,
      "logps/chosen": -396.0,
      "logps/rejected": -468.0,
      "loss": 0.1557,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.375,
      "rewards/margins": 6.65625,
      "rewards/rejected": -24.0,
      "step": 3310
    },
    {
      "epoch": 0.4243082625087865,
      "grad_norm": 8.9669865493656,
      "learning_rate": 1.3884202690012465e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.80078125,
      "logps/chosen": -402.0,
      "logps/rejected": -432.0,
      "loss": 0.1629,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -17.75,
      "rewards/margins": 6.53125,
      "rewards/rejected": -24.25,
      "step": 3320
    },
    {
      "epoch": 0.4255862994440539,
      "grad_norm": 7.357437627405348,
      "learning_rate": 1.3863339864267636e-06,
      "logits/chosen": -1.1640625,
      "logits/rejected": -0.8125,
      "logps/chosen": -370.0,
      "logps/rejected": -430.0,
      "loss": 0.152,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.875,
      "rewards/margins": 7.0625,
      "rewards/rejected": -24.0,
      "step": 3330
    },
    {
      "epoch": 0.42686433637932136,
      "grad_norm": 7.467529833429661,
      "learning_rate": 1.3842570804119655e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.921875,
      "logps/chosen": -380.0,
      "logps/rejected": -444.0,
      "loss": 0.1451,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -17.25,
      "rewards/margins": 7.5,
      "rewards/rejected": -24.75,
      "step": 3340
    },
    {
      "epoch": 0.42814237331458876,
      "grad_norm": 5.796634143094144,
      "learning_rate": 1.3821894809301763e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.80078125,
      "logps/chosen": -404.0,
      "logps/rejected": -436.0,
      "loss": 0.132,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.625,
      "rewards/margins": 7.15625,
      "rewards/rejected": -24.75,
      "step": 3350
    },
    {
      "epoch": 0.4294204102498562,
      "grad_norm": 8.127845689577871,
      "learning_rate": 1.3801311186847081e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.765625,
      "logps/chosen": -390.0,
      "logps/rejected": -446.0,
      "loss": 0.1421,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.0,
      "rewards/margins": 7.34375,
      "rewards/rejected": -24.375,
      "step": 3360
    },
    {
      "epoch": 0.4306984471851236,
      "grad_norm": 7.959998820521283,
      "learning_rate": 1.378081925099109e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.76953125,
      "logps/chosen": -370.0,
      "logps/rejected": -444.0,
      "loss": 0.135,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.375,
      "rewards/margins": 6.96875,
      "rewards/rejected": -23.375,
      "step": 3370
    },
    {
      "epoch": 0.4319764841203911,
      "grad_norm": 6.2696403620436865,
      "learning_rate": 1.376041832307563e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.71875,
      "logps/chosen": -364.0,
      "logps/rejected": -432.0,
      "loss": 0.1536,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.375,
      "rewards/rejected": -23.75,
      "step": 3380
    },
    {
      "epoch": 0.4332545210556585,
      "grad_norm": 7.7561440507099295,
      "learning_rate": 1.3740107731454524e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.8984375,
      "logps/chosen": -374.0,
      "logps/rejected": -414.0,
      "loss": 0.1281,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -16.75,
      "rewards/margins": 6.59375,
      "rewards/rejected": -23.375,
      "step": 3390
    },
    {
      "epoch": 0.43453255799092594,
      "grad_norm": 7.179476693659551,
      "learning_rate": 1.3719886811400705e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.8984375,
      "logps/chosen": -382.0,
      "logps/rejected": -428.0,
      "loss": 0.1399,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.0,
      "rewards/rejected": -23.25,
      "step": 3400
    },
    {
      "epoch": 0.43581059492619334,
      "grad_norm": 5.306936289656675,
      "learning_rate": 1.3699754905014834e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.91015625,
      "logps/chosen": -370.0,
      "logps/rejected": -412.0,
      "loss": 0.1237,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -17.0,
      "rewards/margins": 6.8125,
      "rewards/rejected": -23.75,
      "step": 3410
    },
    {
      "epoch": 0.4370886318614608,
      "grad_norm": 7.33511917259012,
      "learning_rate": 1.3679711361135388e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.765625,
      "logps/chosen": -372.0,
      "logps/rejected": -432.0,
      "loss": 0.1406,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.0,
      "rewards/margins": 7.15625,
      "rewards/rejected": -24.25,
      "step": 3420
    },
    {
      "epoch": 0.4383666687967282,
      "grad_norm": 7.891421525368514,
      "learning_rate": 1.3659755535250212e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.796875,
      "logps/chosen": -376.0,
      "logps/rejected": -446.0,
      "loss": 0.1446,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.125,
      "rewards/rejected": -23.625,
      "step": 3430
    },
    {
      "epoch": 0.43964470573199566,
      "grad_norm": 4.4977245488764455,
      "learning_rate": 1.3639886789409469e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.87109375,
      "logps/chosen": -372.0,
      "logps/rejected": -408.0,
      "loss": 0.1218,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -17.25,
      "rewards/margins": 5.75,
      "rewards/rejected": -23.0,
      "step": 3440
    },
    {
      "epoch": 0.44092274266726306,
      "grad_norm": 6.192133824919038,
      "learning_rate": 1.3620104492139977e-06,
      "logits/chosen": -1.390625,
      "logits/rejected": -0.79296875,
      "logps/chosen": -370.0,
      "logps/rejected": -434.0,
      "loss": 0.1334,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.375,
      "rewards/margins": 7.53125,
      "rewards/rejected": -24.875,
      "step": 3450
    },
    {
      "epoch": 0.4422007796025305,
      "grad_norm": 6.748695841419781,
      "learning_rate": 1.3600408018360918e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.86328125,
      "logps/chosen": -374.0,
      "logps/rejected": -412.0,
      "loss": 0.1546,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -17.125,
      "rewards/margins": 6.78125,
      "rewards/rejected": -23.875,
      "step": 3460
    },
    {
      "epoch": 0.4434788165377979,
      "grad_norm": 8.576278709172726,
      "learning_rate": 1.3580796749300878e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.859375,
      "logps/chosen": -416.0,
      "logps/rejected": -420.0,
      "loss": 0.1281,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.0,
      "rewards/margins": 7.3125,
      "rewards/rejected": -24.375,
      "step": 3470
    },
    {
      "epoch": 0.44475685347306537,
      "grad_norm": 6.507460744728536,
      "learning_rate": 1.3561270072416209e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.859375,
      "logps/chosen": -406.0,
      "logps/rejected": -436.0,
      "loss": 0.1488,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.75,
      "rewards/margins": 6.96875,
      "rewards/rejected": -24.75,
      "step": 3480
    },
    {
      "epoch": 0.4460348904083328,
      "grad_norm": 5.859961217813401,
      "learning_rate": 1.3541827381310652e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.90625,
      "logps/chosen": -374.0,
      "logps/rejected": -432.0,
      "loss": 0.1209,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.875,
      "rewards/margins": 7.21875,
      "rewards/rejected": -24.125,
      "step": 3490
    },
    {
      "epoch": 0.44731292734360023,
      "grad_norm": 5.743291788596569,
      "learning_rate": 1.3522468075656264e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.8203125,
      "logps/chosen": -356.0,
      "logps/rejected": -434.0,
      "loss": 0.1454,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.625,
      "rewards/margins": 7.28125,
      "rewards/rejected": -23.875,
      "step": 3500
    },
    {
      "epoch": 0.44859096427886763,
      "grad_norm": 8.567744687752718,
      "learning_rate": 1.3503191561115553e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.7890625,
      "logps/chosen": -374.0,
      "logps/rejected": -434.0,
      "loss": 0.1283,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -17.0,
      "rewards/margins": 7.1875,
      "rewards/rejected": -24.125,
      "step": 3510
    },
    {
      "epoch": 0.4498690012141351,
      "grad_norm": 7.21394304984864,
      "learning_rate": 1.348399724926484e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.86328125,
      "logps/chosen": -398.0,
      "logps/rejected": -434.0,
      "loss": 0.1499,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -16.625,
      "rewards/margins": 7.84375,
      "rewards/rejected": -24.5,
      "step": 3520
    },
    {
      "epoch": 0.4511470381494025,
      "grad_norm": 6.8718798225064095,
      "learning_rate": 1.346488455751882e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.78125,
      "logps/chosen": -384.0,
      "logps/rejected": -432.0,
      "loss": 0.1839,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.75,
      "rewards/margins": 6.96875,
      "rewards/rejected": -24.75,
      "step": 3530
    },
    {
      "epoch": 0.45242507508466995,
      "grad_norm": 7.900397417506228,
      "learning_rate": 1.3445852909056286e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.8359375,
      "logps/chosen": -388.0,
      "logps/rejected": -440.0,
      "loss": 0.1669,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.5,
      "rewards/margins": 6.6875,
      "rewards/rejected": -24.125,
      "step": 3540
    },
    {
      "epoch": 0.45370311201993735,
      "grad_norm": 7.580032666610403,
      "learning_rate": 1.3426901732747024e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.91796875,
      "logps/chosen": -408.0,
      "logps/rejected": -424.0,
      "loss": 0.1404,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.5,
      "rewards/margins": 6.75,
      "rewards/rejected": -24.25,
      "step": 3550
    },
    {
      "epoch": 0.4549811489552048,
      "grad_norm": 7.799351090372204,
      "learning_rate": 1.3408030463079818e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.77734375,
      "logps/chosen": -378.0,
      "logps/rejected": -412.0,
      "loss": 0.1573,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.375,
      "rewards/margins": 6.84375,
      "rewards/rejected": -24.25,
      "step": 3560
    },
    {
      "epoch": 0.4562591858904722,
      "grad_norm": 7.816339760338242,
      "learning_rate": 1.3389238540091568e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.73828125,
      "logps/chosen": -376.0,
      "logps/rejected": -432.0,
      "loss": 0.1267,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.25,
      "rewards/margins": 6.84375,
      "rewards/rejected": -24.0,
      "step": 3570
    },
    {
      "epoch": 0.45753722282573966,
      "grad_norm": 9.401171045263284,
      "learning_rate": 1.337052540929751e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.8046875,
      "logps/chosen": -374.0,
      "logps/rejected": -452.0,
      "loss": 0.1277,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.0,
      "rewards/margins": 7.5,
      "rewards/rejected": -24.5,
      "step": 3580
    },
    {
      "epoch": 0.45881525976100707,
      "grad_norm": 9.870017460070583,
      "learning_rate": 1.33518905216225e-06,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.91015625,
      "logps/chosen": -392.0,
      "logps/rejected": -426.0,
      "loss": 0.1469,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.125,
      "rewards/margins": 6.78125,
      "rewards/rejected": -24.0,
      "step": 3590
    },
    {
      "epoch": 0.4600932966962745,
      "grad_norm": 5.500179153225504,
      "learning_rate": 1.3333333333333332e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.85546875,
      "logps/chosen": -382.0,
      "logps/rejected": -444.0,
      "loss": 0.1304,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.0,
      "rewards/margins": 7.3125,
      "rewards/rejected": -24.25,
      "step": 3600
    },
    {
      "epoch": 0.4613713336315419,
      "grad_norm": 3.973386142931335,
      "learning_rate": 1.3314853305972122e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.76953125,
      "logps/chosen": -388.0,
      "logps/rejected": -422.0,
      "loss": 0.1517,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.25,
      "rewards/rejected": -23.75,
      "step": 3610
    },
    {
      "epoch": 0.4626493705668094,
      "grad_norm": 6.111037112873324,
      "learning_rate": 1.3296449906290671e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.8671875,
      "logps/chosen": -376.0,
      "logps/rejected": -420.0,
      "loss": 0.1204,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -16.125,
      "rewards/margins": 6.4375,
      "rewards/rejected": -22.5,
      "step": 3620
    },
    {
      "epoch": 0.4639274075020768,
      "grad_norm": 5.943152067380111,
      "learning_rate": 1.3278122606185844e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.921875,
      "logps/chosen": -368.0,
      "logps/rejected": -416.0,
      "loss": 0.13,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.625,
      "rewards/margins": 7.21875,
      "rewards/rejected": -23.75,
      "step": 3630
    },
    {
      "epoch": 0.46520544443734424,
      "grad_norm": 6.481672436539191,
      "learning_rate": 1.3259870882635918e-06,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.83984375,
      "logps/chosen": -404.0,
      "logps/rejected": -452.0,
      "loss": 0.1597,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -17.5,
      "rewards/margins": 6.28125,
      "rewards/rejected": -23.875,
      "step": 3640
    },
    {
      "epoch": 0.46648348137261164,
      "grad_norm": 4.085915928721287,
      "learning_rate": 1.3241694217637886e-06,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.90625,
      "logps/chosen": -376.0,
      "logps/rejected": -410.0,
      "loss": 0.1331,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.375,
      "rewards/margins": 6.875,
      "rewards/rejected": -24.25,
      "step": 3650
    },
    {
      "epoch": 0.4677615183078791,
      "grad_norm": 7.836128061976743,
      "learning_rate": 1.3223592098145723e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.8984375,
      "logps/chosen": -400.0,
      "logps/rejected": -468.0,
      "loss": 0.1322,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.5,
      "rewards/margins": 8.3125,
      "rewards/rejected": -24.75,
      "step": 3660
    },
    {
      "epoch": 0.4690395552431465,
      "grad_norm": 7.49772140941902,
      "learning_rate": 1.3205564016009555e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.8828125,
      "logps/chosen": -384.0,
      "logps/rejected": -448.0,
      "loss": 0.1144,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.25,
      "rewards/margins": 7.25,
      "rewards/rejected": -24.5,
      "step": 3670
    },
    {
      "epoch": 0.47031759217841396,
      "grad_norm": 5.529251780644596,
      "learning_rate": 1.318760946791574e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.78515625,
      "logps/chosen": -360.0,
      "logps/rejected": -410.0,
      "loss": 0.1248,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -17.0,
      "rewards/margins": 7.65625,
      "rewards/rejected": -24.625,
      "step": 3680
    },
    {
      "epoch": 0.47159562911368136,
      "grad_norm": 8.730661176458746,
      "learning_rate": 1.316972795532786e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.875,
      "logps/chosen": -396.0,
      "logps/rejected": -440.0,
      "loss": 0.1552,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -16.875,
      "rewards/margins": 7.25,
      "rewards/rejected": -24.125,
      "step": 3690
    },
    {
      "epoch": 0.4728736660489488,
      "grad_norm": 3.2419995762732334,
      "learning_rate": 1.3151918984428582e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.82421875,
      "logps/chosen": -372.0,
      "logps/rejected": -424.0,
      "loss": 0.0937,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.9375,
      "rewards/margins": 7.65625,
      "rewards/rejected": -23.625,
      "step": 3700
    },
    {
      "epoch": 0.4741517029842162,
      "grad_norm": 6.188841935853007,
      "learning_rate": 1.313418206606237e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.921875,
      "logps/chosen": -374.0,
      "logps/rejected": -420.0,
      "loss": 0.1252,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -23.125,
      "step": 3710
    },
    {
      "epoch": 0.4754297399194837,
      "grad_norm": 7.468751992893234,
      "learning_rate": 1.3116516715679057e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.87890625,
      "logps/chosen": -376.0,
      "logps/rejected": -410.0,
      "loss": 0.1608,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.25,
      "rewards/margins": 6.84375,
      "rewards/rejected": -23.125,
      "step": 3720
    },
    {
      "epoch": 0.4767077768547511,
      "grad_norm": 5.808655909227352,
      "learning_rate": 1.3098922453278258e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.89453125,
      "logps/chosen": -402.0,
      "logps/rejected": -430.0,
      "loss": 0.1347,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -17.875,
      "rewards/margins": 7.125,
      "rewards/rejected": -25.0,
      "step": 3730
    },
    {
      "epoch": 0.47798581379001853,
      "grad_norm": 6.003123080875553,
      "learning_rate": 1.3081398803354573e-06,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.953125,
      "logps/chosen": -404.0,
      "logps/rejected": -452.0,
      "loss": 0.1239,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.375,
      "rewards/margins": 7.75,
      "rewards/rejected": -25.125,
      "step": 3740
    },
    {
      "epoch": 0.47926385072528593,
      "grad_norm": 3.8591751975851296,
      "learning_rate": 1.3063945294843617e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.90625,
      "logps/chosen": -376.0,
      "logps/rejected": -440.0,
      "loss": 0.1389,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.78125,
      "rewards/rejected": -24.0,
      "step": 3750
    },
    {
      "epoch": 0.4805418876605534,
      "grad_norm": 4.019948083029915,
      "learning_rate": 1.3046561461068843e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.85546875,
      "logps/chosen": -378.0,
      "logps/rejected": -440.0,
      "loss": 0.1261,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.0625,
      "rewards/rejected": -23.25,
      "step": 3760
    },
    {
      "epoch": 0.4818199245958208,
      "grad_norm": 7.528197680804647,
      "learning_rate": 1.3029246839689124e-06,
      "logits/chosen": -1.3515625,
      "logits/rejected": -0.95703125,
      "logps/chosen": -396.0,
      "logps/rejected": -448.0,
      "loss": 0.142,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.5625,
      "rewards/rejected": -24.25,
      "step": 3770
    },
    {
      "epoch": 0.48309796153108825,
      "grad_norm": 6.7744718025634105,
      "learning_rate": 1.3012000972647109e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.890625,
      "logps/chosen": -382.0,
      "logps/rejected": -420.0,
      "loss": 0.1574,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.8125,
      "rewards/margins": 6.5625,
      "rewards/rejected": -22.375,
      "step": 3780
    },
    {
      "epoch": 0.48437599846635565,
      "grad_norm": 5.75171347028417,
      "learning_rate": 1.299482340611832e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.734375,
      "logps/chosen": -360.0,
      "logps/rejected": -426.0,
      "loss": 0.1749,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -15.4375,
      "rewards/margins": 6.28125,
      "rewards/rejected": -21.75,
      "step": 3790
    },
    {
      "epoch": 0.4856540354016231,
      "grad_norm": 8.118201944930425,
      "learning_rate": 1.2977713690461003e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.8046875,
      "logps/chosen": -364.0,
      "logps/rejected": -412.0,
      "loss": 0.1506,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.5,
      "rewards/margins": 6.25,
      "rewards/rejected": -22.75,
      "step": 3800
    },
    {
      "epoch": 0.4869320723368905,
      "grad_norm": 5.781074466455824,
      "learning_rate": 1.296067138016669e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.87109375,
      "logps/chosen": -386.0,
      "logps/rejected": -416.0,
      "loss": 0.137,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -17.0,
      "rewards/margins": 6.78125,
      "rewards/rejected": -23.75,
      "step": 3810
    },
    {
      "epoch": 0.48821010927215797,
      "grad_norm": 8.567547243991521,
      "learning_rate": 1.294369603381147e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.87109375,
      "logps/chosen": -392.0,
      "logps/rejected": -430.0,
      "loss": 0.1392,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.125,
      "rewards/margins": 7.3125,
      "rewards/rejected": -24.5,
      "step": 3820
    },
    {
      "epoch": 0.48948814620742537,
      "grad_norm": 6.390349764077649,
      "learning_rate": 1.2926787214007981e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.8515625,
      "logps/chosen": -404.0,
      "logps/rejected": -432.0,
      "loss": 0.1176,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -17.375,
      "rewards/margins": 7.40625,
      "rewards/rejected": -24.75,
      "step": 3830
    },
    {
      "epoch": 0.4907661831426928,
      "grad_norm": 3.172624398276498,
      "learning_rate": 1.2909944487358056e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.84375,
      "logps/chosen": -376.0,
      "logps/rejected": -408.0,
      "loss": 0.1279,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.71875,
      "rewards/rejected": -23.875,
      "step": 3840
    },
    {
      "epoch": 0.4920442200779602,
      "grad_norm": 6.877494065462421,
      "learning_rate": 1.2893167424406084e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.89453125,
      "logps/chosen": -392.0,
      "logps/rejected": -428.0,
      "loss": 0.1124,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.625,
      "rewards/margins": 6.75,
      "rewards/rejected": -24.375,
      "step": 3850
    },
    {
      "epoch": 0.4933222570132277,
      "grad_norm": 4.7852166106607195,
      "learning_rate": 1.2876455599593008e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.921875,
      "logps/chosen": -396.0,
      "logps/rejected": -440.0,
      "loss": 0.1298,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.75,
      "rewards/margins": 7.9375,
      "rewards/rejected": -25.75,
      "step": 3860
    },
    {
      "epoch": 0.4946002939484951,
      "grad_norm": 6.776250771068472,
      "learning_rate": 1.285980859121099e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.89453125,
      "logps/chosen": -388.0,
      "logps/rejected": -444.0,
      "loss": 0.1449,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -18.0,
      "rewards/margins": 7.59375,
      "rewards/rejected": -25.625,
      "step": 3870
    },
    {
      "epoch": 0.49587833088376254,
      "grad_norm": 7.220973098332561,
      "learning_rate": 1.2843225981358712e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.85546875,
      "logps/chosen": -402.0,
      "logps/rejected": -460.0,
      "loss": 0.1508,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.25,
      "rewards/margins": 7.84375,
      "rewards/rejected": -26.125,
      "step": 3880
    },
    {
      "epoch": 0.49715636781902994,
      "grad_norm": 5.84048599845536,
      "learning_rate": 1.2826707355897317e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.828125,
      "logps/chosen": -400.0,
      "logps/rejected": -460.0,
      "loss": 0.1222,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.75,
      "rewards/margins": 8.125,
      "rewards/rejected": -25.875,
      "step": 3890
    },
    {
      "epoch": 0.4984344047542974,
      "grad_norm": 5.849381490106306,
      "learning_rate": 1.281025230440697e-06,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.83203125,
      "logps/chosen": -398.0,
      "logps/rejected": -446.0,
      "loss": 0.1524,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -18.25,
      "rewards/margins": 7.25,
      "rewards/rejected": -25.375,
      "step": 3900
    },
    {
      "epoch": 0.4997124416895648,
      "grad_norm": 7.420121633210637,
      "learning_rate": 1.2793860420144025e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.8984375,
      "logps/chosen": -408.0,
      "logps/rejected": -454.0,
      "loss": 0.1261,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.625,
      "rewards/margins": 7.8125,
      "rewards/rejected": -25.375,
      "step": 3910
    },
    {
      "epoch": 0.5009904786248323,
      "grad_norm": 4.194460822711977,
      "learning_rate": 1.2777531299998798e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.98046875,
      "logps/chosen": -414.0,
      "logps/rejected": -454.0,
      "loss": 0.1085,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.09375,
      "rewards/rejected": -23.875,
      "step": 3920
    },
    {
      "epoch": 0.5022685155600997,
      "grad_norm": 6.44365877864499,
      "learning_rate": 1.2761264544453928e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.79296875,
      "logps/chosen": -386.0,
      "logps/rejected": -428.0,
      "loss": 0.1137,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -16.25,
      "rewards/margins": 6.21875,
      "rewards/rejected": -22.5,
      "step": 3930
    },
    {
      "epoch": 0.5035465524953671,
      "grad_norm": 9.471471498856264,
      "learning_rate": 1.2745059757543324e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.9765625,
      "logps/chosen": -394.0,
      "logps/rejected": -434.0,
      "loss": 0.132,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.625,
      "rewards/margins": 6.75,
      "rewards/rejected": -24.375,
      "step": 3940
    },
    {
      "epoch": 0.5048245894306346,
      "grad_norm": 6.952525073057976,
      "learning_rate": 1.272891654681168e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.8515625,
      "logps/chosen": -378.0,
      "logps/rejected": -426.0,
      "loss": 0.1468,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -17.125,
      "rewards/margins": 6.71875,
      "rewards/rejected": -23.875,
      "step": 3950
    },
    {
      "epoch": 0.506102626365902,
      "grad_norm": 5.295328329343153,
      "learning_rate": 1.2712834523274563e-06,
      "logits/chosen": -1.3984375,
      "logits/rejected": -0.89453125,
      "logps/chosen": -372.0,
      "logps/rejected": -434.0,
      "loss": 0.1345,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.53125,
      "rewards/rejected": -23.875,
      "step": 3960
    },
    {
      "epoch": 0.5073806633011694,
      "grad_norm": 10.169118805733405,
      "learning_rate": 1.2696813301379032e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.828125,
      "logps/chosen": -378.0,
      "logps/rejected": -422.0,
      "loss": 0.1449,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -16.875,
      "rewards/margins": 7.5,
      "rewards/rejected": -24.375,
      "step": 3970
    },
    {
      "epoch": 0.5086587002364368,
      "grad_norm": 7.809367609321747,
      "learning_rate": 1.2680852498964829e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.84765625,
      "logps/chosen": -368.0,
      "logps/rejected": -424.0,
      "loss": 0.1564,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.625,
      "rewards/margins": 7.53125,
      "rewards/rejected": -24.25,
      "step": 3980
    },
    {
      "epoch": 0.5099367371717043,
      "grad_norm": 5.337338997488873,
      "learning_rate": 1.266495173722607e-06,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.86328125,
      "logps/chosen": -360.0,
      "logps/rejected": -426.0,
      "loss": 0.1521,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.15625,
      "rewards/rejected": -23.5,
      "step": 3990
    },
    {
      "epoch": 0.5112147741069717,
      "grad_norm": 6.18336735157534,
      "learning_rate": 1.2649110640673517e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.87109375,
      "logps/chosen": -360.0,
      "logps/rejected": -394.0,
      "loss": 0.1266,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -15.75,
      "rewards/margins": 6.4375,
      "rewards/rejected": -22.25,
      "step": 4000
    },
    {
      "epoch": 0.5124928110422391,
      "grad_norm": 5.715217290971865,
      "learning_rate": 1.2633328837097308e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.83203125,
      "logps/chosen": -386.0,
      "logps/rejected": -438.0,
      "loss": 0.1193,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.75,
      "rewards/margins": 6.875,
      "rewards/rejected": -23.625,
      "step": 4010
    },
    {
      "epoch": 0.5137708479775065,
      "grad_norm": 7.66224001452475,
      "learning_rate": 1.2617605957530233e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -1.015625,
      "logps/chosen": -382.0,
      "logps/rejected": -424.0,
      "loss": 0.1279,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -17.0,
      "rewards/margins": 6.84375,
      "rewards/rejected": -23.875,
      "step": 4020
    },
    {
      "epoch": 0.515048884912774,
      "grad_norm": 7.035786768866535,
      "learning_rate": 1.2601941636211516e-06,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.921875,
      "logps/chosen": -396.0,
      "logps/rejected": -434.0,
      "loss": 0.1265,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.40625,
      "rewards/rejected": -23.875,
      "step": 4030
    },
    {
      "epoch": 0.5163269218480414,
      "grad_norm": 8.467120293146044,
      "learning_rate": 1.2586335510551052e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.94140625,
      "logps/chosen": -376.0,
      "logps/rejected": -422.0,
      "loss": 0.1493,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.9375,
      "rewards/margins": 6.8125,
      "rewards/rejected": -22.75,
      "step": 4040
    },
    {
      "epoch": 0.5176049587833088,
      "grad_norm": 7.225608426572022,
      "learning_rate": 1.2570787221094177e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.94921875,
      "logps/chosen": -382.0,
      "logps/rejected": -442.0,
      "loss": 0.154,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.3125,
      "rewards/margins": 7.96875,
      "rewards/rejected": -23.25,
      "step": 4050
    },
    {
      "epoch": 0.5188829957185762,
      "grad_norm": 8.960330110133077,
      "learning_rate": 1.255529641148689e-06,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.9375,
      "logps/chosen": -374.0,
      "logps/rejected": -412.0,
      "loss": 0.1613,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.5,
      "rewards/margins": 6.6875,
      "rewards/rejected": -22.25,
      "step": 4060
    },
    {
      "epoch": 0.5201610326538437,
      "grad_norm": 7.587345693895239,
      "learning_rate": 1.2539862728441536e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.7890625,
      "logps/chosen": -364.0,
      "logps/rejected": -418.0,
      "loss": 0.1587,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.8125,
      "rewards/margins": 6.625,
      "rewards/rejected": -22.375,
      "step": 4070
    },
    {
      "epoch": 0.5214390695891111,
      "grad_norm": 8.07316893337428,
      "learning_rate": 1.252448582170299e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.8203125,
      "logps/chosen": -372.0,
      "logps/rejected": -410.0,
      "loss": 0.1059,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -15.4375,
      "rewards/margins": 6.21875,
      "rewards/rejected": -21.625,
      "step": 4080
    },
    {
      "epoch": 0.5227171065243785,
      "grad_norm": 8.777795841233306,
      "learning_rate": 1.2509165344015243e-06,
      "logits/chosen": -1.375,
      "logits/rejected": -0.87890625,
      "logps/chosen": -384.0,
      "logps/rejected": -424.0,
      "loss": 0.1194,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.25,
      "rewards/margins": 7.5625,
      "rewards/rejected": -22.75,
      "step": 4090
    },
    {
      "epoch": 0.5239951434596459,
      "grad_norm": 7.376492220186144,
      "learning_rate": 1.2493900951088486e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.90625,
      "logps/chosen": -372.0,
      "logps/rejected": -410.0,
      "loss": 0.1373,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.1875,
      "rewards/margins": 7.28125,
      "rewards/rejected": -22.5,
      "step": 4100
    },
    {
      "epoch": 0.5252731803949134,
      "grad_norm": 4.836175661802613,
      "learning_rate": 1.2478692301566601e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.83984375,
      "logps/chosen": -376.0,
      "logps/rejected": -444.0,
      "loss": 0.1361,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -15.4375,
      "rewards/margins": 7.5,
      "rewards/rejected": -23.0,
      "step": 4110
    },
    {
      "epoch": 0.5265512173301808,
      "grad_norm": 2.1912168019876788,
      "learning_rate": 1.2463539056995116e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.78515625,
      "logps/chosen": -362.0,
      "logps/rejected": -424.0,
      "loss": 0.1268,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.25,
      "rewards/margins": 6.96875,
      "rewards/rejected": -23.25,
      "step": 4120
    },
    {
      "epoch": 0.5278292542654482,
      "grad_norm": 5.33529847776421,
      "learning_rate": 1.2448440881789541e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.94921875,
      "logps/chosen": -396.0,
      "logps/rejected": -452.0,
      "loss": 0.1343,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.9375,
      "rewards/margins": 8.3125,
      "rewards/rejected": -24.25,
      "step": 4130
    },
    {
      "epoch": 0.5291072912007156,
      "grad_norm": 5.91122985473326,
      "learning_rate": 1.2433397443204184e-06,
      "logits/chosen": -1.40625,
      "logits/rejected": -0.91796875,
      "logps/chosen": -388.0,
      "logps/rejected": -452.0,
      "loss": 0.1366,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.59375,
      "rewards/rejected": -24.375,
      "step": 4140
    },
    {
      "epoch": 0.5303853281359832,
      "grad_norm": 7.954321209531519,
      "learning_rate": 1.2418408411301324e-06,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.88671875,
      "logps/chosen": -372.0,
      "logps/rejected": -428.0,
      "loss": 0.152,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -17.625,
      "rewards/margins": 6.34375,
      "rewards/rejected": -23.875,
      "step": 4150
    },
    {
      "epoch": 0.5316633650712506,
      "grad_norm": 6.267659350329792,
      "learning_rate": 1.2403473458920844e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.80078125,
      "logps/chosen": -358.0,
      "logps/rejected": -406.0,
      "loss": 0.1392,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.3125,
      "rewards/margins": 7.5,
      "rewards/rejected": -22.75,
      "step": 4160
    },
    {
      "epoch": 0.532941402006518,
      "grad_norm": 4.055486021177046,
      "learning_rate": 1.2388592261650217e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.80859375,
      "logps/chosen": -362.0,
      "logps/rejected": -396.0,
      "loss": 0.1079,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.875,
      "rewards/margins": 6.875,
      "rewards/rejected": -22.75,
      "step": 4170
    },
    {
      "epoch": 0.5342194389417854,
      "grad_norm": 7.2619506906943405,
      "learning_rate": 1.2373764497794918e-06,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.03125,
      "logps/chosen": -390.0,
      "logps/rejected": -436.0,
      "loss": 0.1461,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.0,
      "rewards/margins": 6.875,
      "rewards/rejected": -23.875,
      "step": 4180
    },
    {
      "epoch": 0.5354974758770529,
      "grad_norm": 5.364257727612288,
      "learning_rate": 1.2358989848349217e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.91796875,
      "logps/chosen": -366.0,
      "logps/rejected": -408.0,
      "loss": 0.1177,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.625,
      "rewards/margins": 6.71875,
      "rewards/rejected": -23.375,
      "step": 4190
    },
    {
      "epoch": 0.5367755128123203,
      "grad_norm": 6.295173920747968,
      "learning_rate": 1.2344267996967353e-06,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.0234375,
      "logps/chosen": -382.0,
      "logps/rejected": -448.0,
      "loss": 0.1309,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -17.25,
      "rewards/margins": 7.5625,
      "rewards/rejected": -24.875,
      "step": 4200
    },
    {
      "epoch": 0.5380535497475877,
      "grad_norm": 5.594792386133589,
      "learning_rate": 1.2329598629935076e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.83203125,
      "logps/chosen": -382.0,
      "logps/rejected": -428.0,
      "loss": 0.1262,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.21875,
      "rewards/rejected": -23.5,
      "step": 4210
    },
    {
      "epoch": 0.5393315866828551,
      "grad_norm": 7.589837756651567,
      "learning_rate": 1.2314981436141583e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.8828125,
      "logps/chosen": -366.0,
      "logps/rejected": -420.0,
      "loss": 0.1175,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.5,
      "rewards/rejected": -23.625,
      "step": 4220
    },
    {
      "epoch": 0.5406096236181226,
      "grad_norm": 8.069257213461738,
      "learning_rate": 1.2300416107051802e-06,
      "logits/chosen": -1.3828125,
      "logits/rejected": -0.8984375,
      "logps/chosen": -362.0,
      "logps/rejected": -436.0,
      "loss": 0.1436,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.8125,
      "rewards/margins": 7.0625,
      "rewards/rejected": -22.875,
      "step": 4230
    },
    {
      "epoch": 0.54188766055339,
      "grad_norm": 8.822401069484748,
      "learning_rate": 1.2285902336679024e-06,
      "logits/chosen": -1.4140625,
      "logits/rejected": -0.8671875,
      "logps/chosen": -368.0,
      "logps/rejected": -422.0,
      "loss": 0.1044,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.25,
      "rewards/margins": 6.8125,
      "rewards/rejected": -23.125,
      "step": 4240
    },
    {
      "epoch": 0.5431656974886574,
      "grad_norm": 9.687616141577971,
      "learning_rate": 1.2271439821557926e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.8984375,
      "logps/chosen": -364.0,
      "logps/rejected": -414.0,
      "loss": 0.1207,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.8125,
      "rewards/margins": 7.21875,
      "rewards/rejected": -23.0,
      "step": 4250
    },
    {
      "epoch": 0.5444437344239248,
      "grad_norm": 8.622642326345176,
      "learning_rate": 1.225702826071791e-06,
      "logits/chosen": -1.46875,
      "logits/rejected": -0.96875,
      "logps/chosen": -374.0,
      "logps/rejected": -430.0,
      "loss": 0.1447,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -16.0,
      "rewards/margins": 7.46875,
      "rewards/rejected": -23.5,
      "step": 4260
    },
    {
      "epoch": 0.5457217713591923,
      "grad_norm": 7.855367178541955,
      "learning_rate": 1.2242667355656797e-06,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.9296875,
      "logps/chosen": -378.0,
      "logps/rejected": -430.0,
      "loss": 0.1363,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.125,
      "rewards/margins": 7.78125,
      "rewards/rejected": -22.875,
      "step": 4270
    },
    {
      "epoch": 0.5469998082944597,
      "grad_norm": 7.698289553789727,
      "learning_rate": 1.2228356810314862e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.9453125,
      "logps/chosen": -362.0,
      "logps/rejected": -418.0,
      "loss": 0.1492,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -15.375,
      "rewards/margins": 6.71875,
      "rewards/rejected": -22.125,
      "step": 4280
    },
    {
      "epoch": 0.5482778452297271,
      "grad_norm": 8.210918110223535,
      "learning_rate": 1.2214096331049186e-06,
      "logits/chosen": -1.3515625,
      "logits/rejected": -0.8515625,
      "logps/chosen": -356.0,
      "logps/rejected": -410.0,
      "loss": 0.1238,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -13.8125,
      "rewards/margins": 7.59375,
      "rewards/rejected": -21.375,
      "step": 4290
    },
    {
      "epoch": 0.5495558821649945,
      "grad_norm": 7.80225558271295,
      "learning_rate": 1.2199885626608373e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.87109375,
      "logps/chosen": -352.0,
      "logps/rejected": -412.0,
      "loss": 0.1355,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -15.0,
      "rewards/margins": 7.875,
      "rewards/rejected": -22.875,
      "step": 4300
    },
    {
      "epoch": 0.550833919100262,
      "grad_norm": 7.396685946554367,
      "learning_rate": 1.2185724408107546e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.95703125,
      "logps/chosen": -364.0,
      "logps/rejected": -426.0,
      "loss": 0.1444,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -15.1875,
      "rewards/margins": 7.21875,
      "rewards/rejected": -22.375,
      "step": 4310
    },
    {
      "epoch": 0.5521119560355294,
      "grad_norm": 8.038504027446018,
      "learning_rate": 1.2171612389003689e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.8125,
      "logps/chosen": -356.0,
      "logps/rejected": -426.0,
      "loss": 0.1315,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.5,
      "rewards/margins": 7.3125,
      "rewards/rejected": -22.75,
      "step": 4320
    },
    {
      "epoch": 0.5533899929707968,
      "grad_norm": 8.338061461024187,
      "learning_rate": 1.2157549285071297e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.7578125,
      "logps/chosen": -376.0,
      "logps/rejected": -416.0,
      "loss": 0.1104,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.375,
      "rewards/margins": 6.59375,
      "rewards/rejected": -22.875,
      "step": 4330
    },
    {
      "epoch": 0.5546680299060642,
      "grad_norm": 8.296028800610419,
      "learning_rate": 1.2143534814378327e-06,
      "logits/chosen": -1.3671875,
      "logits/rejected": -0.94921875,
      "logps/chosen": -382.0,
      "logps/rejected": -424.0,
      "loss": 0.1246,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.25,
      "rewards/margins": 6.53125,
      "rewards/rejected": -22.75,
      "step": 4340
    },
    {
      "epoch": 0.5559460668413317,
      "grad_norm": 5.382077484968552,
      "learning_rate": 1.2129568697262454e-06,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.89453125,
      "logps/chosen": -372.0,
      "logps/rejected": -428.0,
      "loss": 0.1153,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.625,
      "rewards/margins": 7.84375,
      "rewards/rejected": -23.5,
      "step": 4350
    },
    {
      "epoch": 0.5572241037765991,
      "grad_norm": 5.448804011826712,
      "learning_rate": 1.2115650656307653e-06,
      "logits/chosen": -1.3828125,
      "logits/rejected": -0.96875,
      "logps/chosen": -382.0,
      "logps/rejected": -402.0,
      "loss": 0.1119,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -15.875,
      "rewards/margins": 7.46875,
      "rewards/rejected": -23.375,
      "step": 4360
    },
    {
      "epoch": 0.5585021407118665,
      "grad_norm": 7.8554182191636786,
      "learning_rate": 1.210178041632103e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.8125,
      "logps/chosen": -372.0,
      "logps/rejected": -444.0,
      "loss": 0.1383,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.625,
      "rewards/margins": 7.625,
      "rewards/rejected": -24.25,
      "step": 4370
    },
    {
      "epoch": 0.559780177647134,
      "grad_norm": 6.195537947423835,
      "learning_rate": 1.2087957704309988e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.96875,
      "logps/chosen": -384.0,
      "logps/rejected": -410.0,
      "loss": 0.1211,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.5,
      "rewards/margins": 6.90625,
      "rewards/rejected": -23.375,
      "step": 4380
    },
    {
      "epoch": 0.5610582145824015,
      "grad_norm": 9.813903798949418,
      "learning_rate": 1.2074182249459642e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.8671875,
      "logps/chosen": -374.0,
      "logps/rejected": -402.0,
      "loss": 0.1463,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.8125,
      "rewards/margins": 7.375,
      "rewards/rejected": -23.25,
      "step": 4390
    },
    {
      "epoch": 0.5623362515176689,
      "grad_norm": 6.682128234316747,
      "learning_rate": 1.2060453783110545e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.9140625,
      "logps/chosen": -372.0,
      "logps/rejected": -408.0,
      "loss": 0.1298,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -16.375,
      "rewards/margins": 6.90625,
      "rewards/rejected": -23.25,
      "step": 4400
    },
    {
      "epoch": 0.5636142884529363,
      "grad_norm": 6.79332680051349,
      "learning_rate": 1.2046772038736682e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.89453125,
      "logps/chosen": -364.0,
      "logps/rejected": -396.0,
      "loss": 0.1107,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -15.625,
      "rewards/margins": 7.125,
      "rewards/rejected": -22.75,
      "step": 4410
    },
    {
      "epoch": 0.5648923253882037,
      "grad_norm": 4.435439224632074,
      "learning_rate": 1.2033136751923736e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.9140625,
      "logps/chosen": -384.0,
      "logps/rejected": -450.0,
      "loss": 0.1369,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.40625,
      "rewards/rejected": -23.625,
      "step": 4420
    },
    {
      "epoch": 0.5661703623234712,
      "grad_norm": 5.842652695966885,
      "learning_rate": 1.201954766034762e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.875,
      "logps/chosen": -382.0,
      "logps/rejected": -430.0,
      "loss": 0.154,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.3125,
      "rewards/rejected": -23.625,
      "step": 4430
    },
    {
      "epoch": 0.5674483992587386,
      "grad_norm": 6.872268677684548,
      "learning_rate": 1.2006004503753285e-06,
      "logits/chosen": -1.390625,
      "logits/rejected": -0.9453125,
      "logps/chosen": -362.0,
      "logps/rejected": -448.0,
      "loss": 0.1266,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.25,
      "rewards/margins": 6.6875,
      "rewards/rejected": -24.0,
      "step": 4440
    },
    {
      "epoch": 0.568726436194006,
      "grad_norm": 9.386572790503253,
      "learning_rate": 1.1992507023933782e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.98046875,
      "logps/chosen": -404.0,
      "logps/rejected": -428.0,
      "loss": 0.1381,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -16.875,
      "rewards/margins": 6.8125,
      "rewards/rejected": -23.625,
      "step": 4450
    },
    {
      "epoch": 0.5700044731292734,
      "grad_norm": 6.2635119973685365,
      "learning_rate": 1.1979054964709597e-06,
      "logits/chosen": -1.375,
      "logits/rejected": -0.9375,
      "logps/chosen": -394.0,
      "logps/rejected": -444.0,
      "loss": 0.1305,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.75,
      "rewards/rejected": -24.125,
      "step": 4460
    },
    {
      "epoch": 0.5712825100645409,
      "grad_norm": 7.249791670394813,
      "learning_rate": 1.1965648071908207e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.91796875,
      "logps/chosen": -376.0,
      "logps/rejected": -428.0,
      "loss": 0.1144,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.25,
      "rewards/margins": 6.625,
      "rewards/rejected": -23.875,
      "step": 4470
    },
    {
      "epoch": 0.5725605469998083,
      "grad_norm": 5.517251771633542,
      "learning_rate": 1.1952286093343935e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.87109375,
      "logps/chosen": -380.0,
      "logps/rejected": -426.0,
      "loss": 0.1166,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.5,
      "rewards/rejected": -24.0,
      "step": 4480
    },
    {
      "epoch": 0.5738385839350757,
      "grad_norm": 8.237839278967805,
      "learning_rate": 1.1938968778798005e-06,
      "logits/chosen": -1.3671875,
      "logits/rejected": -0.98828125,
      "logps/chosen": -396.0,
      "logps/rejected": -446.0,
      "loss": 0.1366,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -16.875,
      "rewards/margins": 7.15625,
      "rewards/rejected": -24.0,
      "step": 4490
    },
    {
      "epoch": 0.5751166208703431,
      "grad_norm": 6.961810325293726,
      "learning_rate": 1.1925695879998878e-06,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.87890625,
      "logps/chosen": -400.0,
      "logps/rejected": -458.0,
      "loss": 0.1243,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.625,
      "rewards/margins": 7.625,
      "rewards/rejected": -25.25,
      "step": 4500
    },
    {
      "epoch": 0.5763946578056106,
      "grad_norm": 8.544441333180373,
      "learning_rate": 1.1912467150602794e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.91796875,
      "logps/chosen": -370.0,
      "logps/rejected": -452.0,
      "loss": 0.1221,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.0,
      "rewards/margins": 8.5,
      "rewards/rejected": -25.5,
      "step": 4510
    },
    {
      "epoch": 0.577672694740878,
      "grad_norm": 6.737115879916741,
      "learning_rate": 1.189928234617459e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.8125,
      "logps/chosen": -376.0,
      "logps/rejected": -418.0,
      "loss": 0.1214,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -16.875,
      "rewards/margins": 7.5,
      "rewards/rejected": -24.375,
      "step": 4520
    },
    {
      "epoch": 0.5789507316761454,
      "grad_norm": 5.55086970039351,
      "learning_rate": 1.1886141224168716e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.83984375,
      "logps/chosen": -388.0,
      "logps/rejected": -438.0,
      "loss": 0.1189,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.0,
      "rewards/margins": 7.8125,
      "rewards/rejected": -24.75,
      "step": 4530
    },
    {
      "epoch": 0.5802287686114128,
      "grad_norm": 6.35245930526078,
      "learning_rate": 1.1873043543910495e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.8515625,
      "logps/chosen": -380.0,
      "logps/rejected": -434.0,
      "loss": 0.1122,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.4375,
      "rewards/rejected": -23.875,
      "step": 4540
    },
    {
      "epoch": 0.5815068055466803,
      "grad_norm": 8.02396984404215,
      "learning_rate": 1.1859989066577617e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.8046875,
      "logps/chosen": -372.0,
      "logps/rejected": -438.0,
      "loss": 0.1272,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -15.375,
      "rewards/margins": 7.875,
      "rewards/rejected": -23.25,
      "step": 4550
    },
    {
      "epoch": 0.5827848424819477,
      "grad_norm": 6.07141478524121,
      "learning_rate": 1.1846977555181846e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.86328125,
      "logps/chosen": -378.0,
      "logps/rejected": -432.0,
      "loss": 0.133,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.46875,
      "rewards/rejected": -23.625,
      "step": 4560
    },
    {
      "epoch": 0.5840628794172151,
      "grad_norm": 6.6295475948572955,
      "learning_rate": 1.1834008774550946e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.828125,
      "logps/chosen": -376.0,
      "logps/rejected": -422.0,
      "loss": 0.1233,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.6875,
      "rewards/margins": 6.90625,
      "rewards/rejected": -22.625,
      "step": 4570
    },
    {
      "epoch": 0.5853409163524825,
      "grad_norm": 8.053365421425838,
      "learning_rate": 1.1821082491310835e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.84765625,
      "logps/chosen": -370.0,
      "logps/rejected": -424.0,
      "loss": 0.1168,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.875,
      "rewards/rejected": -24.0,
      "step": 4580
    },
    {
      "epoch": 0.58661895328775,
      "grad_norm": 5.728951868806657,
      "learning_rate": 1.1808198473867937e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.93359375,
      "logps/chosen": -384.0,
      "logps/rejected": -420.0,
      "loss": 0.1303,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.46875,
      "rewards/rejected": -24.0,
      "step": 4590
    },
    {
      "epoch": 0.5878969902230174,
      "grad_norm": 6.227974143571599,
      "learning_rate": 1.179535649239177e-06,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.875,
      "logps/chosen": -366.0,
      "logps/rejected": -426.0,
      "loss": 0.1122,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.0,
      "rewards/margins": 8.375,
      "rewards/rejected": -23.375,
      "step": 4600
    },
    {
      "epoch": 0.5891750271582848,
      "grad_norm": 7.386140212898927,
      "learning_rate": 1.178255631879771e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.8125,
      "logps/chosen": -364.0,
      "logps/rejected": -404.0,
      "loss": 0.1071,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.25,
      "rewards/margins": 7.4375,
      "rewards/rejected": -22.75,
      "step": 4610
    },
    {
      "epoch": 0.5904530640935522,
      "grad_norm": 4.967226503717122,
      "learning_rate": 1.1769797726729992e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.875,
      "logps/chosen": -374.0,
      "logps/rejected": -426.0,
      "loss": 0.1057,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.25,
      "rewards/margins": 7.3125,
      "rewards/rejected": -22.5,
      "step": 4620
    },
    {
      "epoch": 0.5917311010288198,
      "grad_norm": 6.780674250637313,
      "learning_rate": 1.1757080491544881e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.8515625,
      "logps/chosen": -376.0,
      "logps/rejected": -440.0,
      "loss": 0.1051,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.5625,
      "rewards/margins": 7.90625,
      "rewards/rejected": -23.5,
      "step": 4630
    },
    {
      "epoch": 0.5930091379640872,
      "grad_norm": 10.081960340436574,
      "learning_rate": 1.1744404390294068e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.9296875,
      "logps/chosen": -376.0,
      "logps/rejected": -420.0,
      "loss": 0.1117,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -15.4375,
      "rewards/margins": 6.65625,
      "rewards/rejected": -22.125,
      "step": 4640
    },
    {
      "epoch": 0.5942871748993546,
      "grad_norm": 5.261421488836751,
      "learning_rate": 1.1731769201708264e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.97265625,
      "logps/chosen": -352.0,
      "logps/rejected": -408.0,
      "loss": 0.1358,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -15.1875,
      "rewards/margins": 7.375,
      "rewards/rejected": -22.625,
      "step": 4650
    },
    {
      "epoch": 0.595565211834622,
      "grad_norm": 7.074169827324963,
      "learning_rate": 1.1719174706180952e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.89453125,
      "logps/chosen": -382.0,
      "logps/rejected": -420.0,
      "loss": 0.1378,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.6875,
      "rewards/margins": 7.625,
      "rewards/rejected": -23.25,
      "step": 4660
    },
    {
      "epoch": 0.5968432487698895,
      "grad_norm": 6.771768143397207,
      "learning_rate": 1.1706620685752386e-06,
      "logits/chosen": -1.375,
      "logits/rejected": -0.93359375,
      "logps/chosen": -364.0,
      "logps/rejected": -412.0,
      "loss": 0.1255,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.3125,
      "rewards/margins": 7.625,
      "rewards/rejected": -22.875,
      "step": 4670
    },
    {
      "epoch": 0.5981212857051569,
      "grad_norm": 4.382119608637236,
      "learning_rate": 1.1694106924093723e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.90234375,
      "logps/chosen": -360.0,
      "logps/rejected": -412.0,
      "loss": 0.1079,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.625,
      "rewards/margins": 8.1875,
      "rewards/rejected": -23.875,
      "step": 4680
    },
    {
      "epoch": 0.5993993226404243,
      "grad_norm": 11.777896364238343,
      "learning_rate": 1.1681633206491381e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.91796875,
      "logps/chosen": -378.0,
      "logps/rejected": -422.0,
      "loss": 0.1312,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.5,
      "rewards/margins": 8.1875,
      "rewards/rejected": -24.625,
      "step": 4690
    },
    {
      "epoch": 0.6006773595756917,
      "grad_norm": 5.183927716766896,
      "learning_rate": 1.1669199319831564e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.86328125,
      "logps/chosen": -380.0,
      "logps/rejected": -418.0,
      "loss": 0.1419,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.53125,
      "rewards/rejected": -24.0,
      "step": 4700
    },
    {
      "epoch": 0.6019553965109592,
      "grad_norm": 7.652295457074283,
      "learning_rate": 1.1656805052584958e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.74609375,
      "logps/chosen": -380.0,
      "logps/rejected": -420.0,
      "loss": 0.1177,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.0,
      "rewards/margins": 7.4375,
      "rewards/rejected": -23.5,
      "step": 4710
    },
    {
      "epoch": 0.6032334334462266,
      "grad_norm": 3.0566699247987947,
      "learning_rate": 1.164445019479164e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.8203125,
      "logps/chosen": -360.0,
      "logps/rejected": -404.0,
      "loss": 0.0978,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.5625,
      "rewards/margins": 7.75,
      "rewards/rejected": -22.375,
      "step": 4720
    },
    {
      "epoch": 0.604511470381494,
      "grad_norm": 6.947497645801134,
      "learning_rate": 1.1632134538046105e-06,
      "logits/chosen": -1.3515625,
      "logits/rejected": -0.875,
      "logps/chosen": -340.0,
      "logps/rejected": -408.0,
      "loss": 0.1294,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -14.8125,
      "rewards/margins": 6.75,
      "rewards/rejected": -21.5,
      "step": 4730
    },
    {
      "epoch": 0.6057895073167614,
      "grad_norm": 10.119691553753803,
      "learning_rate": 1.1619857875482536e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.7890625,
      "logps/chosen": -352.0,
      "logps/rejected": -392.0,
      "loss": 0.0962,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -14.9375,
      "rewards/margins": 7.46875,
      "rewards/rejected": -22.375,
      "step": 4740
    },
    {
      "epoch": 0.6070675442520289,
      "grad_norm": 6.481295494104017,
      "learning_rate": 1.1607620001760185e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.9375,
      "logps/chosen": -356.0,
      "logps/rejected": -398.0,
      "loss": 0.1354,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.0,
      "rewards/rejected": -23.125,
      "step": 4750
    },
    {
      "epoch": 0.6083455811872963,
      "grad_norm": 6.8447858042522745,
      "learning_rate": 1.1595420713048968e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.98828125,
      "logps/chosen": -380.0,
      "logps/rejected": -408.0,
      "loss": 0.1327,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.6875,
      "rewards/margins": 7.46875,
      "rewards/rejected": -23.125,
      "step": 4760
    },
    {
      "epoch": 0.6096236181225637,
      "grad_norm": 3.5043562217998687,
      "learning_rate": 1.1583259807015182e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.8359375,
      "logps/chosen": -372.0,
      "logps/rejected": -420.0,
      "loss": 0.1081,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.59375,
      "rewards/rejected": -23.625,
      "step": 4770
    },
    {
      "epoch": 0.6109016550578311,
      "grad_norm": 5.579227140790423,
      "learning_rate": 1.1571137082807434e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -1.0078125,
      "logps/chosen": -380.0,
      "logps/rejected": -410.0,
      "loss": 0.1194,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.03125,
      "rewards/rejected": -23.25,
      "step": 4780
    },
    {
      "epoch": 0.6121796919930986,
      "grad_norm": 5.444055418812265,
      "learning_rate": 1.155905234104269e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.74609375,
      "logps/chosen": -368.0,
      "logps/rejected": -432.0,
      "loss": 0.1486,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -15.875,
      "rewards/margins": 8.0,
      "rewards/rejected": -23.875,
      "step": 4790
    },
    {
      "epoch": 0.613457728928366,
      "grad_norm": 9.264921390379444,
      "learning_rate": 1.1547005383792516e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.82421875,
      "logps/chosen": -364.0,
      "logps/rejected": -420.0,
      "loss": 0.1062,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.125,
      "rewards/margins": 7.53125,
      "rewards/rejected": -22.625,
      "step": 4800
    },
    {
      "epoch": 0.6147357658636334,
      "grad_norm": 8.657680133145305,
      "learning_rate": 1.1534996014569446e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.81640625,
      "logps/chosen": -354.0,
      "logps/rejected": -410.0,
      "loss": 0.1392,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.9375,
      "rewards/margins": 7.46875,
      "rewards/rejected": -22.375,
      "step": 4810
    },
    {
      "epoch": 0.6160138027989008,
      "grad_norm": 6.488465026619422,
      "learning_rate": 1.1523024038313547e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.8359375,
      "logps/chosen": -356.0,
      "logps/rejected": -410.0,
      "loss": 0.1034,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -14.375,
      "rewards/margins": 8.4375,
      "rewards/rejected": -22.75,
      "step": 4820
    },
    {
      "epoch": 0.6172918397341683,
      "grad_norm": 5.119168492208587,
      "learning_rate": 1.1511089261379083e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.921875,
      "logps/chosen": -366.0,
      "logps/rejected": -420.0,
      "loss": 0.1369,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -14.3125,
      "rewards/margins": 8.3125,
      "rewards/rejected": -22.625,
      "step": 4830
    },
    {
      "epoch": 0.6185698766694357,
      "grad_norm": 6.227767107773455,
      "learning_rate": 1.149919149152138e-06,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.90234375,
      "logps/chosen": -366.0,
      "logps/rejected": -422.0,
      "loss": 0.1429,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.25,
      "rewards/margins": 7.125,
      "rewards/rejected": -21.375,
      "step": 4840
    },
    {
      "epoch": 0.6198479136047031,
      "grad_norm": 4.458115277184433,
      "learning_rate": 1.148733053788381e-06,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.80859375,
      "logps/chosen": -360.0,
      "logps/rejected": -404.0,
      "loss": 0.1287,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -14.9375,
      "rewards/margins": 7.4375,
      "rewards/rejected": -22.375,
      "step": 4850
    },
    {
      "epoch": 0.6211259505399706,
      "grad_norm": 8.119513853649673,
      "learning_rate": 1.1475506210984938e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.80078125,
      "logps/chosen": -356.0,
      "logps/rejected": -410.0,
      "loss": 0.1503,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -14.9375,
      "rewards/margins": 7.4375,
      "rewards/rejected": -22.375,
      "step": 4860
    },
    {
      "epoch": 0.6224039874752381,
      "grad_norm": 7.340814621972273,
      "learning_rate": 1.1463718322705807e-06,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.8515625,
      "logps/chosen": -364.0,
      "logps/rejected": -420.0,
      "loss": 0.107,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -14.625,
      "rewards/margins": 7.75,
      "rewards/rejected": -22.375,
      "step": 4870
    },
    {
      "epoch": 0.6236820244105055,
      "grad_norm": 4.340126429302955,
      "learning_rate": 1.1451966686277364e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.859375,
      "logps/chosen": -362.0,
      "logps/rejected": -424.0,
      "loss": 0.1257,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.5,
      "rewards/margins": 6.8125,
      "rewards/rejected": -22.375,
      "step": 4880
    },
    {
      "epoch": 0.6249600613457729,
      "grad_norm": 6.362814431540742,
      "learning_rate": 1.1440251116268034e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.8046875,
      "logps/chosen": -392.0,
      "logps/rejected": -442.0,
      "loss": 0.1502,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -16.375,
      "rewards/margins": 7.28125,
      "rewards/rejected": -23.625,
      "step": 4890
    },
    {
      "epoch": 0.6262380982810403,
      "grad_norm": 6.638988083662024,
      "learning_rate": 1.1428571428571428e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.98828125,
      "logps/chosen": -386.0,
      "logps/rejected": -432.0,
      "loss": 0.1016,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.0,
      "rewards/margins": 7.375,
      "rewards/rejected": -24.375,
      "step": 4900
    },
    {
      "epoch": 0.6275161352163078,
      "grad_norm": 5.498202281128867,
      "learning_rate": 1.14169274403942e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.79296875,
      "logps/chosen": -360.0,
      "logps/rejected": -424.0,
      "loss": 0.1214,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.0,
      "rewards/margins": 7.625,
      "rewards/rejected": -23.75,
      "step": 4910
    },
    {
      "epoch": 0.6287941721515752,
      "grad_norm": 12.825988995751318,
      "learning_rate": 1.140531897024402e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.8515625,
      "logps/chosen": -372.0,
      "logps/rejected": -430.0,
      "loss": 0.1227,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.375,
      "rewards/rejected": -23.5,
      "step": 4920
    },
    {
      "epoch": 0.6300722090868426,
      "grad_norm": 4.449920674717063,
      "learning_rate": 1.13937458379177e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.92578125,
      "logps/chosen": -366.0,
      "logps/rejected": -422.0,
      "loss": 0.1235,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.125,
      "rewards/margins": 7.625,
      "rewards/rejected": -22.75,
      "step": 4930
    },
    {
      "epoch": 0.63135024602211,
      "grad_norm": 6.865521262296344,
      "learning_rate": 1.1382207864489444e-06,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.875,
      "logps/chosen": -386.0,
      "logps/rejected": -412.0,
      "loss": 0.1216,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -15.75,
      "rewards/margins": 6.84375,
      "rewards/rejected": -22.625,
      "step": 4940
    },
    {
      "epoch": 0.6326282829573775,
      "grad_norm": 9.287739284268005,
      "learning_rate": 1.1370704872299223e-06,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.98046875,
      "logps/chosen": -372.0,
      "logps/rejected": -412.0,
      "loss": 0.1071,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.375,
      "rewards/margins": 6.6875,
      "rewards/rejected": -22.0,
      "step": 4950
    },
    {
      "epoch": 0.6339063198926449,
      "grad_norm": 6.534465166619105,
      "learning_rate": 1.1359236684941295e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.90234375,
      "logps/chosen": -376.0,
      "logps/rejected": -420.0,
      "loss": 0.1257,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.875,
      "rewards/margins": 7.25,
      "rewards/rejected": -23.125,
      "step": 4960
    },
    {
      "epoch": 0.6351843568279123,
      "grad_norm": 7.235171623819455,
      "learning_rate": 1.1347803127252839e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.90234375,
      "logps/chosen": -380.0,
      "logps/rejected": -426.0,
      "loss": 0.0894,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -15.5625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -22.625,
      "step": 4970
    },
    {
      "epoch": 0.6364623937631797,
      "grad_norm": 8.579088053132145,
      "learning_rate": 1.1336404025302715e-06,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.83203125,
      "logps/chosen": -376.0,
      "logps/rejected": -434.0,
      "loss": 0.1403,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.5625,
      "rewards/rejected": -23.875,
      "step": 4980
    },
    {
      "epoch": 0.6377404306984472,
      "grad_norm": 9.911891227367601,
      "learning_rate": 1.1325039206380352e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.8828125,
      "logps/chosen": -382.0,
      "logps/rejected": -432.0,
      "loss": 0.1589,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -16.75,
      "rewards/margins": 6.96875,
      "rewards/rejected": -23.75,
      "step": 4990
    },
    {
      "epoch": 0.6390184676337146,
      "grad_norm": 5.143274254916214,
      "learning_rate": 1.131370849898476e-06,
      "logits/chosen": -1.3515625,
      "logits/rejected": -0.98828125,
      "logps/chosen": -402.0,
      "logps/rejected": -450.0,
      "loss": 0.1252,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -17.5,
      "rewards/margins": 7.875,
      "rewards/rejected": -25.375,
      "step": 5000
    },
    {
      "epoch": 0.6390184676337146,
      "eval_logits/chosen": -1.28125,
      "eval_logits/rejected": -0.87109375,
      "eval_logps/chosen": -384.0,
      "eval_logps/rejected": -426.0,
      "eval_loss": 0.22941964864730835,
      "eval_rewards/accuracies": 0.9068260192871094,
      "eval_rewards/chosen": -17.375,
      "eval_rewards/margins": 6.09375,
      "eval_rewards/rejected": -23.5,
      "eval_runtime": 1569.0951,
      "eval_samples_per_second": 35.375,
      "eval_steps_per_second": 0.553,
      "step": 5000
    },
    {
      "epoch": 0.640296504568982,
      "grad_norm": 9.943365241192351,
      "learning_rate": 1.130241173281366e-06,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.8671875,
      "logps/chosen": -382.0,
      "logps/rejected": -436.0,
      "loss": 0.1218,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -17.125,
      "rewards/margins": 6.84375,
      "rewards/rejected": -24.0,
      "step": 5010
    },
    {
      "epoch": 0.6415745415042494,
      "grad_norm": 5.5600226095595975,
      "learning_rate": 1.1291148738752732e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.79296875,
      "logps/chosen": -372.0,
      "logps/rejected": -424.0,
      "loss": 0.1064,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.875,
      "rewards/margins": 6.53125,
      "rewards/rejected": -23.5,
      "step": 5020
    },
    {
      "epoch": 0.6428525784395169,
      "grad_norm": 6.070648025731513,
      "learning_rate": 1.1279919348864981e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.8984375,
      "logps/chosen": -364.0,
      "logps/rejected": -438.0,
      "loss": 0.1209,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.75,
      "rewards/margins": 7.84375,
      "rewards/rejected": -23.625,
      "step": 5030
    },
    {
      "epoch": 0.6441306153747843,
      "grad_norm": 7.747103812778883,
      "learning_rate": 1.126872339638022e-06,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.828125,
      "logps/chosen": -372.0,
      "logps/rejected": -434.0,
      "loss": 0.1189,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.09375,
      "rewards/rejected": -23.5,
      "step": 5040
    },
    {
      "epoch": 0.6454086523100517,
      "grad_norm": 7.172356311286467,
      "learning_rate": 1.1257560715684668e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.8828125,
      "logps/chosen": -368.0,
      "logps/rejected": -428.0,
      "loss": 0.1109,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.0,
      "rewards/rejected": -23.75,
      "step": 5050
    },
    {
      "epoch": 0.6466866892453191,
      "grad_norm": 6.508686210988629,
      "learning_rate": 1.1246431142310665e-06,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.875,
      "logps/chosen": -378.0,
      "logps/rejected": -418.0,
      "loss": 0.1434,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -15.875,
      "rewards/margins": 7.28125,
      "rewards/rejected": -23.125,
      "step": 5060
    },
    {
      "epoch": 0.6479647261805866,
      "grad_norm": 10.916084902055635,
      "learning_rate": 1.1235334512926484e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.796875,
      "logps/chosen": -372.0,
      "logps/rejected": -398.0,
      "loss": 0.1191,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -15.3125,
      "rewards/margins": 7.1875,
      "rewards/rejected": -22.5,
      "step": 5070
    },
    {
      "epoch": 0.649242763115854,
      "grad_norm": 5.94157907083879,
      "learning_rate": 1.1224270665326274e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.8671875,
      "logps/chosen": -384.0,
      "logps/rejected": -418.0,
      "loss": 0.1231,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -15.9375,
      "rewards/margins": 7.625,
      "rewards/rejected": -23.625,
      "step": 5080
    },
    {
      "epoch": 0.6505208000511215,
      "grad_norm": 8.666564367577438,
      "learning_rate": 1.12132394384201e-06,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.95703125,
      "logps/chosen": -388.0,
      "logps/rejected": -424.0,
      "loss": 0.1336,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.53125,
      "rewards/rejected": -23.625,
      "step": 5090
    },
    {
      "epoch": 0.6517988369863889,
      "grad_norm": 6.372772488162212,
      "learning_rate": 1.1202240672224076e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.87109375,
      "logps/chosen": -380.0,
      "logps/rejected": -446.0,
      "loss": 0.0935,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.3125,
      "rewards/rejected": -24.0,
      "step": 5100
    },
    {
      "epoch": 0.6530768739216564,
      "grad_norm": 4.1012834354852705,
      "learning_rate": 1.1191274207850654e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.78515625,
      "logps/chosen": -376.0,
      "logps/rejected": -432.0,
      "loss": 0.1152,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.125,
      "rewards/rejected": -23.375,
      "step": 5110
    },
    {
      "epoch": 0.6543549108569238,
      "grad_norm": 6.198635181242776,
      "learning_rate": 1.1180339887498948e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.890625,
      "logps/chosen": -386.0,
      "logps/rejected": -438.0,
      "loss": 0.127,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.625,
      "rewards/margins": 7.25,
      "rewards/rejected": -24.875,
      "step": 5120
    },
    {
      "epoch": 0.6556329477921912,
      "grad_norm": 7.825089870420605,
      "learning_rate": 1.1169437554445213e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.90625,
      "logps/chosen": -380.0,
      "logps/rejected": -442.0,
      "loss": 0.142,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.4375,
      "rewards/rejected": -23.75,
      "step": 5130
    },
    {
      "epoch": 0.6569109847274586,
      "grad_norm": 7.228872248409647,
      "learning_rate": 1.1158567053033413e-06,
      "logits/chosen": -1.34375,
      "logits/rejected": -1.0,
      "logps/chosen": -384.0,
      "logps/rejected": -436.0,
      "loss": 0.129,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.0,
      "rewards/rejected": -23.75,
      "step": 5140
    },
    {
      "epoch": 0.6581890216627261,
      "grad_norm": 9.508097749992194,
      "learning_rate": 1.1147728228665882e-06,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.9921875,
      "logps/chosen": -374.0,
      "logps/rejected": -410.0,
      "loss": 0.1306,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.625,
      "rewards/margins": 7.25,
      "rewards/rejected": -23.875,
      "step": 5150
    },
    {
      "epoch": 0.6594670585979935,
      "grad_norm": 5.068575819063334,
      "learning_rate": 1.1136920927794092e-06,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.828125,
      "logps/chosen": -356.0,
      "logps/rejected": -420.0,
      "loss": 0.1415,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.0,
      "rewards/margins": 7.71875,
      "rewards/rejected": -23.75,
      "step": 5160
    },
    {
      "epoch": 0.6607450955332609,
      "grad_norm": 6.657131810317916,
      "learning_rate": 1.1126144997909508e-06,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.96484375,
      "logps/chosen": -382.0,
      "logps/rejected": -416.0,
      "loss": 0.1348,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -16.125,
      "rewards/margins": 7.0,
      "rewards/rejected": -23.125,
      "step": 5170
    },
    {
      "epoch": 0.6620231324685283,
      "grad_norm": 6.368247988169162,
      "learning_rate": 1.1115400287534568e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.8046875,
      "logps/chosen": -370.0,
      "logps/rejected": -438.0,
      "loss": 0.1324,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.5625,
      "rewards/rejected": -24.125,
      "step": 5180
    },
    {
      "epoch": 0.6633011694037958,
      "grad_norm": 10.813531400650158,
      "learning_rate": 1.110468664621372e-06,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.98828125,
      "logps/chosen": -374.0,
      "logps/rejected": -442.0,
      "loss": 0.1475,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.59375,
      "rewards/rejected": -24.25,
      "step": 5190
    },
    {
      "epoch": 0.6645792063390632,
      "grad_norm": 8.148626983417193,
      "learning_rate": 1.1094003924504583e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.921875,
      "logps/chosen": -368.0,
      "logps/rejected": -438.0,
      "loss": 0.1001,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.125,
      "rewards/margins": 8.1875,
      "rewards/rejected": -24.375,
      "step": 5200
    },
    {
      "epoch": 0.6658572432743306,
      "grad_norm": 10.472498499286655,
      "learning_rate": 1.1083351973969191e-06,
      "logits/chosen": -1.375,
      "logits/rejected": -0.890625,
      "logps/chosen": -388.0,
      "logps/rejected": -444.0,
      "loss": 0.1551,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -16.625,
      "rewards/margins": 7.6875,
      "rewards/rejected": -24.375,
      "step": 5210
    },
    {
      "epoch": 0.667135280209598,
      "grad_norm": 7.818619333695082,
      "learning_rate": 1.107273064716533e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.8359375,
      "logps/chosen": -378.0,
      "logps/rejected": -428.0,
      "loss": 0.1195,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.625,
      "rewards/margins": 6.8125,
      "rewards/rejected": -24.375,
      "step": 5220
    },
    {
      "epoch": 0.6684133171448655,
      "grad_norm": 3.5139779640106052,
      "learning_rate": 1.1062139797637962e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.78515625,
      "logps/chosen": -390.0,
      "logps/rejected": -444.0,
      "loss": 0.0902,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.625,
      "rewards/margins": 7.84375,
      "rewards/rejected": -24.5,
      "step": 5230
    },
    {
      "epoch": 0.6696913540801329,
      "grad_norm": 6.097659714880926,
      "learning_rate": 1.1051579279910751e-06,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.77734375,
      "logps/chosen": -388.0,
      "logps/rejected": -450.0,
      "loss": 0.1511,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -18.0,
      "rewards/margins": 6.875,
      "rewards/rejected": -24.875,
      "step": 5240
    },
    {
      "epoch": 0.6709693910154003,
      "grad_norm": 10.40542713662468,
      "learning_rate": 1.1041048949477667e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.8125,
      "logps/chosen": -390.0,
      "logps/rejected": -448.0,
      "loss": 0.1169,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -18.625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -25.375,
      "step": 5250
    },
    {
      "epoch": 0.6722474279506677,
      "grad_norm": 5.232738816000125,
      "learning_rate": 1.1030548662794673e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.875,
      "logps/chosen": -424.0,
      "logps/rejected": -476.0,
      "loss": 0.0991,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.375,
      "rewards/margins": 7.75,
      "rewards/rejected": -26.25,
      "step": 5260
    },
    {
      "epoch": 0.6735254648859352,
      "grad_norm": 10.484186136231385,
      "learning_rate": 1.102007827727152e-06,
      "logits/chosen": -1.3984375,
      "logits/rejected": -0.890625,
      "logps/chosen": -386.0,
      "logps/rejected": -448.0,
      "loss": 0.124,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.875,
      "rewards/margins": 7.5,
      "rewards/rejected": -25.5,
      "step": 5270
    },
    {
      "epoch": 0.6748035018212026,
      "grad_norm": 2.649784687066309,
      "learning_rate": 1.1009637651263607e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.80859375,
      "logps/chosen": -400.0,
      "logps/rejected": -436.0,
      "loss": 0.1352,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.75,
      "rewards/margins": 7.84375,
      "rewards/rejected": -25.5,
      "step": 5280
    },
    {
      "epoch": 0.67608153875647,
      "grad_norm": 5.544441774892429,
      "learning_rate": 1.0999226644063927e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.88671875,
      "logps/chosen": -396.0,
      "logps/rejected": -438.0,
      "loss": 0.1072,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.625,
      "rewards/margins": 7.28125,
      "rewards/rejected": -24.875,
      "step": 5290
    },
    {
      "epoch": 0.6773595756917374,
      "grad_norm": 5.684042252826867,
      "learning_rate": 1.0988845115895123e-06,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.828125,
      "logps/chosen": -386.0,
      "logps/rejected": -446.0,
      "loss": 0.1116,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.25,
      "rewards/margins": 6.96875,
      "rewards/rejected": -25.125,
      "step": 5300
    },
    {
      "epoch": 0.678637612627005,
      "grad_norm": 7.3485638962020134,
      "learning_rate": 1.0978492927901574e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.828125,
      "logps/chosen": -384.0,
      "logps/rejected": -436.0,
      "loss": 0.085,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.875,
      "rewards/margins": 7.8125,
      "rewards/rejected": -24.75,
      "step": 5310
    },
    {
      "epoch": 0.6799156495622724,
      "grad_norm": 6.22879409212486,
      "learning_rate": 1.0968169942141634e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.91015625,
      "logps/chosen": -388.0,
      "logps/rejected": -440.0,
      "loss": 0.1145,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.375,
      "rewards/margins": 7.71875,
      "rewards/rejected": -25.0,
      "step": 5320
    },
    {
      "epoch": 0.6811936864975398,
      "grad_norm": 8.704470239559303,
      "learning_rate": 1.0957876021579874e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.84375,
      "logps/chosen": -392.0,
      "logps/rejected": -430.0,
      "loss": 0.1205,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.375,
      "rewards/margins": 8.375,
      "rewards/rejected": -25.75,
      "step": 5330
    },
    {
      "epoch": 0.6824717234328072,
      "grad_norm": 8.09551431803133,
      "learning_rate": 1.0947611030079466e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.92578125,
      "logps/chosen": -394.0,
      "logps/rejected": -448.0,
      "loss": 0.1057,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.875,
      "rewards/margins": 7.84375,
      "rewards/rejected": -25.75,
      "step": 5340
    },
    {
      "epoch": 0.6837497603680747,
      "grad_norm": 5.627771576357481,
      "learning_rate": 1.0937374832394612e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.9140625,
      "logps/chosen": -368.0,
      "logps/rejected": -438.0,
      "loss": 0.1008,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.0,
      "rewards/margins": 8.0,
      "rewards/rejected": -25.0,
      "step": 5350
    },
    {
      "epoch": 0.6850277973033421,
      "grad_norm": 7.2958473476530115,
      "learning_rate": 1.092716729416306e-06,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.96875,
      "logps/chosen": -416.0,
      "logps/rejected": -466.0,
      "loss": 0.1353,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -18.0,
      "rewards/margins": 7.8125,
      "rewards/rejected": -25.75,
      "step": 5360
    },
    {
      "epoch": 0.6863058342386095,
      "grad_norm": 6.879958981669206,
      "learning_rate": 1.0916988281898703e-06,
      "logits/chosen": -1.3515625,
      "logits/rejected": -0.92578125,
      "logps/chosen": -388.0,
      "logps/rejected": -452.0,
      "loss": 0.1188,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -18.0,
      "rewards/margins": 6.9375,
      "rewards/rejected": -25.0,
      "step": 5370
    },
    {
      "epoch": 0.6875838711738769,
      "grad_norm": 5.560935014074663,
      "learning_rate": 1.0906837662984237e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.875,
      "logps/chosen": -392.0,
      "logps/rejected": -450.0,
      "loss": 0.1131,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.375,
      "rewards/margins": 7.03125,
      "rewards/rejected": -25.375,
      "step": 5380
    },
    {
      "epoch": 0.6888619081091444,
      "grad_norm": 7.196720617368745,
      "learning_rate": 1.089671530566391e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.96484375,
      "logps/chosen": -392.0,
      "logps/rejected": -456.0,
      "loss": 0.1186,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -17.125,
      "rewards/margins": 6.875,
      "rewards/rejected": -24.0,
      "step": 5390
    },
    {
      "epoch": 0.6901399450444118,
      "grad_norm": 3.82674144762196,
      "learning_rate": 1.0886621079036346e-06,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.94140625,
      "logps/chosen": -402.0,
      "logps/rejected": -440.0,
      "loss": 0.0903,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.375,
      "rewards/margins": 8.9375,
      "rewards/rejected": -25.375,
      "step": 5400
    },
    {
      "epoch": 0.6914179819796792,
      "grad_norm": 2.709334023316459,
      "learning_rate": 1.0876554853047417e-06,
      "logits/chosen": -1.3515625,
      "logits/rejected": -0.86328125,
      "logps/chosen": -394.0,
      "logps/rejected": -438.0,
      "loss": 0.0868,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.25,
      "rewards/margins": 8.625,
      "rewards/rejected": -25.875,
      "step": 5410
    },
    {
      "epoch": 0.6926960189149466,
      "grad_norm": 6.198929878384152,
      "learning_rate": 1.0866516498483225e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.92578125,
      "logps/chosen": -388.0,
      "logps/rejected": -430.0,
      "loss": 0.121,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.375,
      "rewards/margins": 7.59375,
      "rewards/rejected": -25.0,
      "step": 5420
    },
    {
      "epoch": 0.6939740558502141,
      "grad_norm": 7.3085908096997345,
      "learning_rate": 1.0856505886963116e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.859375,
      "logps/chosen": -384.0,
      "logps/rejected": -432.0,
      "loss": 0.1184,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.875,
      "rewards/margins": 8.0,
      "rewards/rejected": -24.875,
      "step": 5430
    },
    {
      "epoch": 0.6952520927854815,
      "grad_norm": 7.035308830742478,
      "learning_rate": 1.0846522890932808e-06,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.8984375,
      "logps/chosen": -396.0,
      "logps/rejected": -456.0,
      "loss": 0.1252,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.25,
      "rewards/margins": 8.0,
      "rewards/rejected": -25.25,
      "step": 5440
    },
    {
      "epoch": 0.6965301297207489,
      "grad_norm": 2.912419518607702,
      "learning_rate": 1.0836567383657542e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.8125,
      "logps/chosen": -378.0,
      "logps/rejected": -450.0,
      "loss": 0.093,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.0,
      "rewards/margins": 7.875,
      "rewards/rejected": -25.875,
      "step": 5450
    },
    {
      "epoch": 0.6978081666560163,
      "grad_norm": 4.073401970008896,
      "learning_rate": 1.0826639239215334e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.87890625,
      "logps/chosen": -384.0,
      "logps/rejected": -438.0,
      "loss": 0.1085,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.0,
      "rewards/margins": 7.84375,
      "rewards/rejected": -25.75,
      "step": 5460
    },
    {
      "epoch": 0.6990862035912838,
      "grad_norm": 6.317297665100115,
      "learning_rate": 1.0816738332490292e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.8515625,
      "logps/chosen": -384.0,
      "logps/rejected": -440.0,
      "loss": 0.111,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -17.875,
      "rewards/margins": 8.3125,
      "rewards/rejected": -26.125,
      "step": 5470
    },
    {
      "epoch": 0.7003642405265512,
      "grad_norm": 10.562800338017448,
      "learning_rate": 1.0806864539165982e-06,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.70703125,
      "logps/chosen": -400.0,
      "logps/rejected": -458.0,
      "loss": 0.1303,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -18.5,
      "rewards/margins": 8.4375,
      "rewards/rejected": -27.0,
      "step": 5480
    },
    {
      "epoch": 0.7016422774618186,
      "grad_norm": 9.234530221360567,
      "learning_rate": 1.0797017735718878e-06,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.9296875,
      "logps/chosen": -398.0,
      "logps/rejected": -448.0,
      "loss": 0.1089,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -18.625,
      "rewards/margins": 8.0625,
      "rewards/rejected": -26.75,
      "step": 5490
    },
    {
      "epoch": 0.702920314397086,
      "grad_norm": 9.293946676950851,
      "learning_rate": 1.0787197799411874e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.84765625,
      "logps/chosen": -402.0,
      "logps/rejected": -458.0,
      "loss": 0.1271,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.375,
      "rewards/margins": 7.15625,
      "rewards/rejected": -26.5,
      "step": 5500
    },
    {
      "epoch": 0.7041983513323535,
      "grad_norm": 5.628605698272265,
      "learning_rate": 1.0777404608287846e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.82421875,
      "logps/chosen": -378.0,
      "logps/rejected": -448.0,
      "loss": 0.1126,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.0,
      "rewards/margins": 7.9375,
      "rewards/rejected": -26.875,
      "step": 5510
    },
    {
      "epoch": 0.7054763882676209,
      "grad_norm": 8.191597836266926,
      "learning_rate": 1.0767638041163309e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.90625,
      "logps/chosen": -400.0,
      "logps/rejected": -448.0,
      "loss": 0.1048,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.125,
      "rewards/margins": 8.5625,
      "rewards/rejected": -26.75,
      "step": 5520
    },
    {
      "epoch": 0.7067544252028883,
      "grad_norm": 5.485846761306661,
      "learning_rate": 1.0757897977622107e-06,
      "logits/chosen": -1.1796875,
      "logits/rejected": -0.859375,
      "logps/chosen": -396.0,
      "logps/rejected": -466.0,
      "loss": 0.1029,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.625,
      "rewards/margins": 7.53125,
      "rewards/rejected": -25.25,
      "step": 5530
    },
    {
      "epoch": 0.7080324621381557,
      "grad_norm": 4.03246849228728,
      "learning_rate": 1.074818429800918e-06,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.96875,
      "logps/chosen": -376.0,
      "logps/rejected": -444.0,
      "loss": 0.1044,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.5,
      "rewards/margins": 7.78125,
      "rewards/rejected": -25.25,
      "step": 5540
    },
    {
      "epoch": 0.7093104990734233,
      "grad_norm": 6.122379700331336,
      "learning_rate": 1.073849688342439e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.80859375,
      "logps/chosen": -380.0,
      "logps/rejected": -426.0,
      "loss": 0.1101,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.125,
      "rewards/margins": 7.84375,
      "rewards/rejected": -25.0,
      "step": 5550
    },
    {
      "epoch": 0.7105885360086907,
      "grad_norm": 3.517058024556956,
      "learning_rate": 1.07288356157164e-06,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.94921875,
      "logps/chosen": -404.0,
      "logps/rejected": -444.0,
      "loss": 0.0965,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.0,
      "rewards/margins": 7.53125,
      "rewards/rejected": -25.5,
      "step": 5560
    },
    {
      "epoch": 0.711866572943958,
      "grad_norm": 5.335961424536332,
      "learning_rate": 1.0719200377476648e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.91796875,
      "logps/chosen": -400.0,
      "logps/rejected": -452.0,
      "loss": 0.1173,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.875,
      "rewards/margins": 8.625,
      "rewards/rejected": -26.5,
      "step": 5570
    },
    {
      "epoch": 0.7131446098792255,
      "grad_norm": 4.717200426218658,
      "learning_rate": 1.0709591052033317e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.921875,
      "logps/chosen": -376.0,
      "logps/rejected": -472.0,
      "loss": 0.1115,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.125,
      "rewards/margins": 7.78125,
      "rewards/rejected": -25.875,
      "step": 5580
    },
    {
      "epoch": 0.714422646814493,
      "grad_norm": 14.288058111026375,
      "learning_rate": 1.0700007523445435e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.8359375,
      "logps/chosen": -412.0,
      "logps/rejected": -450.0,
      "loss": 0.1022,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.25,
      "rewards/margins": 8.25,
      "rewards/rejected": -26.5,
      "step": 5590
    },
    {
      "epoch": 0.7157006837497604,
      "grad_norm": 7.907628085592736,
      "learning_rate": 1.0690449676496976e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.921875,
      "logps/chosen": -386.0,
      "logps/rejected": -446.0,
      "loss": 0.1008,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.0,
      "rewards/margins": 8.5625,
      "rewards/rejected": -25.5,
      "step": 5600
    },
    {
      "epoch": 0.7169787206850278,
      "grad_norm": 5.680103359262543,
      "learning_rate": 1.0680917396691054e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.984375,
      "logps/chosen": -402.0,
      "logps/rejected": -464.0,
      "loss": 0.1334,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.375,
      "rewards/margins": 7.3125,
      "rewards/rejected": -24.625,
      "step": 5610
    },
    {
      "epoch": 0.7182567576202952,
      "grad_norm": 5.938534941674571,
      "learning_rate": 1.0671410570244164e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.890625,
      "logps/chosen": -384.0,
      "logps/rejected": -434.0,
      "loss": 0.1298,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.125,
      "rewards/margins": 8.1875,
      "rewards/rejected": -25.25,
      "step": 5620
    },
    {
      "epoch": 0.7195347945555627,
      "grad_norm": 7.433030765186112,
      "learning_rate": 1.0661929084080466e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.88671875,
      "logps/chosen": -404.0,
      "logps/rejected": -446.0,
      "loss": 0.1109,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -17.5,
      "rewards/margins": 7.75,
      "rewards/rejected": -25.25,
      "step": 5630
    },
    {
      "epoch": 0.7208128314908301,
      "grad_norm": 5.726437141217463,
      "learning_rate": 1.0652472825826149e-06,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.79296875,
      "logps/chosen": -376.0,
      "logps/rejected": -436.0,
      "loss": 0.0935,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.8125,
      "rewards/rejected": -24.5,
      "step": 5640
    },
    {
      "epoch": 0.7220908684260975,
      "grad_norm": 10.175305330128218,
      "learning_rate": 1.0643041683803828e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.85546875,
      "logps/chosen": -402.0,
      "logps/rejected": -450.0,
      "loss": 0.1324,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.75,
      "rewards/margins": 8.4375,
      "rewards/rejected": -26.25,
      "step": 5650
    },
    {
      "epoch": 0.7233689053613649,
      "grad_norm": 10.455194197853197,
      "learning_rate": 1.063363554702701e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.9609375,
      "logps/chosen": -396.0,
      "logps/rejected": -434.0,
      "loss": 0.1176,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.5,
      "rewards/margins": 7.8125,
      "rewards/rejected": -25.25,
      "step": 5660
    },
    {
      "epoch": 0.7246469422966324,
      "grad_norm": 8.292176406440538,
      "learning_rate": 1.0624254305194609e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.87109375,
      "logps/chosen": -404.0,
      "logps/rejected": -470.0,
      "loss": 0.109,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.75,
      "rewards/margins": 8.625,
      "rewards/rejected": -25.375,
      "step": 5670
    },
    {
      "epoch": 0.7259249792318998,
      "grad_norm": 5.493512866983232,
      "learning_rate": 1.0614897848685505e-06,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.83984375,
      "logps/chosen": -374.0,
      "logps/rejected": -420.0,
      "loss": 0.0868,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.6875,
      "rewards/margins": 7.84375,
      "rewards/rejected": -23.5,
      "step": 5680
    },
    {
      "epoch": 0.7272030161671672,
      "grad_norm": 4.981783886104641,
      "learning_rate": 1.0605566068553173e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.8671875,
      "logps/chosen": -394.0,
      "logps/rejected": -446.0,
      "loss": 0.1077,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.75,
      "rewards/margins": 7.90625,
      "rewards/rejected": -24.625,
      "step": 5690
    },
    {
      "epoch": 0.7284810531024346,
      "grad_norm": 4.238006946577378,
      "learning_rate": 1.0596258856520351e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.9296875,
      "logps/chosen": -370.0,
      "logps/rejected": -454.0,
      "loss": 0.1582,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -15.875,
      "rewards/margins": 8.4375,
      "rewards/rejected": -24.25,
      "step": 5700
    },
    {
      "epoch": 0.7297590900377021,
      "grad_norm": 8.988503404570658,
      "learning_rate": 1.0586976104973764e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.8203125,
      "logps/chosen": -390.0,
      "logps/rejected": -430.0,
      "loss": 0.1237,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.5,
      "rewards/margins": 8.1875,
      "rewards/rejected": -24.625,
      "step": 5710
    },
    {
      "epoch": 0.7310371269729695,
      "grad_norm": 9.960499761337907,
      "learning_rate": 1.05777177069589e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.87109375,
      "logps/chosen": -388.0,
      "logps/rejected": -428.0,
      "loss": 0.1252,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.125,
      "rewards/margins": 8.0625,
      "rewards/rejected": -24.25,
      "step": 5720
    },
    {
      "epoch": 0.7323151639082369,
      "grad_norm": 3.4309092396644076,
      "learning_rate": 1.0568483556174834e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.87109375,
      "logps/chosen": -372.0,
      "logps/rejected": -414.0,
      "loss": 0.1271,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.5,
      "rewards/margins": 7.03125,
      "rewards/rejected": -23.5,
      "step": 5730
    },
    {
      "epoch": 0.7335932008435043,
      "grad_norm": 5.398761456491691,
      "learning_rate": 1.055927354696909e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.76171875,
      "logps/chosen": -370.0,
      "logps/rejected": -418.0,
      "loss": 0.1143,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.625,
      "rewards/margins": 6.875,
      "rewards/rejected": -23.5,
      "step": 5740
    },
    {
      "epoch": 0.7348712377787718,
      "grad_norm": 6.967913186379111,
      "learning_rate": 1.0550087574332592e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.83203125,
      "logps/chosen": -376.0,
      "logps/rejected": -442.0,
      "loss": 0.1152,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -17.625,
      "rewards/margins": 7.34375,
      "rewards/rejected": -25.0,
      "step": 5750
    },
    {
      "epoch": 0.7361492747140392,
      "grad_norm": 10.273405336527505,
      "learning_rate": 1.0540925533894598e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.859375,
      "logps/chosen": -378.0,
      "logps/rejected": -432.0,
      "loss": 0.0947,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -16.25,
      "rewards/margins": 8.125,
      "rewards/rejected": -24.375,
      "step": 5760
    },
    {
      "epoch": 0.7374273116493066,
      "grad_norm": 5.751813423158511,
      "learning_rate": 1.053178732191775e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.8828125,
      "logps/chosen": -372.0,
      "logps/rejected": -428.0,
      "loss": 0.0827,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.125,
      "rewards/margins": 7.84375,
      "rewards/rejected": -25.0,
      "step": 5770
    },
    {
      "epoch": 0.738705348584574,
      "grad_norm": 6.174577480430647,
      "learning_rate": 1.0522672835293127e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.78125,
      "logps/chosen": -390.0,
      "logps/rejected": -440.0,
      "loss": 0.1281,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -18.125,
      "rewards/margins": 7.46875,
      "rewards/rejected": -25.625,
      "step": 5780
    },
    {
      "epoch": 0.7399833855198416,
      "grad_norm": 5.272307453157359,
      "learning_rate": 1.0513581971535365e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.8984375,
      "logps/chosen": -384.0,
      "logps/rejected": -450.0,
      "loss": 0.1191,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -18.125,
      "rewards/margins": 7.25,
      "rewards/rejected": -25.375,
      "step": 5790
    },
    {
      "epoch": 0.741261422455109,
      "grad_norm": 8.04801478464015,
      "learning_rate": 1.0504514628777803e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.859375,
      "logps/chosen": -392.0,
      "logps/rejected": -460.0,
      "loss": 0.1327,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.125,
      "rewards/margins": 7.6875,
      "rewards/rejected": -25.75,
      "step": 5800
    },
    {
      "epoch": 0.7425394593903764,
      "grad_norm": 6.7474010637213,
      "learning_rate": 1.0495470705767713e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.8515625,
      "logps/chosen": -406.0,
      "logps/rejected": -448.0,
      "loss": 0.1047,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.875,
      "rewards/margins": 7.71875,
      "rewards/rejected": -25.5,
      "step": 5810
    },
    {
      "epoch": 0.7438174963256438,
      "grad_norm": 5.49973577281345,
      "learning_rate": 1.0486450101861527e-06,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.88671875,
      "logps/chosen": -396.0,
      "logps/rejected": -440.0,
      "loss": 0.1273,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.125,
      "rewards/margins": 7.34375,
      "rewards/rejected": -25.5,
      "step": 5820
    },
    {
      "epoch": 0.7450955332609113,
      "grad_norm": 9.253819599635936,
      "learning_rate": 1.0477452717020143e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.7890625,
      "logps/chosen": -388.0,
      "logps/rejected": -436.0,
      "loss": 0.1287,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.5,
      "rewards/margins": 7.5,
      "rewards/rejected": -25.0,
      "step": 5830
    },
    {
      "epoch": 0.7463735701961787,
      "grad_norm": 7.540614440211587,
      "learning_rate": 1.0468478451804272e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.8125,
      "logps/chosen": -398.0,
      "logps/rejected": -450.0,
      "loss": 0.1349,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.75,
      "rewards/margins": 8.0,
      "rewards/rejected": -25.75,
      "step": 5840
    },
    {
      "epoch": 0.7476516071314461,
      "grad_norm": 6.489194549092509,
      "learning_rate": 1.0459527207369814e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.79296875,
      "logps/chosen": -398.0,
      "logps/rejected": -466.0,
      "loss": 0.1169,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.625,
      "rewards/margins": 8.375,
      "rewards/rejected": -26.0,
      "step": 5850
    },
    {
      "epoch": 0.7489296440667135,
      "grad_norm": 5.98460127488196,
      "learning_rate": 1.0450598885463281e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.74609375,
      "logps/chosen": -374.0,
      "logps/rejected": -458.0,
      "loss": 0.1086,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.75,
      "rewards/margins": 7.90625,
      "rewards/rejected": -25.75,
      "step": 5860
    },
    {
      "epoch": 0.750207681001981,
      "grad_norm": 9.167114669643905,
      "learning_rate": 1.0441693388417282e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.8125,
      "logps/chosen": -396.0,
      "logps/rejected": -440.0,
      "loss": 0.1165,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -18.125,
      "rewards/margins": 7.84375,
      "rewards/rejected": -25.875,
      "step": 5870
    },
    {
      "epoch": 0.7514857179372484,
      "grad_norm": 5.929677180336639,
      "learning_rate": 1.0432810619146023e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.8359375,
      "logps/chosen": -390.0,
      "logps/rejected": -456.0,
      "loss": 0.1456,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.125,
      "rewards/margins": 7.6875,
      "rewards/rejected": -25.75,
      "step": 5880
    },
    {
      "epoch": 0.7527637548725158,
      "grad_norm": 8.178784014156028,
      "learning_rate": 1.042395048114086e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.859375,
      "logps/chosen": -406.0,
      "logps/rejected": -470.0,
      "loss": 0.1237,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.625,
      "rewards/margins": 7.21875,
      "rewards/rejected": -24.875,
      "step": 5890
    },
    {
      "epoch": 0.7540417918077832,
      "grad_norm": 5.723634836660045,
      "learning_rate": 1.041511287846591e-06,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.80859375,
      "logps/chosen": -394.0,
      "logps/rejected": -448.0,
      "loss": 0.1079,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -18.5,
      "rewards/margins": 7.5,
      "rewards/rejected": -26.0,
      "step": 5900
    },
    {
      "epoch": 0.7553198287430507,
      "grad_norm": 7.479351066360936,
      "learning_rate": 1.0406297715753674e-06,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.94140625,
      "logps/chosen": -400.0,
      "logps/rejected": -440.0,
      "loss": 0.1552,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -18.625,
      "rewards/margins": 7.28125,
      "rewards/rejected": -25.875,
      "step": 5910
    },
    {
      "epoch": 0.7565978656783181,
      "grad_norm": 6.644773187317755,
      "learning_rate": 1.0397504898200726e-06,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.9453125,
      "logps/chosen": -420.0,
      "logps/rejected": -446.0,
      "loss": 0.1149,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -18.875,
      "rewards/margins": 7.28125,
      "rewards/rejected": -26.25,
      "step": 5920
    },
    {
      "epoch": 0.7578759026135855,
      "grad_norm": 6.086570653189906,
      "learning_rate": 1.0388734331563415e-06,
      "logits/chosen": -1.375,
      "logits/rejected": -0.77734375,
      "logps/chosen": -408.0,
      "logps/rejected": -458.0,
      "loss": 0.137,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -18.625,
      "rewards/margins": 7.59375,
      "rewards/rejected": -26.25,
      "step": 5930
    },
    {
      "epoch": 0.7591539395488529,
      "grad_norm": 6.542128102089513,
      "learning_rate": 1.037998592215364e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.890625,
      "logps/chosen": -422.0,
      "logps/rejected": -466.0,
      "loss": 0.1158,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.625,
      "rewards/margins": 8.1875,
      "rewards/rejected": -26.75,
      "step": 5940
    },
    {
      "epoch": 0.7604319764841204,
      "grad_norm": 3.831950402609419,
      "learning_rate": 1.037125957683463e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.87890625,
      "logps/chosen": -426.0,
      "logps/rejected": -470.0,
      "loss": 0.1197,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.75,
      "rewards/margins": 7.84375,
      "rewards/rejected": -25.625,
      "step": 5950
    },
    {
      "epoch": 0.7617100134193878,
      "grad_norm": 7.620695441071331,
      "learning_rate": 1.0362555203016794e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.86328125,
      "logps/chosen": -384.0,
      "logps/rejected": -458.0,
      "loss": 0.1093,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.25,
      "rewards/margins": 7.34375,
      "rewards/rejected": -25.625,
      "step": 5960
    },
    {
      "epoch": 0.7629880503546552,
      "grad_norm": 6.837198458763995,
      "learning_rate": 1.035387270865359e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.89453125,
      "logps/chosen": -406.0,
      "logps/rejected": -436.0,
      "loss": 0.1313,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.375,
      "rewards/margins": 8.0625,
      "rewards/rejected": -26.375,
      "step": 5970
    },
    {
      "epoch": 0.7642660872899226,
      "grad_norm": 4.085127025348929,
      "learning_rate": 1.0345212002237434e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.87890625,
      "logps/chosen": -422.0,
      "logps/rejected": -434.0,
      "loss": 0.1147,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -18.25,
      "rewards/margins": 6.625,
      "rewards/rejected": -24.875,
      "step": 5980
    },
    {
      "epoch": 0.7655441242251901,
      "grad_norm": 6.020263570625957,
      "learning_rate": 1.0336572992795644e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.79296875,
      "logps/chosen": -396.0,
      "logps/rejected": -450.0,
      "loss": 0.101,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.375,
      "rewards/margins": 8.5625,
      "rewards/rejected": -26.0,
      "step": 5990
    },
    {
      "epoch": 0.7668221611604575,
      "grad_norm": 5.945418583558677,
      "learning_rate": 1.0327955589886444e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.94921875,
      "logps/chosen": -394.0,
      "logps/rejected": -452.0,
      "loss": 0.0949,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.5,
      "rewards/margins": 7.40625,
      "rewards/rejected": -25.875,
      "step": 6000
    },
    {
      "epoch": 0.7681001980957249,
      "grad_norm": 7.740896346039646,
      "learning_rate": 1.0319359703594971e-06,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.9296875,
      "logps/chosen": -398.0,
      "logps/rejected": -446.0,
      "loss": 0.0928,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.625,
      "rewards/margins": 8.0625,
      "rewards/rejected": -25.75,
      "step": 6010
    },
    {
      "epoch": 0.7693782350309923,
      "grad_norm": 5.582331057962313,
      "learning_rate": 1.0310785244529341e-06,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.765625,
      "logps/chosen": -406.0,
      "logps/rejected": -500.0,
      "loss": 0.0793,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.25,
      "rewards/margins": 8.5,
      "rewards/rejected": -26.75,
      "step": 6020
    },
    {
      "epoch": 0.7706562719662599,
      "grad_norm": 7.403066380643566,
      "learning_rate": 1.0302232123816746e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.7421875,
      "logps/chosen": -406.0,
      "logps/rejected": -456.0,
      "loss": 0.1162,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.0,
      "rewards/margins": 8.3125,
      "rewards/rejected": -27.25,
      "step": 6030
    },
    {
      "epoch": 0.7719343089015273,
      "grad_norm": 6.700977976838253,
      "learning_rate": 1.0293700253099576e-06,
      "logits/chosen": -1.25,
      "logits/rejected": -0.80859375,
      "logps/chosen": -386.0,
      "logps/rejected": -442.0,
      "loss": 0.1003,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -18.125,
      "rewards/margins": 7.75,
      "rewards/rejected": -26.0,
      "step": 6040
    },
    {
      "epoch": 0.7732123458367947,
      "grad_norm": 5.996560392790137,
      "learning_rate": 1.02851895445316e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.828125,
      "logps/chosen": -390.0,
      "logps/rejected": -442.0,
      "loss": 0.1154,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.875,
      "rewards/margins": 8.375,
      "rewards/rejected": -26.25,
      "step": 6050
    },
    {
      "epoch": 0.7744903827720621,
      "grad_norm": 6.822068417676302,
      "learning_rate": 1.0276699910774159e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.671875,
      "logps/chosen": -404.0,
      "logps/rejected": -452.0,
      "loss": 0.1006,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.5,
      "rewards/margins": 8.4375,
      "rewards/rejected": -26.0,
      "step": 6060
    },
    {
      "epoch": 0.7757684197073296,
      "grad_norm": 6.797046747943064,
      "learning_rate": 1.0268231264992398e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.80078125,
      "logps/chosen": -374.0,
      "logps/rejected": -446.0,
      "loss": 0.153,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.25,
      "rewards/margins": 7.5625,
      "rewards/rejected": -24.875,
      "step": 6070
    },
    {
      "epoch": 0.777046456642597,
      "grad_norm": 4.636876400763272,
      "learning_rate": 1.0259783520851542e-06,
      "logits/chosen": -1.125,
      "logits/rejected": -0.85546875,
      "logps/chosen": -370.0,
      "logps/rejected": -450.0,
      "loss": 0.1141,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.625,
      "rewards/margins": 7.8125,
      "rewards/rejected": -25.5,
      "step": 6080
    },
    {
      "epoch": 0.7783244935778644,
      "grad_norm": 7.145984591675959,
      "learning_rate": 1.0251356592513193e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.7265625,
      "logps/chosen": -402.0,
      "logps/rejected": -440.0,
      "loss": 0.1041,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.5,
      "rewards/margins": 7.625,
      "rewards/rejected": -26.125,
      "step": 6090
    },
    {
      "epoch": 0.7796025305131318,
      "grad_norm": 3.750977551311327,
      "learning_rate": 1.0242950394631678e-06,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.7890625,
      "logps/chosen": -394.0,
      "logps/rejected": -448.0,
      "loss": 0.0804,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.375,
      "rewards/margins": 7.40625,
      "rewards/rejected": -25.75,
      "step": 6100
    },
    {
      "epoch": 0.7808805674483993,
      "grad_norm": 11.751148659341426,
      "learning_rate": 1.0234564842350404e-06,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.75,
      "logps/chosen": -382.0,
      "logps/rejected": -442.0,
      "loss": 0.1076,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.0,
      "rewards/margins": 7.46875,
      "rewards/rejected": -25.375,
      "step": 6110
    },
    {
      "epoch": 0.7821586043836667,
      "grad_norm": 6.97114600924647,
      "learning_rate": 1.0226199851298272e-06,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.94921875,
      "logps/chosen": -388.0,
      "logps/rejected": -452.0,
      "loss": 0.0977,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.875,
      "rewards/margins": 8.25,
      "rewards/rejected": -26.0,
      "step": 6120
    },
    {
      "epoch": 0.7834366413189341,
      "grad_norm": 5.959262854771312,
      "learning_rate": 1.0217855337586106e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.80859375,
      "logps/chosen": -398.0,
      "logps/rejected": -462.0,
      "loss": 0.0961,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.875,
      "rewards/margins": 8.375,
      "rewards/rejected": -26.25,
      "step": 6130
    },
    {
      "epoch": 0.7847146782542015,
      "grad_norm": 6.191808169666578,
      "learning_rate": 1.0209531217803119e-06,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.859375,
      "logps/chosen": -402.0,
      "logps/rejected": -460.0,
      "loss": 0.1033,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -18.0,
      "rewards/margins": 8.3125,
      "rewards/rejected": -26.25,
      "step": 6140
    },
    {
      "epoch": 0.785992715189469,
      "grad_norm": 5.4715136896051995,
      "learning_rate": 1.0201227409013412e-06,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.8984375,
      "logps/chosen": -386.0,
      "logps/rejected": -468.0,
      "loss": 0.1029,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.375,
      "rewards/margins": 9.5625,
      "rewards/rejected": -27.0,
      "step": 6150
    },
    {
      "epoch": 0.7872707521247364,
      "grad_norm": 5.3883153573848075,
      "learning_rate": 1.0192943828752509e-06,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.74609375,
      "logps/chosen": -390.0,
      "logps/rejected": -450.0,
      "loss": 0.1048,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.5,
      "rewards/margins": 8.9375,
      "rewards/rejected": -25.5,
      "step": 6160
    },
    {
      "epoch": 0.7885487890600038,
      "grad_norm": 8.33652916061245,
      "learning_rate": 1.0184680395023912e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.91796875,
      "logps/chosen": -392.0,
      "logps/rejected": -432.0,
      "loss": 0.1144,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.90625,
      "rewards/rejected": -24.25,
      "step": 6170
    },
    {
      "epoch": 0.7898268259952712,
      "grad_norm": 4.492565798840593,
      "learning_rate": 1.0176437026295688e-06,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.7734375,
      "logps/chosen": -376.0,
      "logps/rejected": -460.0,
      "loss": 0.1003,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.75,
      "rewards/margins": 8.4375,
      "rewards/rejected": -25.125,
      "step": 6180
    },
    {
      "epoch": 0.7911048629305387,
      "grad_norm": 6.094491863551988,
      "learning_rate": 1.0168213641497094e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.77734375,
      "logps/chosen": -382.0,
      "logps/rejected": -420.0,
      "loss": 0.1092,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.5,
      "rewards/margins": 8.3125,
      "rewards/rejected": -24.75,
      "step": 6190
    },
    {
      "epoch": 0.7923828998658061,
      "grad_norm": 10.555967067742259,
      "learning_rate": 1.016001016001524e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.84375,
      "logps/chosen": -396.0,
      "logps/rejected": -446.0,
      "loss": 0.1122,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.625,
      "rewards/margins": 7.4375,
      "rewards/rejected": -25.125,
      "step": 6200
    },
    {
      "epoch": 0.7936609368010735,
      "grad_norm": 8.608164645687246,
      "learning_rate": 1.0151826501691747e-06,
      "logits/chosen": -1.3515625,
      "logits/rejected": -0.84375,
      "logps/chosen": -408.0,
      "logps/rejected": -442.0,
      "loss": 0.154,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -19.125,
      "rewards/margins": 7.25,
      "rewards/rejected": -26.375,
      "step": 6210
    },
    {
      "epoch": 0.7949389737363409,
      "grad_norm": 9.243438465959024,
      "learning_rate": 1.0143662586819475e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.80078125,
      "logps/chosen": -404.0,
      "logps/rejected": -458.0,
      "loss": 0.1307,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -18.75,
      "rewards/margins": 7.40625,
      "rewards/rejected": -26.125,
      "step": 6220
    },
    {
      "epoch": 0.7962170106716084,
      "grad_norm": 4.368448051574361,
      "learning_rate": 1.0135518336139257e-06,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.7734375,
      "logps/chosen": -398.0,
      "logps/rejected": -446.0,
      "loss": 0.1169,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -19.125,
      "rewards/margins": 6.625,
      "rewards/rejected": -25.75,
      "step": 6230
    },
    {
      "epoch": 0.7974950476068758,
      "grad_norm": 5.715972480459543,
      "learning_rate": 1.0127393670836666e-06,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.84375,
      "logps/chosen": -400.0,
      "logps/rejected": -452.0,
      "loss": 0.1091,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -19.0,
      "rewards/margins": 6.6875,
      "rewards/rejected": -25.75,
      "step": 6240
    },
    {
      "epoch": 0.7987730845421432,
      "grad_norm": 6.368654489954981,
      "learning_rate": 1.0119288512538813e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.9140625,
      "logps/chosen": -400.0,
      "logps/rejected": -450.0,
      "loss": 0.1324,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.25,
      "rewards/margins": 6.96875,
      "rewards/rejected": -26.25,
      "step": 6250
    },
    {
      "epoch": 0.8000511214774106,
      "grad_norm": 5.504275128017138,
      "learning_rate": 1.0111202783311173e-06,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.91796875,
      "logps/chosen": -414.0,
      "logps/rejected": -454.0,
      "loss": 0.0929,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.0,
      "rewards/margins": 7.25,
      "rewards/rejected": -26.25,
      "step": 6260
    },
    {
      "epoch": 0.8013291584126782,
      "grad_norm": 4.6810257025210635,
      "learning_rate": 1.010313640565443e-06,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.7890625,
      "logps/chosen": -398.0,
      "logps/rejected": -438.0,
      "loss": 0.1227,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.25,
      "rewards/margins": 7.21875,
      "rewards/rejected": -25.5,
      "step": 6270
    },
    {
      "epoch": 0.8026071953479456,
      "grad_norm": 5.051065386100453,
      "learning_rate": 1.0095089302501373e-06,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.8359375,
      "logps/chosen": -414.0,
      "logps/rejected": -470.0,
      "loss": 0.1185,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.25,
      "rewards/margins": 7.875,
      "rewards/rejected": -26.125,
      "step": 6280
    },
    {
      "epoch": 0.803885232283213,
      "grad_norm": 7.80134475379633,
      "learning_rate": 1.0087061397213787e-06,
      "logits/chosen": -1.1875,
      "logits/rejected": -0.75,
      "logps/chosen": -390.0,
      "logps/rejected": -448.0,
      "loss": 0.1224,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -17.5,
      "rewards/margins": 8.0,
      "rewards/rejected": -25.5,
      "step": 6290
    },
    {
      "epoch": 0.8051632692184804,
      "grad_norm": 7.098498831182669,
      "learning_rate": 1.007905261357939e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.796875,
      "logps/chosen": -364.0,
      "logps/rejected": -446.0,
      "loss": 0.0926,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -17.25,
      "rewards/margins": 8.8125,
      "rewards/rejected": -26.0,
      "step": 6300
    },
    {
      "epoch": 0.8064413061537479,
      "grad_norm": 6.72421233967875,
      "learning_rate": 1.0071062875808811e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.79296875,
      "logps/chosen": -390.0,
      "logps/rejected": -454.0,
      "loss": 0.1099,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.125,
      "rewards/margins": 7.8125,
      "rewards/rejected": -25.875,
      "step": 6310
    },
    {
      "epoch": 0.8077193430890153,
      "grad_norm": 4.9195005371128575,
      "learning_rate": 1.0063092108532552e-06,
      "logits/chosen": -1.1484375,
      "logits/rejected": -0.74609375,
      "logps/chosen": -404.0,
      "logps/rejected": -452.0,
      "loss": 0.118,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.375,
      "rewards/margins": 7.125,
      "rewards/rejected": -25.375,
      "step": 6320
    },
    {
      "epoch": 0.8089973800242827,
      "grad_norm": 6.391822162636114,
      "learning_rate": 1.005514023679802e-06,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.76953125,
      "logps/chosen": -392.0,
      "logps/rejected": -450.0,
      "loss": 0.1187,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.75,
      "rewards/margins": 7.65625,
      "rewards/rejected": -25.375,
      "step": 6330
    },
    {
      "epoch": 0.8102754169595501,
      "grad_norm": 6.295484204215971,
      "learning_rate": 1.0047207186066567e-06,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.90625,
      "logps/chosen": -400.0,
      "logps/rejected": -450.0,
      "loss": 0.1045,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.25,
      "rewards/margins": 8.125,
      "rewards/rejected": -26.375,
      "step": 6340
    },
    {
      "epoch": 0.8115534538948176,
      "grad_norm": 8.363678344182592,
      "learning_rate": 1.0039292882210538e-06,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.87109375,
      "logps/chosen": -406.0,
      "logps/rejected": -458.0,
      "loss": 0.1126,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -18.125,
      "rewards/margins": 7.46875,
      "rewards/rejected": -25.625,
      "step": 6350
    },
    {
      "epoch": 0.812831490830085,
      "grad_norm": 9.975725681335167,
      "learning_rate": 1.0031397251510382e-06,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.92578125,
      "logps/chosen": -390.0,
      "logps/rejected": -466.0,
      "loss": 0.1071,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.875,
      "rewards/margins": 8.25,
      "rewards/rejected": -26.125,
      "step": 6360
    },
    {
      "epoch": 0.8141095277653524,
      "grad_norm": 5.246411259707353,
      "learning_rate": 1.0023520220651762e-06,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.82421875,
      "logps/chosen": -388.0,
      "logps/rejected": -444.0,
      "loss": 0.0887,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.25,
      "rewards/margins": 7.96875,
      "rewards/rejected": -25.125,
      "step": 6370
    },
    {
      "epoch": 0.8153875647006198,
      "grad_norm": 6.226657539635892,
      "learning_rate": 1.0015661716722687e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.84375,
      "logps/chosen": -376.0,
      "logps/rejected": -466.0,
      "loss": 0.107,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.625,
      "rewards/margins": 8.375,
      "rewards/rejected": -26.0,
      "step": 6380
    },
    {
      "epoch": 0.8166656016358873,
      "grad_norm": 4.532065669777639,
      "learning_rate": 1.0007821667210687e-06,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.86328125,
      "logps/chosen": -412.0,
      "logps/rejected": -458.0,
      "loss": 0.1054,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.125,
      "rewards/margins": 8.6875,
      "rewards/rejected": -26.875,
      "step": 6390
    },
    {
      "epoch": 0.8179436385711547,
      "grad_norm": 5.97694313762022,
      "learning_rate": 1e-06,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.9296875,
      "logps/chosen": -396.0,
      "logps/rejected": -470.0,
      "loss": 0.1293,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -19.625,
      "rewards/margins": 7.21875,
      "rewards/rejected": -26.875,
      "step": 6400
    },
    {
      "epoch": 0.8192216755064221,
      "grad_norm": 6.669376996851472,
      "learning_rate": 9.992196643368784e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.93359375,
      "logps/chosen": -396.0,
      "logps/rejected": -460.0,
      "loss": 0.1308,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -19.375,
      "rewards/margins": 8.1875,
      "rewards/rejected": -27.625,
      "step": 6410
    },
    {
      "epoch": 0.8204997124416895,
      "grad_norm": 6.0675813063564155,
      "learning_rate": 9.984411525986355e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.84765625,
      "logps/chosen": -408.0,
      "logps/rejected": -482.0,
      "loss": 0.0989,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.875,
      "rewards/margins": 8.5625,
      "rewards/rejected": -27.5,
      "step": 6420
    },
    {
      "epoch": 0.821777749376957,
      "grad_norm": 4.4935928117048,
      "learning_rate": 9.97664457691046e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.79296875,
      "logps/chosen": -400.0,
      "logps/rejected": -484.0,
      "loss": 0.0977,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -19.125,
      "rewards/margins": 8.625,
      "rewards/rejected": -27.75,
      "step": 6430
    },
    {
      "epoch": 0.8230557863122244,
      "grad_norm": 5.916239194854684,
      "learning_rate": 9.968895725584535e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -0.796875,
      "logps/chosen": -406.0,
      "logps/rejected": -456.0,
      "loss": 0.1147,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -18.875,
      "rewards/margins": 8.8125,
      "rewards/rejected": -27.75,
      "step": 6440
    },
    {
      "epoch": 0.8243338232474918,
      "grad_norm": 5.608269763208908,
      "learning_rate": 9.961164901835046e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.828125,
      "logps/chosen": -396.0,
      "logps/rejected": -468.0,
      "loss": 0.0985,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -19.0,
      "rewards/margins": 7.875,
      "rewards/rejected": -26.875,
      "step": 6450
    },
    {
      "epoch": 0.8256118601827592,
      "grad_norm": 5.644324307288136,
      "learning_rate": 9.95345203586879e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.8828125,
      "logps/chosen": -378.0,
      "logps/rejected": -452.0,
      "loss": 0.1066,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -18.875,
      "rewards/margins": 8.375,
      "rewards/rejected": -27.25,
      "step": 6460
    },
    {
      "epoch": 0.8268898971180267,
      "grad_norm": 6.111999964069675,
      "learning_rate": 9.94575705827027e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -0.890625,
      "logps/chosen": -410.0,
      "logps/rejected": -482.0,
      "loss": 0.1132,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -18.625,
      "rewards/margins": 8.5,
      "rewards/rejected": -27.125,
      "step": 6470
    },
    {
      "epoch": 0.8281679340532941,
      "grad_norm": 6.397092640776829,
      "learning_rate": 9.938079899999065e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.87890625,
      "logps/chosen": -412.0,
      "logps/rejected": -442.0,
      "loss": 0.1272,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -18.875,
      "rewards/margins": 7.4375,
      "rewards/rejected": -26.375,
      "step": 6480
    },
    {
      "epoch": 0.8294459709885615,
      "grad_norm": 3.971228626840262,
      "learning_rate": 9.930420492387219e-07,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.953125,
      "logps/chosen": -396.0,
      "logps/rejected": -458.0,
      "loss": 0.1087,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -18.125,
      "rewards/margins": 7.84375,
      "rewards/rejected": -26.0,
      "step": 6490
    },
    {
      "epoch": 0.8307240079238289,
      "grad_norm": 7.84308834927562,
      "learning_rate": 9.922778767136676e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.796875,
      "logps/chosen": -394.0,
      "logps/rejected": -478.0,
      "loss": 0.1112,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.0,
      "rewards/margins": 8.3125,
      "rewards/rejected": -25.25,
      "step": 6500
    },
    {
      "epoch": 0.8320020448590965,
      "grad_norm": 6.338506077786089,
      "learning_rate": 9.915154656316713e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.83984375,
      "logps/chosen": -396.0,
      "logps/rejected": -450.0,
      "loss": 0.1082,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -18.0,
      "rewards/margins": 7.8125,
      "rewards/rejected": -25.75,
      "step": 6510
    },
    {
      "epoch": 0.8332800817943639,
      "grad_norm": 4.784218763762756,
      "learning_rate": 9.907548092361398e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -0.8515625,
      "logps/chosen": -416.0,
      "logps/rejected": -464.0,
      "loss": 0.099,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -18.75,
      "rewards/margins": 7.875,
      "rewards/rejected": -26.75,
      "step": 6520
    },
    {
      "epoch": 0.8345581187296313,
      "grad_norm": 8.668894884094035,
      "learning_rate": 9.899959008067097e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -0.9453125,
      "logps/chosen": -398.0,
      "logps/rejected": -488.0,
      "loss": 0.1191,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -18.875,
      "rewards/margins": 7.96875,
      "rewards/rejected": -26.875,
      "step": 6530
    },
    {
      "epoch": 0.8358361556648987,
      "grad_norm": 5.980532754008555,
      "learning_rate": 9.892387336589959e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -0.76953125,
      "logps/chosen": -390.0,
      "logps/rejected": -434.0,
      "loss": 0.1028,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.0,
      "rewards/margins": 7.5625,
      "rewards/rejected": -24.625,
      "step": 6540
    },
    {
      "epoch": 0.8371141926001662,
      "grad_norm": 5.78790132127344,
      "learning_rate": 9.884833011443446e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.828125,
      "logps/chosen": -364.0,
      "logps/rejected": -446.0,
      "loss": 0.1138,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -16.375,
      "rewards/margins": 8.1875,
      "rewards/rejected": -24.5,
      "step": 6550
    },
    {
      "epoch": 0.8383922295354336,
      "grad_norm": 9.679887148661745,
      "learning_rate": 9.877295966495897e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -0.8203125,
      "logps/chosen": -356.0,
      "logps/rejected": -408.0,
      "loss": 0.1159,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -15.625,
      "rewards/margins": 7.5,
      "rewards/rejected": -23.125,
      "step": 6560
    },
    {
      "epoch": 0.839670266470701,
      "grad_norm": 26.963282345674976,
      "learning_rate": 9.86977613596807e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.8828125,
      "logps/chosen": -362.0,
      "logps/rejected": -414.0,
      "loss": 0.1104,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -16.25,
      "rewards/margins": 7.5625,
      "rewards/rejected": -23.875,
      "step": 6570
    },
    {
      "epoch": 0.8409483034059684,
      "grad_norm": 4.316685370200527,
      "learning_rate": 9.862273454430757e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.8203125,
      "logps/chosen": -396.0,
      "logps/rejected": -432.0,
      "loss": 0.0904,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.125,
      "rewards/margins": 7.96875,
      "rewards/rejected": -25.125,
      "step": 6580
    },
    {
      "epoch": 0.8422263403412359,
      "grad_norm": 5.52511924590449,
      "learning_rate": 9.85478785680238e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.8046875,
      "logps/chosen": -404.0,
      "logps/rejected": -458.0,
      "loss": 0.1107,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -18.25,
      "rewards/margins": 7.6875,
      "rewards/rejected": -26.0,
      "step": 6590
    },
    {
      "epoch": 0.8435043772765033,
      "grad_norm": 5.148494010557906,
      "learning_rate": 9.847319278346618e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.921875,
      "logps/chosen": -394.0,
      "logps/rejected": -450.0,
      "loss": 0.1012,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -16.875,
      "rewards/margins": 8.375,
      "rewards/rejected": -25.25,
      "step": 6600
    },
    {
      "epoch": 0.8447824142117707,
      "grad_norm": 6.240996227975338,
      "learning_rate": 9.839867654670063e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.84375,
      "logps/chosen": -390.0,
      "logps/rejected": -444.0,
      "loss": 0.0755,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.875,
      "rewards/margins": 7.21875,
      "rewards/rejected": -25.125,
      "step": 6610
    },
    {
      "epoch": 0.8460604511470381,
      "grad_norm": 3.2102163000017816,
      "learning_rate": 9.832432921719876e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.76171875,
      "logps/chosen": -368.0,
      "logps/rejected": -428.0,
      "loss": 0.092,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.0,
      "rewards/margins": 8.3125,
      "rewards/rejected": -25.25,
      "step": 6620
    },
    {
      "epoch": 0.8473384880823056,
      "grad_norm": 3.8654286143112127,
      "learning_rate": 9.825015015781493e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.91796875,
      "logps/chosen": -390.0,
      "logps/rejected": -474.0,
      "loss": 0.0953,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.125,
      "rewards/margins": 8.625,
      "rewards/rejected": -25.75,
      "step": 6630
    },
    {
      "epoch": 0.848616525017573,
      "grad_norm": 4.737501283444343,
      "learning_rate": 9.81761387347632e-07,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.94921875,
      "logps/chosen": -376.0,
      "logps/rejected": -430.0,
      "loss": 0.1049,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -16.625,
      "rewards/margins": 8.75,
      "rewards/rejected": -25.375,
      "step": 6640
    },
    {
      "epoch": 0.8498945619528404,
      "grad_norm": 4.822080281074219,
      "learning_rate": 9.810229431759452e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.84765625,
      "logps/chosen": -384.0,
      "logps/rejected": -446.0,
      "loss": 0.1027,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -16.625,
      "rewards/margins": 8.9375,
      "rewards/rejected": -25.5,
      "step": 6650
    },
    {
      "epoch": 0.8511725988881078,
      "grad_norm": 7.27615608890228,
      "learning_rate": 9.802861627917437e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -0.7734375,
      "logps/chosen": -392.0,
      "logps/rejected": -456.0,
      "loss": 0.0933,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.125,
      "rewards/margins": 7.65625,
      "rewards/rejected": -25.875,
      "step": 6660
    },
    {
      "epoch": 0.8524506358233753,
      "grad_norm": 4.181432142480873,
      "learning_rate": 9.795510399566016e-07,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.8984375,
      "logps/chosen": -398.0,
      "logps/rejected": -444.0,
      "loss": 0.0982,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.0,
      "rewards/margins": 7.25,
      "rewards/rejected": -25.375,
      "step": 6670
    },
    {
      "epoch": 0.8537286727586427,
      "grad_norm": 6.838461474873084,
      "learning_rate": 9.788175684647926e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.8515625,
      "logps/chosen": -400.0,
      "logps/rejected": -466.0,
      "loss": 0.1056,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.375,
      "rewards/margins": 8.0625,
      "rewards/rejected": -26.375,
      "step": 6680
    },
    {
      "epoch": 0.8550067096939101,
      "grad_norm": 2.2501930712137077,
      "learning_rate": 9.780857421430687e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.84375,
      "logps/chosen": -398.0,
      "logps/rejected": -440.0,
      "loss": 0.1093,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -19.0,
      "rewards/margins": 7.8125,
      "rewards/rejected": -26.75,
      "step": 6690
    },
    {
      "epoch": 0.8562847466291775,
      "grad_norm": 7.108654217365872,
      "learning_rate": 9.773555548504417e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.9609375,
      "logps/chosen": -408.0,
      "logps/rejected": -444.0,
      "loss": 0.0936,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.75,
      "rewards/margins": 7.96875,
      "rewards/rejected": -26.625,
      "step": 6700
    },
    {
      "epoch": 0.857562783564445,
      "grad_norm": 5.248963938696396,
      "learning_rate": 9.76627000477968e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.0078125,
      "logps/chosen": -388.0,
      "logps/rejected": -446.0,
      "loss": 0.0974,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -18.875,
      "rewards/margins": 7.78125,
      "rewards/rejected": -26.625,
      "step": 6710
    },
    {
      "epoch": 0.8588408204997124,
      "grad_norm": 9.870924745131335,
      "learning_rate": 9.75900072948533e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.90625,
      "logps/chosen": -400.0,
      "logps/rejected": -458.0,
      "loss": 0.0955,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -19.375,
      "rewards/margins": 8.0,
      "rewards/rejected": -27.375,
      "step": 6720
    },
    {
      "epoch": 0.8601188574349798,
      "grad_norm": 4.17667739901276,
      "learning_rate": 9.751747662166388e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -0.80078125,
      "logps/chosen": -404.0,
      "logps/rejected": -484.0,
      "loss": 0.0926,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.375,
      "rewards/margins": 8.4375,
      "rewards/rejected": -27.75,
      "step": 6730
    },
    {
      "epoch": 0.8613968943702472,
      "grad_norm": 4.999108485104864,
      "learning_rate": 9.744510742681917e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.91796875,
      "logps/chosen": -410.0,
      "logps/rejected": -472.0,
      "loss": 0.0909,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -19.5,
      "rewards/margins": 8.5625,
      "rewards/rejected": -28.125,
      "step": 6740
    },
    {
      "epoch": 0.8626749313055148,
      "grad_norm": 10.328672114039316,
      "learning_rate": 9.737289911202953e-07,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.8359375,
      "logps/chosen": -376.0,
      "logps/rejected": -438.0,
      "loss": 0.1038,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.25,
      "rewards/margins": 8.4375,
      "rewards/rejected": -26.625,
      "step": 6750
    },
    {
      "epoch": 0.8639529682407822,
      "grad_norm": 7.119008787085872,
      "learning_rate": 9.730085108210398e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.98828125,
      "logps/chosen": -404.0,
      "logps/rejected": -448.0,
      "loss": 0.087,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.5,
      "rewards/margins": 8.5,
      "rewards/rejected": -26.0,
      "step": 6760
    },
    {
      "epoch": 0.8652310051760496,
      "grad_norm": 9.895527333540302,
      "learning_rate": 9.72289627449298e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.85546875,
      "logps/chosen": -416.0,
      "logps/rejected": -478.0,
      "loss": 0.1015,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -19.75,
      "rewards/margins": 8.3125,
      "rewards/rejected": -28.0,
      "step": 6770
    },
    {
      "epoch": 0.866509042111317,
      "grad_norm": 5.127124841901257,
      "learning_rate": 9.715723351145206e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -0.84375,
      "logps/chosen": -404.0,
      "logps/rejected": -470.0,
      "loss": 0.1061,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.5,
      "rewards/margins": 8.9375,
      "rewards/rejected": -28.5,
      "step": 6780
    },
    {
      "epoch": 0.8677870790465845,
      "grad_norm": 4.169197068856286,
      "learning_rate": 9.70856627956532e-07,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.86328125,
      "logps/chosen": -404.0,
      "logps/rejected": -446.0,
      "loss": 0.1306,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.0,
      "rewards/margins": 8.375,
      "rewards/rejected": -27.5,
      "step": 6790
    },
    {
      "epoch": 0.8690651159818519,
      "grad_norm": 8.628448694699976,
      "learning_rate": 9.701425001453318e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.765625,
      "logps/chosen": -386.0,
      "logps/rejected": -444.0,
      "loss": 0.097,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.5,
      "rewards/margins": 7.78125,
      "rewards/rejected": -26.375,
      "step": 6800
    },
    {
      "epoch": 0.8703431529171193,
      "grad_norm": 9.036513188150279,
      "learning_rate": 9.694299458808932e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -0.91796875,
      "logps/chosen": -404.0,
      "logps/rejected": -472.0,
      "loss": 0.0908,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.125,
      "rewards/margins": 8.4375,
      "rewards/rejected": -27.5,
      "step": 6810
    },
    {
      "epoch": 0.8716211898523867,
      "grad_norm": 5.709630056648207,
      "learning_rate": 9.687189593929655e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.86328125,
      "logps/chosen": -384.0,
      "logps/rejected": -448.0,
      "loss": 0.1043,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.5,
      "rewards/margins": 8.5,
      "rewards/rejected": -27.0,
      "step": 6820
    },
    {
      "epoch": 0.8728992267876542,
      "grad_norm": 7.530654626447082,
      "learning_rate": 9.680095349408789e-07,
      "logits/chosen": -1.4921875,
      "logits/rejected": -0.9296875,
      "logps/chosen": -402.0,
      "logps/rejected": -472.0,
      "loss": 0.1002,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -18.5,
      "rewards/margins": 9.0,
      "rewards/rejected": -27.5,
      "step": 6830
    },
    {
      "epoch": 0.8741772637229216,
      "grad_norm": 4.421593327933572,
      "learning_rate": 9.673016668133487e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -0.93359375,
      "logps/chosen": -414.0,
      "logps/rejected": -472.0,
      "loss": 0.1008,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -19.375,
      "rewards/margins": 7.90625,
      "rewards/rejected": -27.25,
      "step": 6840
    },
    {
      "epoch": 0.875455300658189,
      "grad_norm": 7.57612352655515,
      "learning_rate": 9.66595349328283e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -0.92578125,
      "logps/chosen": -408.0,
      "logps/rejected": -466.0,
      "loss": 0.0921,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.25,
      "rewards/margins": 8.4375,
      "rewards/rejected": -27.75,
      "step": 6850
    },
    {
      "epoch": 0.8767333375934564,
      "grad_norm": 2.4421367918502535,
      "learning_rate": 9.658905768325902e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -0.95703125,
      "logps/chosen": -404.0,
      "logps/rejected": -466.0,
      "loss": 0.1036,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.75,
      "rewards/margins": 7.5,
      "rewards/rejected": -27.25,
      "step": 6860
    },
    {
      "epoch": 0.8780113745287239,
      "grad_norm": 8.653568168360595,
      "learning_rate": 9.651873437019902e-07,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.8671875,
      "logps/chosen": -416.0,
      "logps/rejected": -472.0,
      "loss": 0.0947,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -19.125,
      "rewards/margins": 7.875,
      "rewards/rejected": -27.0,
      "step": 6870
    },
    {
      "epoch": 0.8792894114639913,
      "grad_norm": 4.6444731326630215,
      "learning_rate": 9.644856443408243e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.89453125,
      "logps/chosen": -390.0,
      "logps/rejected": -470.0,
      "loss": 0.0845,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -18.875,
      "rewards/margins": 7.8125,
      "rewards/rejected": -26.625,
      "step": 6880
    },
    {
      "epoch": 0.8805674483992587,
      "grad_norm": 9.077298632231,
      "learning_rate": 9.637854731818697e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.87109375,
      "logps/chosen": -402.0,
      "logps/rejected": -470.0,
      "loss": 0.0753,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -18.75,
      "rewards/margins": 8.4375,
      "rewards/rejected": -27.25,
      "step": 6890
    },
    {
      "epoch": 0.8818454853345261,
      "grad_norm": 14.350014097066584,
      "learning_rate": 9.630868246861536e-07,
      "logits/chosen": -1.453125,
      "logits/rejected": -1.0078125,
      "logps/chosen": -398.0,
      "logps/rejected": -474.0,
      "loss": 0.1302,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.875,
      "rewards/margins": 9.0,
      "rewards/rejected": -26.875,
      "step": 6900
    },
    {
      "epoch": 0.8831235222697936,
      "grad_norm": 6.029621755569458,
      "learning_rate": 9.623896933427685e-07,
      "logits/chosen": -1.46875,
      "logits/rejected": -0.96484375,
      "logps/chosen": -402.0,
      "logps/rejected": -478.0,
      "loss": 0.0828,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.875,
      "rewards/margins": 8.625,
      "rewards/rejected": -26.5,
      "step": 6910
    },
    {
      "epoch": 0.884401559205061,
      "grad_norm": 9.902513349457543,
      "learning_rate": 9.6169407366869e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.9296875,
      "logps/chosen": -392.0,
      "logps/rejected": -448.0,
      "loss": 0.1183,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -18.625,
      "rewards/margins": 8.125,
      "rewards/rejected": -26.75,
      "step": 6920
    },
    {
      "epoch": 0.8856795961403284,
      "grad_norm": 5.0910373249075,
      "learning_rate": 9.609999602085963e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.96484375,
      "logps/chosen": -398.0,
      "logps/rejected": -460.0,
      "loss": 0.1077,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -18.75,
      "rewards/margins": 8.4375,
      "rewards/rejected": -27.125,
      "step": 6930
    },
    {
      "epoch": 0.8869576330755958,
      "grad_norm": 8.868316787798724,
      "learning_rate": 9.603073475346872e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.91015625,
      "logps/chosen": -418.0,
      "logps/rejected": -470.0,
      "loss": 0.111,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -18.875,
      "rewards/margins": 8.3125,
      "rewards/rejected": -27.25,
      "step": 6940
    },
    {
      "epoch": 0.8882356700108633,
      "grad_norm": 7.20608103565963,
      "learning_rate": 9.596162302465074e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -0.83984375,
      "logps/chosen": -398.0,
      "logps/rejected": -472.0,
      "loss": 0.088,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.125,
      "rewards/margins": 8.0,
      "rewards/rejected": -26.125,
      "step": 6950
    },
    {
      "epoch": 0.8895137069461307,
      "grad_norm": 3.8858939721080485,
      "learning_rate": 9.589266029707683e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -0.8515625,
      "logps/chosen": -392.0,
      "logps/rejected": -446.0,
      "loss": 0.0981,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.75,
      "rewards/margins": 8.5625,
      "rewards/rejected": -26.375,
      "step": 6960
    },
    {
      "epoch": 0.8907917438813981,
      "grad_norm": 5.184740804532246,
      "learning_rate": 9.582384603611731e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.97265625,
      "logps/chosen": -388.0,
      "logps/rejected": -456.0,
      "loss": 0.1145,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -18.25,
      "rewards/margins": 8.125,
      "rewards/rejected": -26.5,
      "step": 6970
    },
    {
      "epoch": 0.8920697808166655,
      "grad_norm": 6.950255030193362,
      "learning_rate": 9.575517970982428e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -1.0078125,
      "logps/chosen": -416.0,
      "logps/rejected": -448.0,
      "loss": 0.0861,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.125,
      "rewards/margins": 8.125,
      "rewards/rejected": -26.25,
      "step": 6980
    },
    {
      "epoch": 0.8933478177519331,
      "grad_norm": 11.000459679297034,
      "learning_rate": 9.568666078891436e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.9296875,
      "logps/chosen": -404.0,
      "logps/rejected": -478.0,
      "loss": 0.1335,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -18.625,
      "rewards/margins": 8.4375,
      "rewards/rejected": -27.0,
      "step": 6990
    },
    {
      "epoch": 0.8946258546872005,
      "grad_norm": 4.060670236478793,
      "learning_rate": 9.561828874675149e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -0.97265625,
      "logps/chosen": -392.0,
      "logps/rejected": -448.0,
      "loss": 0.0921,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.5,
      "rewards/margins": 8.875,
      "rewards/rejected": -26.375,
      "step": 7000
    },
    {
      "epoch": 0.8959038916224679,
      "grad_norm": 7.591354627917325,
      "learning_rate": 9.555006305933e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -0.79296875,
      "logps/chosen": -392.0,
      "logps/rejected": -452.0,
      "loss": 0.1094,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.75,
      "rewards/margins": 8.0,
      "rewards/rejected": -25.75,
      "step": 7010
    },
    {
      "epoch": 0.8971819285577353,
      "grad_norm": 5.4951390849271915,
      "learning_rate": 9.548198320525771e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -0.96875,
      "logps/chosen": -392.0,
      "logps/rejected": -454.0,
      "loss": 0.1163,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.75,
      "rewards/margins": 8.0625,
      "rewards/rejected": -26.75,
      "step": 7020
    },
    {
      "epoch": 0.8984599654930028,
      "grad_norm": 7.762928893976843,
      "learning_rate": 9.54140486657392e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.0390625,
      "logps/chosen": -430.0,
      "logps/rejected": -490.0,
      "loss": 0.0923,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.125,
      "rewards/margins": 8.3125,
      "rewards/rejected": -27.5,
      "step": 7030
    },
    {
      "epoch": 0.8997380024282702,
      "grad_norm": 4.82000845387724,
      "learning_rate": 9.534625892455922e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.91015625,
      "logps/chosen": -396.0,
      "logps/rejected": -462.0,
      "loss": 0.0919,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -19.0,
      "rewards/margins": 8.0625,
      "rewards/rejected": -27.125,
      "step": 7040
    },
    {
      "epoch": 0.9010160393635376,
      "grad_norm": 8.686858468668953,
      "learning_rate": 9.527861346806618e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -0.94921875,
      "logps/chosen": -408.0,
      "logps/rejected": -458.0,
      "loss": 0.0824,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -18.625,
      "rewards/margins": 8.5625,
      "rewards/rejected": -27.125,
      "step": 7050
    },
    {
      "epoch": 0.902294076298805,
      "grad_norm": 6.143309596051915,
      "learning_rate": 9.521111178515582e-07,
      "logits/chosen": -1.515625,
      "logits/rejected": -1.140625,
      "logps/chosen": -408.0,
      "logps/rejected": -470.0,
      "loss": 0.0994,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.75,
      "rewards/margins": 8.0,
      "rewards/rejected": -26.75,
      "step": 7060
    },
    {
      "epoch": 0.9035721132340725,
      "grad_norm": 7.084432231348941,
      "learning_rate": 9.514375336725502e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.91015625,
      "logps/chosen": -416.0,
      "logps/rejected": -476.0,
      "loss": 0.094,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -20.125,
      "rewards/margins": 7.5625,
      "rewards/rejected": -27.75,
      "step": 7070
    },
    {
      "epoch": 0.9048501501693399,
      "grad_norm": 6.506184929830165,
      "learning_rate": 9.507653770830566e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -0.98828125,
      "logps/chosen": -396.0,
      "logps/rejected": -448.0,
      "loss": 0.1119,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.125,
      "rewards/margins": 7.9375,
      "rewards/rejected": -27.0,
      "step": 7080
    },
    {
      "epoch": 0.9061281871046073,
      "grad_norm": 6.277089211557529,
      "learning_rate": 9.500946430474869e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.81640625,
      "logps/chosen": -396.0,
      "logps/rejected": -460.0,
      "loss": 0.112,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -19.375,
      "rewards/margins": 7.75,
      "rewards/rejected": -27.125,
      "step": 7090
    },
    {
      "epoch": 0.9074062240398747,
      "grad_norm": 6.002817944655785,
      "learning_rate": 9.494253265550825e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -0.8984375,
      "logps/chosen": -396.0,
      "logps/rejected": -476.0,
      "loss": 0.1086,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.125,
      "rewards/margins": 8.75,
      "rewards/rejected": -27.875,
      "step": 7100
    },
    {
      "epoch": 0.9086842609751422,
      "grad_norm": 5.836519382062422,
      "learning_rate": 9.4875742261976e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.9921875,
      "logps/chosen": -412.0,
      "logps/rejected": -454.0,
      "loss": 0.1136,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -20.375,
      "rewards/margins": 7.1875,
      "rewards/rejected": -27.5,
      "step": 7110
    },
    {
      "epoch": 0.9099622979104096,
      "grad_norm": 4.845573347766808,
      "learning_rate": 9.480909262799544e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.921875,
      "logps/chosen": -410.0,
      "logps/rejected": -466.0,
      "loss": 0.1099,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.75,
      "rewards/margins": 7.6875,
      "rewards/rejected": -27.375,
      "step": 7120
    },
    {
      "epoch": 0.911240334845677,
      "grad_norm": 9.52331805075016,
      "learning_rate": 9.47425832598465e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -0.8515625,
      "logps/chosen": -396.0,
      "logps/rejected": -448.0,
      "loss": 0.1064,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -19.125,
      "rewards/margins": 7.125,
      "rewards/rejected": -26.25,
      "step": 7130
    },
    {
      "epoch": 0.9125183717809444,
      "grad_norm": 4.896260470665807,
      "learning_rate": 9.467621366623017e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.7890625,
      "logps/chosen": -418.0,
      "logps/rejected": -490.0,
      "loss": 0.0798,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.5,
      "rewards/margins": 8.0,
      "rewards/rejected": -27.5,
      "step": 7140
    },
    {
      "epoch": 0.9137964087162119,
      "grad_norm": 7.932951806230017,
      "learning_rate": 9.46099833582532e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.890625,
      "logps/chosen": -404.0,
      "logps/rejected": -482.0,
      "loss": 0.0805,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.25,
      "rewards/margins": 8.4375,
      "rewards/rejected": -27.625,
      "step": 7150
    },
    {
      "epoch": 0.9150744456514793,
      "grad_norm": 5.064210390406986,
      "learning_rate": 9.45438918494131e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -0.9296875,
      "logps/chosen": -410.0,
      "logps/rejected": -484.0,
      "loss": 0.105,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -19.75,
      "rewards/margins": 8.25,
      "rewards/rejected": -28.0,
      "step": 7160
    },
    {
      "epoch": 0.9163524825867467,
      "grad_norm": 10.126473408984506,
      "learning_rate": 9.447793865558291e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.8671875,
      "logps/chosen": -410.0,
      "logps/rejected": -450.0,
      "loss": 0.1167,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -20.25,
      "rewards/margins": 7.5625,
      "rewards/rejected": -27.875,
      "step": 7170
    },
    {
      "epoch": 0.9176305195220141,
      "grad_norm": 4.173762064022671,
      "learning_rate": 9.441212329499659e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -0.8671875,
      "logps/chosen": -402.0,
      "logps/rejected": -450.0,
      "loss": 0.1075,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.5,
      "rewards/margins": 8.125,
      "rewards/rejected": -27.625,
      "step": 7180
    },
    {
      "epoch": 0.9189085564572816,
      "grad_norm": 5.591517718338325,
      "learning_rate": 9.434644528823399e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.87109375,
      "logps/chosen": -402.0,
      "logps/rejected": -446.0,
      "loss": 0.0748,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.75,
      "rewards/margins": 8.125,
      "rewards/rejected": -26.875,
      "step": 7190
    },
    {
      "epoch": 0.920186593392549,
      "grad_norm": 8.819963317751089,
      "learning_rate": 9.428090415820634e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.7734375,
      "logps/chosen": -410.0,
      "logps/rejected": -478.0,
      "loss": 0.1154,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -19.75,
      "rewards/margins": 8.375,
      "rewards/rejected": -28.125,
      "step": 7200
    },
    {
      "epoch": 0.9214646303278164,
      "grad_norm": 9.824617072237464,
      "learning_rate": 9.42154994301416e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.87109375,
      "logps/chosen": -414.0,
      "logps/rejected": -452.0,
      "loss": 0.104,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.125,
      "rewards/margins": 8.5625,
      "rewards/rejected": -27.625,
      "step": 7210
    },
    {
      "epoch": 0.9227426672630838,
      "grad_norm": 6.684397112673839,
      "learning_rate": 9.415023063157008e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.875,
      "logps/chosen": -422.0,
      "logps/rejected": -460.0,
      "loss": 0.0962,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.125,
      "rewards/margins": 9.0625,
      "rewards/rejected": -28.25,
      "step": 7220
    },
    {
      "epoch": 0.9240207041983514,
      "grad_norm": 5.674054473226296,
      "learning_rate": 9.408509729231009e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.8203125,
      "logps/chosen": -404.0,
      "logps/rejected": -476.0,
      "loss": 0.0876,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -20.125,
      "rewards/margins": 8.5625,
      "rewards/rejected": -28.625,
      "step": 7230
    },
    {
      "epoch": 0.9252987411336188,
      "grad_norm": 4.539304177020339,
      "learning_rate": 9.402009894445369e-07,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.91796875,
      "logps/chosen": -408.0,
      "logps/rejected": -450.0,
      "loss": 0.1177,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -19.625,
      "rewards/margins": 8.6875,
      "rewards/rejected": -28.25,
      "step": 7240
    },
    {
      "epoch": 0.9265767780688862,
      "grad_norm": 6.646371411645457,
      "learning_rate": 9.395523512235255e-07,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.84375,
      "logps/chosen": -410.0,
      "logps/rejected": -468.0,
      "loss": 0.1164,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -20.125,
      "rewards/margins": 7.65625,
      "rewards/rejected": -27.75,
      "step": 7250
    },
    {
      "epoch": 0.9278548150041536,
      "grad_norm": 7.341582869071921,
      "learning_rate": 9.389050536260404e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.84375,
      "logps/chosen": -424.0,
      "logps/rejected": -488.0,
      "loss": 0.1027,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -20.25,
      "rewards/margins": 7.71875,
      "rewards/rejected": -28.0,
      "step": 7260
    },
    {
      "epoch": 0.9291328519394211,
      "grad_norm": 5.7230567566428086,
      "learning_rate": 9.382590920403722e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.7890625,
      "logps/chosen": -408.0,
      "logps/rejected": -460.0,
      "loss": 0.1105,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.625,
      "rewards/margins": 7.5,
      "rewards/rejected": -27.125,
      "step": 7270
    },
    {
      "epoch": 0.9304108888746885,
      "grad_norm": 4.592080418362118,
      "learning_rate": 9.376144618769908e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.84375,
      "logps/chosen": -410.0,
      "logps/rejected": -476.0,
      "loss": 0.0812,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -20.0,
      "rewards/margins": 8.5,
      "rewards/rejected": -28.5,
      "step": 7280
    },
    {
      "epoch": 0.9316889258099559,
      "grad_norm": 8.891208862275935,
      "learning_rate": 9.369711585684086e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.8359375,
      "logps/chosen": -412.0,
      "logps/rejected": -470.0,
      "loss": 0.094,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -20.25,
      "rewards/margins": 8.3125,
      "rewards/rejected": -28.5,
      "step": 7290
    },
    {
      "epoch": 0.9329669627452233,
      "grad_norm": 5.250457513623724,
      "learning_rate": 9.363291775690445e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -0.765625,
      "logps/chosen": -414.0,
      "logps/rejected": -494.0,
      "loss": 0.1188,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -19.0,
      "rewards/margins": 9.0,
      "rewards/rejected": -28.0,
      "step": 7300
    },
    {
      "epoch": 0.9342449996804908,
      "grad_norm": 5.095499150945258,
      "learning_rate": 9.356885143550886e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.7734375,
      "logps/chosen": -404.0,
      "logps/rejected": -472.0,
      "loss": 0.0978,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -20.0,
      "rewards/margins": 8.3125,
      "rewards/rejected": -28.25,
      "step": 7310
    },
    {
      "epoch": 0.9355230366157582,
      "grad_norm": 6.392933104947357,
      "learning_rate": 9.350491644243688e-07,
      "logits/chosen": -1.296875,
      "logits/rejected": -0.86328125,
      "logps/chosen": -398.0,
      "logps/rejected": -462.0,
      "loss": 0.1013,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -20.125,
      "rewards/margins": 8.25,
      "rewards/rejected": -28.375,
      "step": 7320
    },
    {
      "epoch": 0.9368010735510256,
      "grad_norm": 9.03894263064198,
      "learning_rate": 9.344111232962179e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.94140625,
      "logps/chosen": -396.0,
      "logps/rejected": -470.0,
      "loss": 0.1037,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -20.0,
      "rewards/margins": 8.5,
      "rewards/rejected": -28.5,
      "step": 7330
    },
    {
      "epoch": 0.938079110486293,
      "grad_norm": 6.968089257153338,
      "learning_rate": 9.337743865113415e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -0.828125,
      "logps/chosen": -422.0,
      "logps/rejected": -492.0,
      "loss": 0.1093,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.5,
      "rewards/margins": 8.125,
      "rewards/rejected": -27.625,
      "step": 7340
    },
    {
      "epoch": 0.9393571474215605,
      "grad_norm": 5.550681009561365,
      "learning_rate": 9.331389496316868e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.85546875,
      "logps/chosen": -408.0,
      "logps/rejected": -478.0,
      "loss": 0.089,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.25,
      "rewards/margins": 9.125,
      "rewards/rejected": -27.375,
      "step": 7350
    },
    {
      "epoch": 0.9406351843568279,
      "grad_norm": 10.08073972705786,
      "learning_rate": 9.325048082403138e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.86328125,
      "logps/chosen": -434.0,
      "logps/rejected": -478.0,
      "loss": 0.1233,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -19.625,
      "rewards/margins": 8.5625,
      "rewards/rejected": -28.25,
      "step": 7360
    },
    {
      "epoch": 0.9419132212920953,
      "grad_norm": 5.711820699613827,
      "learning_rate": 9.318719579412648e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -0.94140625,
      "logps/chosen": -424.0,
      "logps/rejected": -464.0,
      "loss": 0.1121,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -19.0,
      "rewards/margins": 8.375,
      "rewards/rejected": -27.25,
      "step": 7370
    },
    {
      "epoch": 0.9431912582273627,
      "grad_norm": 1.3019802926885833,
      "learning_rate": 9.312403943594374e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -0.89453125,
      "logps/chosen": -380.0,
      "logps/rejected": -446.0,
      "loss": 0.1133,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -18.375,
      "rewards/margins": 7.53125,
      "rewards/rejected": -25.875,
      "step": 7380
    },
    {
      "epoch": 0.9444692951626302,
      "grad_norm": 4.5877987540344565,
      "learning_rate": 9.306101131404582e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.9140625,
      "logps/chosen": -380.0,
      "logps/rejected": -448.0,
      "loss": 0.096,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.5,
      "rewards/margins": 7.46875,
      "rewards/rejected": -25.0,
      "step": 7390
    },
    {
      "epoch": 0.9457473320978976,
      "grad_norm": 6.8418753963204635,
      "learning_rate": 9.299811099505542e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -0.8984375,
      "logps/chosen": -404.0,
      "logps/rejected": -462.0,
      "loss": 0.095,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.75,
      "rewards/margins": 8.0625,
      "rewards/rejected": -25.75,
      "step": 7400
    },
    {
      "epoch": 0.947025369033165,
      "grad_norm": 10.125313004693552,
      "learning_rate": 9.293533804764305e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -0.91796875,
      "logps/chosen": -396.0,
      "logps/rejected": -470.0,
      "loss": 0.0939,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.75,
      "rewards/margins": 8.8125,
      "rewards/rejected": -26.5,
      "step": 7410
    },
    {
      "epoch": 0.9483034059684324,
      "grad_norm": 5.477594419223949,
      "learning_rate": 9.28726920425144e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -1.0078125,
      "logps/chosen": -404.0,
      "logps/rejected": -474.0,
      "loss": 0.095,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -19.25,
      "rewards/margins": 7.90625,
      "rewards/rejected": -27.25,
      "step": 7420
    },
    {
      "epoch": 0.9495814429037,
      "grad_norm": 7.648705626066345,
      "learning_rate": 9.281017255239815e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.828125,
      "logps/chosen": -398.0,
      "logps/rejected": -480.0,
      "loss": 0.0933,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -18.375,
      "rewards/margins": 8.5,
      "rewards/rejected": -26.875,
      "step": 7430
    },
    {
      "epoch": 0.9508594798389673,
      "grad_norm": 4.755919242260372,
      "learning_rate": 9.274777915203365e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.828125,
      "logps/chosen": -386.0,
      "logps/rejected": -482.0,
      "loss": 0.0849,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.875,
      "rewards/margins": 7.71875,
      "rewards/rejected": -25.625,
      "step": 7440
    },
    {
      "epoch": 0.9521375167742347,
      "grad_norm": 6.892319879617283,
      "learning_rate": 9.268551141815875e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.03125,
      "logps/chosen": -400.0,
      "logps/rejected": -462.0,
      "loss": 0.0988,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -17.625,
      "rewards/margins": 7.84375,
      "rewards/rejected": -25.5,
      "step": 7450
    },
    {
      "epoch": 0.9534155537095022,
      "grad_norm": 7.516569220258863,
      "learning_rate": 9.262336892949784e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.85546875,
      "logps/chosen": -386.0,
      "logps/rejected": -438.0,
      "loss": 0.1132,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -17.375,
      "rewards/margins": 7.9375,
      "rewards/rejected": -25.375,
      "step": 7460
    },
    {
      "epoch": 0.9546935906447697,
      "grad_norm": 6.14885277590864,
      "learning_rate": 9.256135126674977e-07,
      "logits/chosen": -1.3125,
      "logits/rejected": -0.88671875,
      "logps/chosen": -370.0,
      "logps/rejected": -440.0,
      "loss": 0.1038,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.75,
      "rewards/margins": 8.125,
      "rewards/rejected": -25.875,
      "step": 7470
    },
    {
      "epoch": 0.9559716275800371,
      "grad_norm": 7.074390587346651,
      "learning_rate": 9.249945801257605e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -0.98828125,
      "logps/chosen": -386.0,
      "logps/rejected": -452.0,
      "loss": 0.1074,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.75,
      "rewards/margins": 8.4375,
      "rewards/rejected": -26.25,
      "step": 7480
    },
    {
      "epoch": 0.9572496645153045,
      "grad_norm": 7.647355382722027,
      "learning_rate": 9.243768875158902e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.83203125,
      "logps/chosen": -392.0,
      "logps/rejected": -444.0,
      "loss": 0.1125,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.75,
      "rewards/margins": 8.25,
      "rewards/rejected": -26.0,
      "step": 7490
    },
    {
      "epoch": 0.9585277014505719,
      "grad_norm": 6.8241864263665395,
      "learning_rate": 9.23760430703401e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.9375,
      "logps/chosen": -386.0,
      "logps/rejected": -428.0,
      "loss": 0.1088,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.25,
      "rewards/margins": 7.75,
      "rewards/rejected": -25.0,
      "step": 7500
    },
    {
      "epoch": 0.9598057383858394,
      "grad_norm": 10.84108936997761,
      "learning_rate": 9.231452055730832e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -0.92578125,
      "logps/chosen": -408.0,
      "logps/rejected": -448.0,
      "loss": 0.0959,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.25,
      "rewards/margins": 8.5,
      "rewards/rejected": -26.75,
      "step": 7510
    },
    {
      "epoch": 0.9610837753211068,
      "grad_norm": 3.7681597840522723,
      "learning_rate": 9.225312080288851e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -0.875,
      "logps/chosen": -374.0,
      "logps/rejected": -444.0,
      "loss": 0.0887,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.125,
      "rewards/margins": 8.125,
      "rewards/rejected": -25.25,
      "step": 7520
    },
    {
      "epoch": 0.9623618122563742,
      "grad_norm": 6.924106224014419,
      "learning_rate": 9.219184339938013e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -0.9609375,
      "logps/chosen": -394.0,
      "logps/rejected": -468.0,
      "loss": 0.1041,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.875,
      "rewards/margins": 8.375,
      "rewards/rejected": -26.125,
      "step": 7530
    },
    {
      "epoch": 0.9636398491916416,
      "grad_norm": 4.155710222673749,
      "learning_rate": 9.213068794097574e-07,
      "logits/chosen": -1.4375,
      "logits/rejected": -0.9375,
      "logps/chosen": -392.0,
      "logps/rejected": -466.0,
      "loss": 0.0815,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -17.375,
      "rewards/margins": 9.3125,
      "rewards/rejected": -26.75,
      "step": 7540
    },
    {
      "epoch": 0.9649178861269091,
      "grad_norm": 6.812800589378033,
      "learning_rate": 9.206965402374975e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -0.90625,
      "logps/chosen": -382.0,
      "logps/rejected": -450.0,
      "loss": 0.0916,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.25,
      "rewards/margins": 8.0625,
      "rewards/rejected": -25.375,
      "step": 7550
    },
    {
      "epoch": 0.9661959230621765,
      "grad_norm": 5.711565872815511,
      "learning_rate": 9.200874124564723e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -0.875,
      "logps/chosen": -414.0,
      "logps/rejected": -446.0,
      "loss": 0.1045,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -17.875,
      "rewards/margins": 8.5625,
      "rewards/rejected": -26.5,
      "step": 7560
    },
    {
      "epoch": 0.9674739599974439,
      "grad_norm": 5.787355024743504,
      "learning_rate": 9.194794920647274e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -0.89453125,
      "logps/chosen": -404.0,
      "logps/rejected": -458.0,
      "loss": 0.0716,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.125,
      "rewards/margins": 9.25,
      "rewards/rejected": -26.375,
      "step": 7570
    },
    {
      "epoch": 0.9687519969327113,
      "grad_norm": 4.485683766819674,
      "learning_rate": 9.188727750787932e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -0.8671875,
      "logps/chosen": -412.0,
      "logps/rejected": -464.0,
      "loss": 0.0961,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -18.625,
      "rewards/margins": 8.5,
      "rewards/rejected": -27.125,
      "step": 7580
    },
    {
      "epoch": 0.9700300338679788,
      "grad_norm": 8.274678059989636,
      "learning_rate": 9.182672575335757e-07,
      "logits/chosen": -1.4609375,
      "logits/rejected": -0.95703125,
      "logps/chosen": -404.0,
      "logps/rejected": -464.0,
      "loss": 0.0896,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.375,
      "rewards/margins": 8.4375,
      "rewards/rejected": -26.75,
      "step": 7590
    },
    {
      "epoch": 0.9713080708032462,
      "grad_norm": 6.75851258453748,
      "learning_rate": 9.176629354822469e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -0.93359375,
      "logps/chosen": -400.0,
      "logps/rejected": -454.0,
      "loss": 0.1075,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.75,
      "rewards/margins": 7.59375,
      "rewards/rejected": -26.375,
      "step": 7600
    },
    {
      "epoch": 0.9725861077385136,
      "grad_norm": 6.972385803240149,
      "learning_rate": 9.170598049961371e-07,
      "logits/chosen": -1.4375,
      "logits/rejected": -0.97265625,
      "logps/chosen": -428.0,
      "logps/rejected": -486.0,
      "loss": 0.0829,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -19.75,
      "rewards/margins": 7.78125,
      "rewards/rejected": -27.5,
      "step": 7610
    },
    {
      "epoch": 0.973864144673781,
      "grad_norm": 2.7128090571847756,
      "learning_rate": 9.164578621646276e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -0.90234375,
      "logps/chosen": -410.0,
      "logps/rejected": -472.0,
      "loss": 0.074,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -19.625,
      "rewards/margins": 8.5,
      "rewards/rejected": -28.125,
      "step": 7620
    },
    {
      "epoch": 0.9751421816090485,
      "grad_norm": 3.849070138362031,
      "learning_rate": 9.15857103095044e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -1.0,
      "logps/chosen": -432.0,
      "logps/rejected": -450.0,
      "loss": 0.1007,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -19.125,
      "rewards/margins": 8.3125,
      "rewards/rejected": -27.375,
      "step": 7630
    },
    {
      "epoch": 0.9764202185443159,
      "grad_norm": 6.431562633387236,
      "learning_rate": 9.15257523912551e-07,
      "logits/chosen": -1.5234375,
      "logits/rejected": -0.88671875,
      "logps/chosen": -414.0,
      "logps/rejected": -486.0,
      "loss": 0.101,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -20.375,
      "rewards/margins": 7.625,
      "rewards/rejected": -28.0,
      "step": 7640
    },
    {
      "epoch": 0.9776982554795833,
      "grad_norm": 11.391952094115743,
      "learning_rate": 9.146591207600472e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -0.91015625,
      "logps/chosen": -414.0,
      "logps/rejected": -478.0,
      "loss": 0.106,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -20.75,
      "rewards/margins": 7.78125,
      "rewards/rejected": -28.5,
      "step": 7650
    },
    {
      "epoch": 0.9789762924148507,
      "grad_norm": 6.839902701154703,
      "learning_rate": 9.140618897980601e-07,
      "logits/chosen": -1.5078125,
      "logits/rejected": -0.96875,
      "logps/chosen": -414.0,
      "logps/rejected": -478.0,
      "loss": 0.0873,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.625,
      "rewards/margins": 9.4375,
      "rewards/rejected": -29.125,
      "step": 7660
    },
    {
      "epoch": 0.9802543293501182,
      "grad_norm": 11.723721817740882,
      "learning_rate": 9.134658272046442e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -0.9375,
      "logps/chosen": -412.0,
      "logps/rejected": -484.0,
      "loss": 0.1036,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -19.125,
      "rewards/margins": 8.8125,
      "rewards/rejected": -28.0,
      "step": 7670
    },
    {
      "epoch": 0.9815323662853856,
      "grad_norm": 9.463621151617769,
      "learning_rate": 9.128709291752768e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.0,
      "logps/chosen": -412.0,
      "logps/rejected": -468.0,
      "loss": 0.0921,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.125,
      "rewards/margins": 8.375,
      "rewards/rejected": -26.5,
      "step": 7680
    },
    {
      "epoch": 0.982810403220653,
      "grad_norm": 8.09278124312972,
      "learning_rate": 9.122771919227568e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -0.88671875,
      "logps/chosen": -390.0,
      "logps/rejected": -460.0,
      "loss": 0.0862,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -18.125,
      "rewards/margins": 8.75,
      "rewards/rejected": -26.875,
      "step": 7690
    },
    {
      "epoch": 0.9840884401559205,
      "grad_norm": 2.6494089369998397,
      "learning_rate": 9.116846116771035e-07,
      "logits/chosen": -1.4375,
      "logits/rejected": -0.8671875,
      "logps/chosen": -406.0,
      "logps/rejected": -462.0,
      "loss": 0.0905,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.625,
      "rewards/margins": 9.75,
      "rewards/rejected": -27.375,
      "step": 7700
    },
    {
      "epoch": 0.985366477091188,
      "grad_norm": 9.065812216470789,
      "learning_rate": 9.110931846854553e-07,
      "logits/chosen": -1.453125,
      "logits/rejected": -1.109375,
      "logps/chosen": -410.0,
      "logps/rejected": -470.0,
      "loss": 0.1097,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.0,
      "rewards/margins": 8.9375,
      "rewards/rejected": -27.0,
      "step": 7710
    },
    {
      "epoch": 0.9866445140264554,
      "grad_norm": 6.975885894428917,
      "learning_rate": 9.105029072119708e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.0234375,
      "logps/chosen": -400.0,
      "logps/rejected": -450.0,
      "loss": 0.1117,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -18.0,
      "rewards/margins": 7.90625,
      "rewards/rejected": -25.875,
      "step": 7720
    },
    {
      "epoch": 0.9879225509617228,
      "grad_norm": 7.774932366066183,
      "learning_rate": 9.099137755377291e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.0078125,
      "logps/chosen": -384.0,
      "logps/rejected": -440.0,
      "loss": 0.0862,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.5,
      "rewards/margins": 7.96875,
      "rewards/rejected": -25.5,
      "step": 7730
    },
    {
      "epoch": 0.9892005878969902,
      "grad_norm": 7.328614709875619,
      "learning_rate": 9.093257859606311e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -0.9921875,
      "logps/chosen": -392.0,
      "logps/rejected": -440.0,
      "loss": 0.0857,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -17.75,
      "rewards/margins": 7.90625,
      "rewards/rejected": -25.625,
      "step": 7740
    },
    {
      "epoch": 0.9904786248322577,
      "grad_norm": 5.84644919222835,
      "learning_rate": 9.087389347953037e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -0.88671875,
      "logps/chosen": -398.0,
      "logps/rejected": -438.0,
      "loss": 0.094,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.25,
      "rewards/margins": 8.5625,
      "rewards/rejected": -25.75,
      "step": 7750
    },
    {
      "epoch": 0.9917566617675251,
      "grad_norm": 4.1964245047527635,
      "learning_rate": 9.081532183729995e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -0.953125,
      "logps/chosen": -386.0,
      "logps/rejected": -454.0,
      "loss": 0.0857,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -18.0,
      "rewards/margins": 8.1875,
      "rewards/rejected": -26.25,
      "step": 7760
    },
    {
      "epoch": 0.9930346987027925,
      "grad_norm": 4.7482220501673895,
      "learning_rate": 9.075686330415037e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -0.99609375,
      "logps/chosen": -400.0,
      "logps/rejected": -450.0,
      "loss": 0.1116,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -18.25,
      "rewards/margins": 8.375,
      "rewards/rejected": -26.5,
      "step": 7770
    },
    {
      "epoch": 0.9943127356380599,
      "grad_norm": 5.4699777172005595,
      "learning_rate": 9.069851751650364e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.0625,
      "logps/chosen": -384.0,
      "logps/rejected": -464.0,
      "loss": 0.0985,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -17.375,
      "rewards/margins": 8.3125,
      "rewards/rejected": -25.75,
      "step": 7780
    },
    {
      "epoch": 0.9955907725733274,
      "grad_norm": 4.541286827527024,
      "learning_rate": 9.064028411241582e-07,
      "logits/chosen": -1.4296875,
      "logits/rejected": -0.9375,
      "logps/chosen": -374.0,
      "logps/rejected": -428.0,
      "loss": 0.0766,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -17.375,
      "rewards/margins": 8.25,
      "rewards/rejected": -25.5,
      "step": 7790
    },
    {
      "epoch": 0.9968688095085948,
      "grad_norm": 8.169648235113476,
      "learning_rate": 9.058216273156764e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -0.91015625,
      "logps/chosen": -384.0,
      "logps/rejected": -450.0,
      "loss": 0.0934,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -17.0,
      "rewards/margins": 8.1875,
      "rewards/rejected": -25.125,
      "step": 7800
    },
    {
      "epoch": 0.9981468464438622,
      "grad_norm": 12.27589610709961,
      "learning_rate": 9.052415301525511e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -0.890625,
      "logps/chosen": -414.0,
      "logps/rejected": -492.0,
      "loss": 0.1047,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -17.625,
      "rewards/margins": 8.875,
      "rewards/rejected": -26.5,
      "step": 7810
    },
    {
      "epoch": 0.9994248833791296,
      "grad_norm": 6.254262593930918,
      "learning_rate": 9.046625460638012e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -0.9296875,
      "logps/chosen": -386.0,
      "logps/rejected": -450.0,
      "loss": 0.0706,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -18.125,
      "rewards/margins": 9.5,
      "rewards/rejected": -27.5,
      "step": 7820
    }
  ],
  "logging_steps": 10,
  "max_steps": 7824,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 5000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}