diff --git "a/checkpoint-5000/trainer_state.json" "b/checkpoint-5000/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-5000/trainer_state.json"
@@ -0,0 +1,7564 @@
+{
+  "best_metric": 0.22941964864730835,
+  "best_model_checkpoint": "models/qwen2.5-3b-orpo-mini-fp/checkpoint-5000",
+  "epoch": 0.6390184676337146,
+  "eval_steps": 5000,
+  "global_step": 5000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00012780369352674293,
+      "grad_norm": 17.79294632846499,
+      "learning_rate": 8e-08,
+      "logits/chosen": -1.6875,
+      "logits/rejected": -1.484375,
+      "logps/chosen": -186.0,
+      "logps/rejected": -200.0,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0012780369352674292,
+      "grad_norm": 19.157016992578,
+      "learning_rate": 8e-07,
+      "logits/chosen": -1.9453125,
+      "logits/rejected": -1.4765625,
+      "logps/chosen": -212.0,
+      "logps/rejected": -182.0,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.2083333283662796,
+      "rewards/chosen": 0.004180908203125,
+      "rewards/margins": 0.001068115234375,
+      "rewards/rejected": 0.0031280517578125,
+      "step": 10
+    },
+    {
+      "epoch": 0.0025560738705348585,
+      "grad_norm": 15.16100498420179,
+      "learning_rate": 1.6e-06,
+      "logits/chosen": -2.015625,
+      "logits/rejected": -1.6640625,
+      "logps/chosen": -224.0,
+      "logps/rejected": -186.0,
+      "loss": 0.6179,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.0400390625,
+      "rewards/margins": 0.1953125,
+      "rewards/rejected": -0.1552734375,
+      "step": 20
+    },
+    {
+      "epoch": 0.0038341108058022877,
+      "grad_norm": 14.514374788683362,
+      "learning_rate": 2.4e-06,
+      "logits/chosen": -2.125,
+      "logits/rejected": -1.8046875,
+      "logps/chosen": -232.0,
+      "logps/rejected": -206.0,
+      "loss": 0.5059,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.42578125,
+      "rewards/margins": 0.8671875,
+      "rewards/rejected": -1.296875,
+      "step": 30
+    },
+    {
+      "epoch": 0.005112147741069717,
+      "grad_norm": 15.141785262237963,
+      "learning_rate": 3.2e-06,
+      "logits/chosen": -2.0,
+      "logits/rejected": -1.8046875,
+      "logps/chosen": -210.0,
+      "logps/rejected": -198.0,
+      "loss": 0.4729,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.37109375,
+      "rewards/margins": 1.375,
+      "rewards/rejected": -1.75,
+      "step": 40
+    },
+    {
+      "epoch": 0.006390184676337146,
+      "grad_norm": 13.196147001717241,
+      "learning_rate": 4e-06,
+      "logits/chosen": -1.9609375,
+      "logits/rejected": -1.640625,
+      "logps/chosen": -229.0,
+      "logps/rejected": -209.0,
+      "loss": 0.4255,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.765625,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -2.53125,
+      "step": 50
+    },
+    {
+      "epoch": 0.007668221611604575,
+      "grad_norm": 10.824664501964739,
+      "learning_rate": 4.8e-06,
+      "logits/chosen": -1.8515625,
+      "logits/rejected": -1.53125,
+      "logps/chosen": -234.0,
+      "logps/rejected": -216.0,
+      "loss": 0.419,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.609375,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -4.25,
+      "step": 60
+    },
+    {
+      "epoch": 0.008946258546872005,
+      "grad_norm": 13.387280930116003,
+      "learning_rate": 5.6e-06,
+      "logits/chosen": -1.890625,
+      "logits/rejected": -1.59375,
+      "logps/chosen": -242.0,
+      "logps/rejected": -230.0,
+      "loss": 0.3932,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.421875,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -4.15625,
+      "step": 70
+    },
+    {
+      "epoch": 0.010224295482139434,
+      "grad_norm": 12.469898753244802,
+      "learning_rate": 6.4e-06,
+      "logits/chosen": -1.9140625,
+      "logits/rejected": -1.5625,
+      "logps/chosen": -245.0,
+      "logps/rejected": -229.0,
+      "loss": 0.3623,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.484375,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -4.34375,
+      "step": 80
+    },
+    {
+      "epoch": 0.011502332417406863,
+      "grad_norm": 12.861662288653752,
+      "learning_rate": 7.2e-06,
+      "logits/chosen": -2.0,
+      "logits/rejected": -1.6484375,
+      "logps/chosen": -239.0,
+      "logps/rejected": -244.0,
+      "loss": 0.3818,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.5,
+      "rewards/margins": 2.171875,
+      "rewards/rejected": -5.6875,
+      "step": 90
+    },
+    {
+      "epoch": 0.012780369352674292,
+      "grad_norm": 11.065773265550133,
+      "learning_rate": 8e-06,
+      "logits/chosen": -1.796875,
+      "logits/rejected": -1.359375,
+      "logps/chosen": -253.0,
+      "logps/rejected": -249.0,
+      "loss": 0.3528,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -3.921875,
+      "rewards/margins": 2.390625,
+      "rewards/rejected": -6.3125,
+      "step": 100
+    },
+    {
+      "epoch": 0.014058406287941722,
+      "grad_norm": 11.175931675292244,
+      "learning_rate": 7.627700713964738e-06,
+      "logits/chosen": -2.078125,
+      "logits/rejected": -1.65625,
+      "logps/chosen": -268.0,
+      "logps/rejected": -272.0,
+      "loss": 0.3401,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -5.6875,
+      "rewards/margins": 3.25,
+      "rewards/rejected": -8.9375,
+      "step": 110
+    },
+    {
+      "epoch": 0.01533644322320915,
+      "grad_norm": 11.576569565074632,
+      "learning_rate": 7.3029674334022146e-06,
+      "logits/chosen": -1.859375,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -264.0,
+      "logps/rejected": -274.0,
+      "loss": 0.3178,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -5.78125,
+      "rewards/margins": 3.125,
+      "rewards/rejected": -8.875,
+      "step": 120
+    },
+    {
+      "epoch": 0.01661448015847658,
+      "grad_norm": 11.049695523890255,
+      "learning_rate": 7.016464154456233e-06,
+      "logits/chosen": -1.703125,
+      "logits/rejected": -1.3046875,
+      "logps/chosen": -244.0,
+      "logps/rejected": -280.0,
+      "loss": 0.3579,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -4.65625,
+      "rewards/margins": 2.625,
+      "rewards/rejected": -7.28125,
+      "step": 130
+    },
+    {
+      "epoch": 0.01789251709374401,
+      "grad_norm": 22.185328155457004,
+      "learning_rate": 6.7612340378281325e-06,
+      "logits/chosen": -1.8203125,
+      "logits/rejected": -1.3515625,
+      "logps/chosen": -262.0,
+      "logps/rejected": -260.0,
+      "loss": 0.3328,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -4.375,
+      "rewards/margins": 2.53125,
+      "rewards/rejected": -6.90625,
+      "step": 140
+    },
+    {
+      "epoch": 0.01917055402901144,
+      "grad_norm": 17.28703916612154,
+      "learning_rate": 6.531972647421809e-06,
+      "logits/chosen": -1.8515625,
+      "logits/rejected": -1.578125,
+      "logps/chosen": -268.0,
+      "logps/rejected": -276.0,
+      "loss": 0.3136,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -5.625,
+      "rewards/margins": 3.640625,
+      "rewards/rejected": -9.3125,
+      "step": 150
+    },
+    {
+      "epoch": 0.020448590964278868,
+      "grad_norm": 8.964686735274466,
+      "learning_rate": 6.3245553203367575e-06,
+      "logits/chosen": -1.6328125,
+      "logits/rejected": -1.25,
+      "logps/chosen": -280.0,
+      "logps/rejected": -286.0,
+      "loss": 0.3425,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -5.3125,
+      "rewards/margins": 3.28125,
+      "rewards/rejected": -8.625,
+      "step": 160
+    },
+    {
+      "epoch": 0.021726627899546297,
+      "grad_norm": 9.864423945583349,
+      "learning_rate": 6.135719910778963e-06,
+      "logits/chosen": -1.796875,
+      "logits/rejected": -1.296875,
+      "logps/chosen": -278.0,
+      "logps/rejected": -298.0,
+      "loss": 0.2862,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 3.734375,
+      "rewards/rejected": -9.9375,
+      "step": 170
+    },
+    {
+      "epoch": 0.023004664834813726,
+      "grad_norm": 70.27629195144868,
+      "learning_rate": 5.962847939999439e-06,
+      "logits/chosen": -1.734375,
+      "logits/rejected": -1.296875,
+      "logps/chosen": -278.0,
+      "logps/rejected": -290.0,
+      "loss": 0.3416,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 3.546875,
+      "rewards/rejected": -9.875,
+      "step": 180
+    },
+    {
+      "epoch": 0.024282701770081155,
+      "grad_norm": 13.874329510771819,
+      "learning_rate": 5.803810000880094e-06,
+      "logits/chosen": -1.6875,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -282.0,
+      "logps/rejected": -282.0,
+      "loss": 0.2947,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -5.65625,
+      "rewards/margins": 3.015625,
+      "rewards/rejected": -8.6875,
+      "step": 190
+    },
+    {
+      "epoch": 0.025560738705348585,
+      "grad_norm": 8.125044719458415,
+      "learning_rate": 5.65685424949238e-06,
+      "logits/chosen": -1.6640625,
+      "logits/rejected": -1.3359375,
+      "logps/chosen": -256.0,
+      "logps/rejected": -270.0,
+      "loss": 0.2684,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 3.40625,
+      "rewards/rejected": -10.4375,
+      "step": 200
+    },
+    {
+      "epoch": 0.026838775640616014,
+      "grad_norm": 12.378410623228271,
+      "learning_rate": 5.5205244747388325e-06,
+      "logits/chosen": -1.8046875,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -292.0,
+      "logps/rejected": -308.0,
+      "loss": 0.3271,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -8.1875,
+      "rewards/margins": 3.96875,
+      "rewards/rejected": -12.125,
+      "step": 210
+    },
+    {
+      "epoch": 0.028116812575883443,
+      "grad_norm": 11.605914067555702,
+      "learning_rate": 5.393598899705936e-06,
+      "logits/chosen": -1.609375,
+      "logits/rejected": -1.234375,
+      "logps/chosen": -286.0,
+      "logps/rejected": -304.0,
+      "loss": 0.3309,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 3.1875,
+      "rewards/rejected": -9.375,
+      "step": 220
+    },
+    {
+      "epoch": 0.029394849511150872,
+      "grad_norm": 10.98243489780871,
+      "learning_rate": 5.275043787166296e-06,
+      "logits/chosen": -1.6484375,
+      "logits/rejected": -1.234375,
+      "logps/chosen": -276.0,
+      "logps/rejected": -288.0,
+      "loss": 0.282,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 3.3125,
+      "rewards/rejected": -9.625,
+      "step": 230
+    },
+    {
+      "epoch": 0.0306728864464183,
+      "grad_norm": 12.146537231707752,
+      "learning_rate": 5.163977794943223e-06,
+      "logits/chosen": -1.7109375,
+      "logits/rejected": -1.3515625,
+      "logps/chosen": -306.0,
+      "logps/rejected": -310.0,
+      "loss": 0.357,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 4.625,
+      "rewards/rejected": -12.4375,
+      "step": 240
+    },
+    {
+      "epoch": 0.03195092338168573,
+      "grad_norm": 10.52586401600269,
+      "learning_rate": 5.059644256269407e-06,
+      "logits/chosen": -1.7578125,
+      "logits/rejected": -1.3359375,
+      "logps/chosen": -280.0,
+      "logps/rejected": -296.0,
+      "loss": 0.326,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 3.578125,
+      "rewards/rejected": -10.4375,
+      "step": 250
+    },
+    {
+      "epoch": 0.03322896031695316,
+      "grad_norm": 9.97681998813517,
+      "learning_rate": 4.961389383568338e-06,
+      "logits/chosen": -1.6484375,
+      "logits/rejected": -1.1796875,
+      "logps/chosen": -276.0,
+      "logps/rejected": -296.0,
+      "loss": 0.2686,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -6.0625,
+      "rewards/margins": 3.5,
+      "rewards/rejected": -9.5625,
+      "step": 260
+    },
+    {
+      "epoch": 0.03450699725222059,
+      "grad_norm": 11.613233025975493,
+      "learning_rate": 4.8686449556014755e-06,
+      "logits/chosen": -1.625,
+      "logits/rejected": -1.109375,
+      "logps/chosen": -290.0,
+      "logps/rejected": -300.0,
+      "loss": 0.2771,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 4.53125,
+      "rewards/rejected": -11.625,
+      "step": 270
+    },
+    {
+      "epoch": 0.03578503418748802,
+      "grad_norm": 9.745923970722576,
+      "learning_rate": 4.780914437337574e-06,
+      "logits/chosen": -1.4453125,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -280.0,
+      "logps/rejected": -306.0,
+      "loss": 0.2908,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 4.15625,
+      "rewards/rejected": -11.4375,
+      "step": 280
+    },
+    {
+      "epoch": 0.03706307112275545,
+      "grad_norm": 31.165695435477424,
+      "learning_rate": 4.697761756117627e-06,
+      "logits/chosen": -1.5078125,
+      "logits/rejected": -0.99609375,
+      "logps/chosen": -288.0,
+      "logps/rejected": -288.0,
+      "loss": 0.3195,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 3.328125,
+      "rewards/rejected": -10.9375,
+      "step": 290
+    },
+    {
+      "epoch": 0.03834110805802288,
+      "grad_norm": 8.224947268900738,
+      "learning_rate": 4.618802153517006e-06,
+      "logits/chosen": -1.578125,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -284.0,
+      "logps/rejected": -294.0,
+      "loss": 0.2811,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -5.78125,
+      "rewards/margins": 3.6875,
+      "rewards/rejected": -9.5,
+      "step": 300
+    },
+    {
+      "epoch": 0.039619144993290306,
+      "grad_norm": 10.714304708458737,
+      "learning_rate": 4.543694673976518e-06,
+      "logits/chosen": -1.5078125,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -278.0,
+      "logps/rejected": -308.0,
+      "loss": 0.2862,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 4.25,
+      "rewards/rejected": -11.1875,
+      "step": 310
+    },
+    {
+      "epoch": 0.040897181928557735,
+      "grad_norm": 12.213286736600438,
+      "learning_rate": 4.472135954999579e-06,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.1328125,
+      "logps/chosen": -276.0,
+      "logps/rejected": -308.0,
+      "loss": 0.2748,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 4.25,
+      "rewards/rejected": -11.375,
+      "step": 320
+    },
+    {
+      "epoch": 0.042175218863825165,
+      "grad_norm": 8.816716689927773,
+      "learning_rate": 4.403855060505443e-06,
+      "logits/chosen": -1.5703125,
+      "logits/rejected": -1.1328125,
+      "logps/chosen": -300.0,
+      "logps/rejected": -320.0,
+      "loss": 0.2611,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -8.0625,
+      "rewards/margins": 4.21875,
+      "rewards/rejected": -12.3125,
+      "step": 330
+    },
+    {
+      "epoch": 0.043453255799092594,
+      "grad_norm": 7.5180022473301396,
+      "learning_rate": 4.338609156373123e-06,
+      "logits/chosen": -1.5234375,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -308.0,
+      "logps/rejected": -304.0,
+      "loss": 0.284,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -8.0625,
+      "rewards/margins": 4.09375,
+      "rewards/rejected": -12.1875,
+      "step": 340
+    },
+    {
+      "epoch": 0.04473129273436002,
+      "grad_norm": 8.837953203359074,
+      "learning_rate": 4.27617987059879e-06,
+      "logits/chosen": -1.5859375,
+      "logits/rejected": -1.1328125,
+      "logps/chosen": -278.0,
+      "logps/rejected": -320.0,
+      "loss": 0.2771,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -8.1875,
+      "rewards/margins": 4.46875,
+      "rewards/rejected": -12.625,
+      "step": 350
+    },
+    {
+      "epoch": 0.04600932966962745,
+      "grad_norm": 12.418392338038274,
+      "learning_rate": 4.216370213557839e-06,
+      "logits/chosen": -1.625,
+      "logits/rejected": -1.203125,
+      "logps/chosen": -314.0,
+      "logps/rejected": -314.0,
+      "loss": 0.3432,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -9.125,
+      "rewards/margins": 4.25,
+      "rewards/rejected": -13.375,
+      "step": 360
+    },
+    {
+      "epoch": 0.04728736660489488,
+      "grad_norm": 11.179079166019818,
+      "learning_rate": 4.15900195928029e-06,
+      "logits/chosen": -1.578125,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -324.0,
+      "logps/rejected": -340.0,
+      "loss": 0.2558,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -9.9375,
+      "rewards/margins": 4.90625,
+      "rewards/rejected": -14.875,
+      "step": 370
+    },
+    {
+      "epoch": 0.04856540354016231,
+      "grad_norm": 10.245281575173605,
+      "learning_rate": 4.103913408340617e-06,
+      "logits/chosen": -1.5,
+      "logits/rejected": -0.99609375,
+      "logps/chosen": -276.0,
+      "logps/rejected": -314.0,
+      "loss": 0.2193,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -8.4375,
+      "rewards/margins": 4.40625,
+      "rewards/rejected": -12.8125,
+      "step": 380
+    },
+    {
+      "epoch": 0.04984344047542974,
+      "grad_norm": 11.694711875966762,
+      "learning_rate": 4.050957468334666e-06,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -334.0,
+      "logps/rejected": -318.0,
+      "loss": 0.2731,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -10.625,
+      "rewards/margins": 4.09375,
+      "rewards/rejected": -14.75,
+      "step": 390
+    },
+    {
+      "epoch": 0.05112147741069717,
+      "grad_norm": 9.12136191566598,
+      "learning_rate": 4e-06,
+      "logits/chosen": -1.484375,
+      "logits/rejected": -1.1875,
+      "logps/chosen": -310.0,
+      "logps/rejected": -336.0,
+      "loss": 0.2437,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -9.125,
+      "rewards/margins": 3.90625,
+      "rewards/rejected": -13.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.0523995143459646,
+      "grad_norm": 9.398967109872249,
+      "learning_rate": 3.950918386598359e-06,
+      "logits/chosen": -1.53125,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -302.0,
+      "logps/rejected": -332.0,
+      "loss": 0.2542,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -8.6875,
+      "rewards/margins": 4.78125,
+      "rewards/rejected": -13.4375,
+      "step": 410
+    },
+    {
+      "epoch": 0.05367755128123203,
+      "grad_norm": 10.947419724503769,
+      "learning_rate": 3.903600291794132e-06,
+      "logits/chosen": -1.4921875,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -310.0,
+      "logps/rejected": -328.0,
+      "loss": 0.2464,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -9.375,
+      "rewards/margins": 4.15625,
+      "rewards/rejected": -13.5,
+      "step": 420
+    },
+    {
+      "epoch": 0.05495558821649946,
+      "grad_norm": 7.357391686206119,
+      "learning_rate": 3.857942577363297e-06,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -1.1171875,
+      "logps/chosen": -318.0,
+      "logps/rejected": -318.0,
+      "loss": 0.2527,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -10.625,
+      "rewards/margins": 3.890625,
+      "rewards/rejected": -14.5,
+      "step": 430
+    },
+    {
+      "epoch": 0.056233625151766886,
+      "grad_norm": 9.167491287146401,
+      "learning_rate": 3.813850356982369e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -324.0,
+      "logps/rejected": -354.0,
+      "loss": 0.303,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -10.75,
+      "rewards/margins": 4.65625,
+      "rewards/rejected": -15.4375,
+      "step": 440
+    },
+    {
+      "epoch": 0.057511662087034315,
+      "grad_norm": 11.92715381879063,
+      "learning_rate": 3.7712361663282537e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -322.0,
+      "logps/rejected": -342.0,
+      "loss": 0.2442,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -10.875,
+      "rewards/margins": 4.625,
+      "rewards/rejected": -15.5,
+      "step": 450
+    },
+    {
+      "epoch": 0.058789699022301745,
+      "grad_norm": 9.152175337110775,
+      "learning_rate": 3.730019232961255e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -294.0,
+      "logps/rejected": -332.0,
+      "loss": 0.255,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -10.375,
+      "rewards/margins": 4.5,
+      "rewards/rejected": -14.875,
+      "step": 460
+    },
+    {
+      "epoch": 0.060067735957569174,
+      "grad_norm": 8.479514710254218,
+      "learning_rate": 3.6901248321155403e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -314.0,
+      "logps/rejected": -348.0,
+      "loss": 0.2572,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -11.1875,
+      "rewards/margins": 4.5625,
+      "rewards/rejected": -15.8125,
+      "step": 470
+    },
+    {
+      "epoch": 0.0613457728928366,
+      "grad_norm": 11.294433704433228,
+      "learning_rate": 3.6514837167011073e-06,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -0.97265625,
+      "logps/chosen": -336.0,
+      "logps/rejected": -354.0,
+      "loss": 0.2726,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -10.9375,
+      "rewards/margins": 4.71875,
+      "rewards/rejected": -15.625,
+      "step": 480
+    },
+    {
+      "epoch": 0.06262380982810403,
+      "grad_norm": 8.721780734979998,
+      "learning_rate": 3.6140316116210052e-06,
+      "logits/chosen": -1.4609375,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -314.0,
+      "logps/rejected": -344.0,
+      "loss": 0.2384,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -10.4375,
+      "rewards/margins": 5.34375,
+      "rewards/rejected": -15.75,
+      "step": 490
+    },
+    {
+      "epoch": 0.06390184676337146,
+      "grad_norm": 6.681639670730452,
+      "learning_rate": 3.5777087639996634e-06,
+      "logits/chosen": -1.5546875,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -304.0,
+      "logps/rejected": -332.0,
+      "loss": 0.2762,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -10.0625,
+      "rewards/margins": 4.34375,
+      "rewards/rejected": -14.4375,
+      "step": 500
+    },
+    {
+      "epoch": 0.06517988369863889,
+      "grad_norm": 8.995120822345854,
+      "learning_rate": 3.5424595421603814e-06,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -0.99609375,
+      "logps/chosen": -328.0,
+      "logps/rejected": -338.0,
+      "loss": 0.2889,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -11.5625,
+      "rewards/margins": 4.5,
+      "rewards/rejected": -16.0,
+      "step": 510
+    },
+    {
+      "epoch": 0.06645792063390632,
+      "grad_norm": 10.770561253198553,
+      "learning_rate": 3.5082320772281165e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.9921875,
+      "logps/chosen": -308.0,
+      "logps/rejected": -346.0,
+      "loss": 0.252,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -11.5,
+      "rewards/margins": 3.984375,
+      "rewards/rejected": -15.4375,
+      "step": 520
+    },
+    {
+      "epoch": 0.06773595756917375,
+      "grad_norm": 6.079296570005446,
+      "learning_rate": 3.474977942104555e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.875,
+      "logps/chosen": -324.0,
+      "logps/rejected": -348.0,
+      "loss": 0.3093,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -10.625,
+      "rewards/margins": 4.6875,
+      "rewards/rejected": -15.3125,
+      "step": 530
+    },
+    {
+      "epoch": 0.06901399450444118,
+      "grad_norm": 12.62206416090569,
+      "learning_rate": 3.442651863295481e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -326.0,
+      "logps/rejected": -356.0,
+      "loss": 0.2329,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -10.4375,
+      "rewards/margins": 5.03125,
+      "rewards/rejected": -15.5,
+      "step": 540
+    },
+    {
+      "epoch": 0.07029203143970861,
+      "grad_norm": 9.176567797536125,
+      "learning_rate": 3.4112114616897665e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -326.0,
+      "logps/rejected": -362.0,
+      "loss": 0.2517,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -12.125,
+      "rewards/margins": 4.78125,
+      "rewards/rejected": -16.875,
+      "step": 550
+    },
+    {
+      "epoch": 0.07157006837497604,
+      "grad_norm": 8.74022454073919,
+      "learning_rate": 3.3806170189140663e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -362.0,
+      "logps/rejected": -378.0,
+      "loss": 0.2659,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -13.5625,
+      "rewards/margins": 4.40625,
+      "rewards/rejected": -18.0,
+      "step": 560
+    },
+    {
+      "epoch": 0.07284810531024347,
+      "grad_norm": 10.773762894101269,
+      "learning_rate": 3.350831266333564e-06,
+      "logits/chosen": -1.4765625,
+      "logits/rejected": -0.97265625,
+      "logps/chosen": -332.0,
+      "logps/rejected": -354.0,
+      "loss": 0.2567,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -13.0625,
+      "rewards/margins": 4.6875,
+      "rewards/rejected": -17.75,
+      "step": 570
+    },
+    {
+      "epoch": 0.0741261422455109,
+      "grad_norm": 8.112499776796634,
+      "learning_rate": 3.3218191941495984e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -326.0,
+      "logps/rejected": -338.0,
+      "loss": 0.2485,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -11.25,
+      "rewards/margins": 4.40625,
+      "rewards/rejected": -15.625,
+      "step": 580
+    },
+    {
+      "epoch": 0.07540417918077832,
+      "grad_norm": 6.874604479059108,
+      "learning_rate": 3.293547878370473e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -338.0,
+      "logps/rejected": -362.0,
+      "loss": 0.2821,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -11.4375,
+      "rewards/margins": 4.28125,
+      "rewards/rejected": -15.75,
+      "step": 590
+    },
+    {
+      "epoch": 0.07668221611604575,
+      "grad_norm": 6.0927407879064335,
+      "learning_rate": 3.2659863237109044e-06,
+      "logits/chosen": -1.4609375,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -346.0,
+      "logps/rejected": -374.0,
+      "loss": 0.2364,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -11.625,
+      "rewards/margins": 4.6875,
+      "rewards/rejected": -16.375,
+      "step": 600
+    },
+    {
+      "epoch": 0.07796025305131318,
+      "grad_norm": 9.236445366800993,
+      "learning_rate": 3.239105320715664e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -342.0,
+      "logps/rejected": -380.0,
+      "loss": 0.2465,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -12.25,
+      "rewards/margins": 5.09375,
+      "rewards/rejected": -17.375,
+      "step": 610
+    },
+    {
+      "epoch": 0.07923828998658061,
+      "grad_norm": 9.205637729690743,
+      "learning_rate": 3.2128773156099956e-06,
+      "logits/chosen": -1.4453125,
+      "logits/rejected": -1.1171875,
+      "logps/chosen": -348.0,
+      "logps/rejected": -358.0,
+      "loss": 0.2138,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -12.0,
+      "rewards/margins": 4.78125,
+      "rewards/rejected": -16.75,
+      "step": 620
+    },
+    {
+      "epoch": 0.08051632692184804,
+      "grad_norm": 7.78576931618227,
+      "learning_rate": 3.187276291558383e-06,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -322.0,
+      "logps/rejected": -338.0,
+      "loss": 0.2459,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -11.3125,
+      "rewards/margins": 4.84375,
+      "rewards/rejected": -16.125,
+      "step": 630
+    },
+    {
+      "epoch": 0.08179436385711547,
+      "grad_norm": 6.241479026309425,
+      "learning_rate": 3.1622776601683788e-06,
+      "logits/chosen": -1.5078125,
+      "logits/rejected": -1.109375,
+      "logps/chosen": -340.0,
+      "logps/rejected": -368.0,
+      "loss": 0.2207,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -11.375,
+      "rewards/margins": 4.90625,
+      "rewards/rejected": -16.25,
+      "step": 640
+    },
+    {
+      "epoch": 0.0830724007923829,
+      "grad_norm": 8.366680007747462,
+      "learning_rate": 3.1378581622109444e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -308.0,
+      "logps/rejected": -362.0,
+      "loss": 0.2124,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -10.75,
+      "rewards/margins": 4.96875,
+      "rewards/rejected": -15.6875,
+      "step": 650
+    },
+    {
+      "epoch": 0.08435043772765033,
+      "grad_norm": 7.6573367267194214,
+      "learning_rate": 3.113995776646092e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -320.0,
+      "logps/rejected": -344.0,
+      "loss": 0.245,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -11.1875,
+      "rewards/margins": 5.15625,
+      "rewards/rejected": -16.375,
+      "step": 660
+    },
+    {
+      "epoch": 0.08562847466291776,
+      "grad_norm": 8.419148525770197,
+      "learning_rate": 3.090669637145023e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -322.0,
+      "logps/rejected": -360.0,
+      "loss": 0.2788,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -11.125,
+      "rewards/margins": 4.5625,
+      "rewards/rejected": -15.6875,
+      "step": 670
+    },
+    {
+      "epoch": 0.08690651159818519,
+      "grad_norm": 9.051815254003762,
+      "learning_rate": 3.0678599553894814e-06,
+      "logits/chosen": -1.4453125,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -336.0,
+      "logps/rejected": -360.0,
+      "loss": 0.2477,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -10.3125,
+      "rewards/margins": 4.9375,
+      "rewards/rejected": -15.25,
+      "step": 680
+    },
+    {
+      "epoch": 0.08818454853345262,
+      "grad_norm": 8.004690186050613,
+      "learning_rate": 3.0455479505075235e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -316.0,
+      "logps/rejected": -342.0,
+      "loss": 0.2206,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -10.5,
+      "rewards/margins": 4.8125,
+      "rewards/rejected": -15.3125,
+      "step": 690
+    },
+    {
+      "epoch": 0.08946258546872005,
+      "grad_norm": 9.039388107114702,
+      "learning_rate": 3.0237157840738173e-06,
+      "logits/chosen": -1.5,
+      "logits/rejected": -1.140625,
+      "logps/chosen": -322.0,
+      "logps/rejected": -354.0,
+      "loss": 0.2307,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -11.5625,
+      "rewards/margins": 4.8125,
+      "rewards/rejected": -16.375,
+      "step": 700
+    },
+    {
+      "epoch": 0.09074062240398748,
+      "grad_norm": 8.831137542819324,
+      "learning_rate": 3.002346500163206e-06,
+      "logits/chosen": -1.4609375,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -324.0,
+      "logps/rejected": -348.0,
+      "loss": 0.2564,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -11.5,
+      "rewards/margins": 4.4375,
+      "rewards/rejected": -15.9375,
+      "step": 710
+    },
+    {
+      "epoch": 0.0920186593392549,
+      "grad_norm": 12.35639948281655,
+      "learning_rate": 2.9814239699997195e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -328.0,
+      "logps/rejected": -350.0,
+      "loss": 0.2241,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -10.75,
+      "rewards/margins": 5.125,
+      "rewards/rejected": -15.875,
+      "step": 720
+    },
+    {
+      "epoch": 0.09329669627452233,
+      "grad_norm": 11.906540090311804,
+      "learning_rate": 2.9609328407904207e-06,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -314.0,
+      "logps/rejected": -336.0,
+      "loss": 0.2208,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -11.375,
+      "rewards/margins": 5.59375,
+      "rewards/rejected": -16.875,
+      "step": 730
+    },
+    {
+      "epoch": 0.09457473320978976,
+      "grad_norm": 9.910361601015193,
+      "learning_rate": 2.940858488375231e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -348.0,
+      "logps/rejected": -366.0,
+      "loss": 0.2578,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -11.625,
+      "rewards/margins": 5.40625,
+      "rewards/rejected": -17.0,
+      "step": 740
+    },
+    {
+      "epoch": 0.09585277014505719,
+      "grad_norm": 7.664973746403084,
+      "learning_rate": 2.9211869733608857e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -308.0,
+      "logps/rejected": -336.0,
+      "loss": 0.2206,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -10.75,
+      "rewards/margins": 5.09375,
+      "rewards/rejected": -15.875,
+      "step": 750
+    },
+    {
+      "epoch": 0.09713080708032462,
+      "grad_norm": 8.923513183014242,
+      "learning_rate": 2.901905000440047e-06,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -332.0,
+      "logps/rejected": -370.0,
+      "loss": 0.2159,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -10.9375,
+      "rewards/margins": 4.5,
+      "rewards/rejected": -15.4375,
+      "step": 760
+    },
+    {
+      "epoch": 0.09840884401559205,
+      "grad_norm": 5.936833041892835,
+      "learning_rate": 2.8829998806257885e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -322.0,
+      "logps/rejected": -356.0,
+      "loss": 0.2303,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -10.75,
+      "rewards/margins": 4.875,
+      "rewards/rejected": -15.625,
+      "step": 770
+    },
+    {
+      "epoch": 0.09968688095085948,
+      "grad_norm": 8.565353722787313,
+      "learning_rate": 2.8644594961577314e-06,
+      "logits/chosen": -1.4609375,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -340.0,
+      "logps/rejected": -376.0,
+      "loss": 0.2265,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -12.1875,
+      "rewards/margins": 5.96875,
+      "rewards/rejected": -18.125,
+      "step": 780
+    },
+    {
+      "epoch": 0.10096491788612691,
+      "grad_norm": 6.009507757496106,
+      "learning_rate": 2.84627226785928e-06,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.125,
+      "logps/chosen": -362.0,
+      "logps/rejected": -380.0,
+      "loss": 0.2396,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 4.5625,
+      "rewards/rejected": -19.0,
+      "step": 790
+    },
+    {
+      "epoch": 0.10224295482139434,
+      "grad_norm": 8.55104947400273,
+      "learning_rate": 2.82842712474619e-06,
+      "logits/chosen": -1.484375,
+      "logits/rejected": -0.95703125,
+      "logps/chosen": -352.0,
+      "logps/rejected": -388.0,
+      "loss": 0.2426,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -14.125,
+      "rewards/margins": 5.21875,
+      "rewards/rejected": -19.375,
+      "step": 800
+    },
+    {
+      "epoch": 0.10352099175666177,
+      "grad_norm": 10.05070126379492,
+      "learning_rate": 2.810913475705226e-06,
+      "logits/chosen": -1.4609375,
+      "logits/rejected": -1.109375,
+      "logps/chosen": -346.0,
+      "logps/rejected": -370.0,
+      "loss": 0.2045,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -13.5,
+      "rewards/margins": 5.59375,
+      "rewards/rejected": -19.0,
+      "step": 810
+    },
+    {
+      "epoch": 0.1047990286919292,
+      "grad_norm": 7.527595357248868,
+      "learning_rate": 2.7937211830783128e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.1171875,
+      "logps/chosen": -338.0,
+      "logps/rejected": -436.0,
+      "loss": 0.2235,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.3125,
+      "rewards/margins": 10.6875,
+      "rewards/rejected": -24.0,
+      "step": 820
+    },
+    {
+      "epoch": 0.10607706562719663,
+      "grad_norm": 8.530666074380585,
+      "learning_rate": 2.776840538002493e-06,
+      "logits/chosen": -1.453125,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -354.0,
+      "logps/rejected": -384.0,
+      "loss": 0.2654,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -14.125,
+      "rewards/margins": 6.09375,
+      "rewards/rejected": -20.25,
+      "step": 830
+    },
+    {
+      "epoch": 0.10735510256246406,
+      "grad_norm": 10.787495247687803,
+      "learning_rate": 2.7602622373694163e-06,
+      "logits/chosen": -1.4765625,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -354.0,
+      "logps/rejected": -388.0,
+      "loss": 0.2232,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -13.0625,
+      "rewards/margins": 5.46875,
+      "rewards/rejected": -18.5,
+      "step": 840
+    },
+    {
+      "epoch": 0.10863313949773148,
+      "grad_norm": 7.8359825960942295,
+      "learning_rate": 2.743977362280141e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -346.0,
+      "logps/rejected": -402.0,
+      "loss": 0.2467,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -12.875,
+      "rewards/margins": 5.40625,
+      "rewards/rejected": -18.25,
+      "step": 850
+    },
+    {
+      "epoch": 0.10991117643299891,
+      "grad_norm": 13.447577856534576,
+      "learning_rate": 2.7279773578818937e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -342.0,
+      "logps/rejected": -388.0,
+      "loss": 0.1986,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -13.25,
+      "rewards/margins": 5.46875,
+      "rewards/rejected": -18.75,
+      "step": 860
+    },
+    {
+      "epoch": 0.11118921336826634,
+      "grad_norm": 7.041609505878959,
+      "learning_rate": 2.7122540144832417e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -354.0,
+      "logps/rejected": -386.0,
+      "loss": 0.2369,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -15.0625,
+      "rewards/margins": 5.21875,
+      "rewards/rejected": -20.25,
+      "step": 870
+    },
+    {
+      "epoch": 0.11246725030353377,
+      "grad_norm": 7.781276448490546,
+      "learning_rate": 2.696799449852968e-06,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -360.0,
+      "logps/rejected": -406.0,
+      "loss": 0.2232,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 5.375,
+      "rewards/rejected": -19.875,
+      "step": 880
+    },
+    {
+      "epoch": 0.1137452872388012,
+      "grad_norm": 9.348557172912107,
+      "learning_rate": 2.6816060926159636e-06,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -388.0,
+      "logps/rejected": -416.0,
+      "loss": 0.2618,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -14.3125,
+      "rewards/margins": 5.84375,
+      "rewards/rejected": -20.125,
+      "step": 890
+    },
+    {
+      "epoch": 0.11502332417406863,
+      "grad_norm": 5.835346764796168,
+      "learning_rate": 2.6666666666666664e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.92578125,
+      "logps/chosen": -342.0,
+      "logps/rejected": -382.0,
+      "loss": 0.2619,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -13.3125,
+      "rewards/margins": 5.1875,
+      "rewards/rejected": -18.5,
+      "step": 900
+    },
+    {
+      "epoch": 0.11630136110933606,
+      "grad_norm": 7.688310083217483,
+      "learning_rate": 2.6519741765271837e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -350.0,
+      "logps/rejected": -378.0,
+      "loss": 0.2111,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -12.25,
+      "rewards/margins": 5.0,
+      "rewards/rejected": -17.25,
+      "step": 910
+    },
+    {
+      "epoch": 0.11757939804460349,
+      "grad_norm": 9.903656681697754,
+      "learning_rate": 2.637521893583148e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -332.0,
+      "logps/rejected": -352.0,
+      "loss": 0.2695,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -12.5,
+      "rewards/margins": 5.78125,
+      "rewards/rejected": -18.375,
+      "step": 920
+    },
+    {
+      "epoch": 0.11885743497987092,
+      "grad_norm": 6.854873808238501,
+      "learning_rate": 2.6233033431358115e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -344.0,
+      "logps/rejected": -388.0,
+      "loss": 0.216,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -13.3125,
+      "rewards/margins": 5.0625,
+      "rewards/rejected": -18.375,
+      "step": 930
+    },
+    {
+      "epoch": 0.12013547191513835,
+      "grad_norm": 8.200143990076109,
+      "learning_rate": 2.6093122922137685e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -332.0,
+      "logps/rejected": -370.0,
+      "loss": 0.2064,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -11.875,
+      "rewards/margins": 5.40625,
+      "rewards/rejected": -17.25,
+      "step": 940
+    },
+    {
+      "epoch": 0.12141350885040578,
+      "grad_norm": 10.801289654465965,
+      "learning_rate": 2.5955427380922006e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.76953125,
+      "logps/chosen": -340.0,
+      "logps/rejected": -392.0,
+      "loss": 0.2281,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -12.5,
+      "rewards/margins": 5.625,
+      "rewards/rejected": -18.125,
+      "step": 950
+    },
+    {
+      "epoch": 0.1226915457856732,
+      "grad_norm": 7.4824512674466614,
+      "learning_rate": 2.5819888974716113e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -338.0,
+      "logps/rejected": -370.0,
+      "loss": 0.2642,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -13.75,
+      "rewards/margins": 5.15625,
+      "rewards/rejected": -18.875,
+      "step": 960
+    },
+    {
+      "epoch": 0.12396958272094064,
+      "grad_norm": 6.19203289064979,
+      "learning_rate": 2.5686451962717425e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -342.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1832,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -13.5,
+      "rewards/margins": 5.46875,
+      "rewards/rejected": -19.0,
+      "step": 970
+    },
+    {
+      "epoch": 0.12524761965620806,
+      "grad_norm": 10.382349426307343,
+      "learning_rate": 2.5555062599997596e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -352.0,
+      "logps/rejected": -372.0,
+      "loss": 0.2144,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -12.625,
+      "rewards/margins": 5.40625,
+      "rewards/rejected": -18.0,
+      "step": 980
+    },
+    {
+      "epoch": 0.1265256565914755,
+      "grad_norm": 7.587426630074104,
+      "learning_rate": 2.5425669046549126e-06,
+      "logits/chosen": -1.1328125,
+      "logits/rejected": -0.765625,
+      "logps/chosen": -308.0,
+      "logps/rejected": -354.0,
+      "loss": 0.1975,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -11.75,
+      "rewards/margins": 5.40625,
+      "rewards/rejected": -17.25,
+      "step": 990
+    },
+    {
+      "epoch": 0.12780369352674292,
+      "grad_norm": 10.223393637584227,
+      "learning_rate": 2.5298221281347034e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.82421875,
+      "logps/chosen": -330.0,
+      "logps/rejected": -364.0,
+      "loss": 0.1985,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -11.625,
+      "rewards/margins": 5.25,
+      "rewards/rejected": -16.875,
+      "step": 1000
+    },
+    {
+      "epoch": 0.12908173046201035,
+      "grad_norm": 10.553274672351115,
+      "learning_rate": 2.5172671021102103e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.765625,
+      "logps/chosen": -330.0,
+      "logps/rejected": -346.0,
+      "loss": 0.2281,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -11.9375,
+      "rewards/margins": 5.34375,
+      "rewards/rejected": -17.25,
+      "step": 1010
+    },
+    {
+      "epoch": 0.13035976739727778,
+      "grad_norm": 8.746281719111414,
+      "learning_rate": 2.504897164340598e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -314.0,
+      "logps/rejected": -356.0,
+      "loss": 0.2421,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -11.5625,
+      "rewards/margins": 5.15625,
+      "rewards/rejected": -16.75,
+      "step": 1020
+    },
+    {
+      "epoch": 0.1316378043325452,
+      "grad_norm": 11.388130950550568,
+      "learning_rate": 2.492707811399023e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -320.0,
+      "logps/rejected": -358.0,
+      "loss": 0.217,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -11.8125,
+      "rewards/margins": 4.875,
+      "rewards/rejected": -16.75,
+      "step": 1030
+    },
+    {
+      "epoch": 0.13291584126781264,
+      "grad_norm": 8.179183929917569,
+      "learning_rate": 2.480694691784169e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.82421875,
+      "logps/chosen": -326.0,
+      "logps/rejected": -382.0,
+      "loss": 0.1904,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -12.625,
+      "rewards/margins": 5.15625,
+      "rewards/rejected": -17.75,
+      "step": 1040
+    },
+    {
+      "epoch": 0.13419387820308007,
+      "grad_norm": 8.452079931882126,
+      "learning_rate": 2.4688535993934706e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.95703125,
+      "logps/chosen": -360.0,
+      "logps/rejected": -376.0,
+      "loss": 0.2162,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -12.5625,
+      "rewards/margins": 5.46875,
+      "rewards/rejected": -18.0,
+      "step": 1050
+    },
+    {
+      "epoch": 0.1354719151383475,
+      "grad_norm": 10.080003508360383,
+      "learning_rate": 2.457180467335805e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -348.0,
+      "logps/rejected": -376.0,
+      "loss": 0.2009,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -12.5625,
+      "rewards/margins": 6.125,
+      "rewards/rejected": -18.75,
+      "step": 1060
+    },
+    {
+      "epoch": 0.13674995207361493,
+      "grad_norm": 4.930020871590482,
+      "learning_rate": 2.4456713620629725e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -354.0,
+      "logps/rejected": -380.0,
+      "loss": 0.1887,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -13.8125,
+      "rewards/margins": 5.125,
+      "rewards/rejected": -18.875,
+      "step": 1070
+    },
+    {
+      "epoch": 0.13802798900888236,
+      "grad_norm": 12.328974970402871,
+      "learning_rate": 2.4343224778007378e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -326.0,
+      "logps/rejected": -378.0,
+      "loss": 0.1887,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -12.6875,
+      "rewards/margins": 5.84375,
+      "rewards/rejected": -18.5,
+      "step": 1080
+    },
+    {
+      "epoch": 0.13930602594414979,
+      "grad_norm": 7.123976380223215,
+      "learning_rate": 2.4231301312615306e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -336.0,
+      "logps/rejected": -368.0,
+      "loss": 0.2072,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -11.9375,
+      "rewards/margins": 5.5625,
+      "rewards/rejected": -17.5,
+      "step": 1090
+    },
+    {
+      "epoch": 0.14058406287941722,
+      "grad_norm": 10.348894955643111,
+      "learning_rate": 2.412090756622109e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -338.0,
+      "logps/rejected": -366.0,
+      "loss": 0.1881,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -12.375,
+      "rewards/margins": 5.65625,
+      "rewards/rejected": -18.0,
+      "step": 1100
+    },
+    {
+      "epoch": 0.14186209981468464,
+      "grad_norm": 9.555252360705285,
+      "learning_rate": 2.401200900750657e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -324.0,
+      "logps/rejected": -360.0,
+      "loss": 0.2139,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -12.6875,
+      "rewards/margins": 5.96875,
+      "rewards/rejected": -18.625,
+      "step": 1110
+    },
+    {
+      "epoch": 0.14314013674995207,
+      "grad_norm": 7.775056482706318,
+      "learning_rate": 2.390457218668787e-06,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -350.0,
+      "logps/rejected": -370.0,
+      "loss": 0.2201,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -13.875,
+      "rewards/margins": 4.5625,
+      "rewards/rejected": -18.375,
+      "step": 1120
+    },
+    {
+      "epoch": 0.1444181736852195,
+      "grad_norm": 7.781531804026228,
+      "learning_rate": 2.379856469234918e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.70703125,
+      "logps/chosen": -368.0,
+      "logps/rejected": -376.0,
+      "loss": 0.2176,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -14.375,
+      "rewards/margins": 5.0625,
+      "rewards/rejected": -19.375,
+      "step": 1130
+    },
+    {
+      "epoch": 0.14569621062048693,
+      "grad_norm": 7.959265653617074,
+      "learning_rate": 2.369395511036369e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -1.0,
+      "logps/chosen": -352.0,
+      "logps/rejected": -376.0,
+      "loss": 0.2298,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -13.625,
+      "rewards/margins": 5.21875,
+      "rewards/rejected": -18.875,
+      "step": 1140
+    },
+    {
+      "epoch": 0.14697424755575436,
+      "grad_norm": 7.77791723985759,
+      "learning_rate": 2.359071298478354e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -348.0,
+      "logps/rejected": -378.0,
+      "loss": 0.2135,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -12.9375,
+      "rewards/margins": 5.28125,
+      "rewards/rejected": -18.25,
+      "step": 1150
+    },
+    {
+      "epoch": 0.1482522844910218,
+      "grad_norm": 8.223034988982452,
+      "learning_rate": 2.3488808780588137e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -338.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1996,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -12.8125,
+      "rewards/margins": 5.75,
+      "rewards/rejected": -18.625,
+      "step": 1160
+    },
+    {
+      "epoch": 0.14953032142628922,
+      "grad_norm": 5.7848293271122255,
+      "learning_rate": 2.3388213848187446e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.0,
+      "logps/chosen": -334.0,
+      "logps/rejected": -370.0,
+      "loss": 0.1698,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -13.1875,
+      "rewards/margins": 5.40625,
+      "rewards/rejected": -18.625,
+      "step": 1170
+    },
+    {
+      "epoch": 0.15080835836155665,
+      "grad_norm": 8.164688622869864,
+      "learning_rate": 2.328890038958328e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -348.0,
+      "logps/rejected": -386.0,
+      "loss": 0.1921,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -13.3125,
+      "rewards/margins": 5.96875,
+      "rewards/rejected": -19.25,
+      "step": 1180
+    },
+    {
+      "epoch": 0.15208639529682408,
+      "grad_norm": 8.04337169941588,
+      "learning_rate": 2.3190841426097937e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.171875,
+      "logps/chosen": -338.0,
+      "logps/rejected": -384.0,
+      "loss": 0.2115,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.0,
+      "rewards/margins": 5.53125,
+      "rewards/rejected": -19.5,
+      "step": 1190
+    },
+    {
+      "epoch": 0.1533644322320915,
+      "grad_norm": 9.703077665095819,
+      "learning_rate": 2.309401076758503e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -348.0,
+      "logps/rejected": -368.0,
+      "loss": 0.2356,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -12.6875,
+      "rewards/margins": 5.09375,
+      "rewards/rejected": -17.75,
+      "step": 1200
+    },
+    {
+      "epoch": 0.15464246916735894,
+      "grad_norm": 8.91817264676939,
+      "learning_rate": 2.299838298304276e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -324.0,
+      "logps/rejected": -374.0,
+      "loss": 0.1952,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -12.0,
+      "rewards/margins": 5.625,
+      "rewards/rejected": -17.625,
+      "step": 1210
+    },
+    {
+      "epoch": 0.15592050610262637,
+      "grad_norm": 8.09612595486752,
+      "learning_rate": 2.2903933372554728e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -334.0,
+      "logps/rejected": -384.0,
+      "loss": 0.2129,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -13.4375,
+      "rewards/margins": 4.96875,
+      "rewards/rejected": -18.375,
+      "step": 1220
+    },
+    {
+      "epoch": 0.1571985430378938,
+      "grad_norm": 8.40687465517894,
+      "learning_rate": 2.281063794048804e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -356.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1936,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -14.0,
+      "rewards/margins": 5.03125,
+      "rewards/rejected": -19.0,
+      "step": 1230
+    },
+    {
+      "epoch": 0.15847657997316122,
+      "grad_norm": 38.143447949912485,
+      "learning_rate": 2.271847336988259e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -346.0,
+      "logps/rejected": -378.0,
+      "loss": 0.1814,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -13.5,
+      "rewards/margins": 5.84375,
+      "rewards/rejected": -19.25,
+      "step": 1240
+    },
+    {
+      "epoch": 0.15975461690842865,
+      "grad_norm": 9.0012252642816,
+      "learning_rate": 2.262741699796952e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.82421875,
+      "logps/chosen": -346.0,
+      "logps/rejected": -382.0,
+      "loss": 0.1959,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -12.5625,
+      "rewards/margins": 6.03125,
+      "rewards/rejected": -18.625,
+      "step": 1250
+    },
+    {
+      "epoch": 0.16103265384369608,
+      "grad_norm": 7.7085887098229575,
+      "learning_rate": 2.253744679276044e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -336.0,
+      "logps/rejected": -368.0,
+      "loss": 0.2488,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -12.625,
+      "rewards/margins": 5.3125,
+      "rewards/rejected": -18.0,
+      "step": 1260
+    },
+    {
+      "epoch": 0.1623106907789635,
+      "grad_norm": 8.516064849377404,
+      "learning_rate": 2.244854133065255e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -360.0,
+      "logps/rejected": -386.0,
+      "loss": 0.2082,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -13.0,
+      "rewards/margins": 4.8125,
+      "rewards/rejected": -17.875,
+      "step": 1270
+    },
+    {
+      "epoch": 0.16358872771423094,
+      "grad_norm": 8.259370651534303,
+      "learning_rate": 2.2360679774997895e-06,
+      "logits/chosen": -1.109375,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -352.0,
+      "logps/rejected": -370.0,
+      "loss": 0.2143,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -14.375,
+      "rewards/margins": 4.6875,
+      "rewards/rejected": -19.125,
+      "step": 1280
+    },
+    {
+      "epoch": 0.16486676464949837,
+      "grad_norm": 9.548931080612926,
+      "learning_rate": 2.2273841855588183e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -358.0,
+      "logps/rejected": -382.0,
+      "loss": 0.2117,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.25,
+      "rewards/margins": 5.5,
+      "rewards/rejected": -19.75,
+      "step": 1290
+    },
+    {
+      "epoch": 0.1661448015847658,
+      "grad_norm": 8.611906621487027,
+      "learning_rate": 2.2188007849009167e-06,
+      "logits/chosen": -1.1953125,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -350.0,
+      "logps/rejected": -394.0,
+      "loss": 0.1978,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -13.6875,
+      "rewards/margins": 5.75,
+      "rewards/rejected": -19.5,
+      "step": 1300
+    },
+    {
+      "epoch": 0.16742283852003323,
+      "grad_norm": 8.954643944245037,
+      "learning_rate": 2.2103158559821502e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -358.0,
+      "logps/rejected": -396.0,
+      "loss": 0.1751,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.0625,
+      "rewards/margins": 5.8125,
+      "rewards/rejected": -19.875,
+      "step": 1310
+    },
+    {
+      "epoch": 0.16870087545530066,
+      "grad_norm": 8.530762142871044,
+      "learning_rate": 2.2019275302527213e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -334.0,
+      "logps/rejected": -382.0,
+      "loss": 0.1949,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.125,
+      "rewards/margins": 5.28125,
+      "rewards/rejected": -19.5,
+      "step": 1320
+    },
+    {
+      "epoch": 0.1699789123905681,
+      "grad_norm": 7.935100904860878,
+      "learning_rate": 2.193633988428327e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.92578125,
+      "logps/chosen": -348.0,
+      "logps/rejected": -390.0,
+      "loss": 0.2028,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -14.0,
+      "rewards/margins": 7.34375,
+      "rewards/rejected": -21.375,
+      "step": 1330
+    },
+    {
+      "epoch": 0.17125694932583552,
+      "grad_norm": 19.10854859862156,
+      "learning_rate": 2.185433458832612e-06,
+      "logits/chosen": -1.453125,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -364.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1786,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 5.90625,
+      "rewards/rejected": -20.375,
+      "step": 1340
+    },
+    {
+      "epoch": 0.17253498626110295,
+      "grad_norm": 6.167399644936153,
+      "learning_rate": 2.177324215807269e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -374.0,
+      "logps/rejected": -396.0,
+      "loss": 0.2088,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -14.625,
+      "rewards/margins": 5.0,
+      "rewards/rejected": -19.625,
+      "step": 1350
+    },
+    {
+      "epoch": 0.17381302319637038,
+      "grad_norm": 8.958518377708664,
+      "learning_rate": 2.1693045781865616e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -378.0,
+      "logps/rejected": -410.0,
+      "loss": 0.2075,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -13.875,
+      "rewards/margins": 6.5,
+      "rewards/rejected": -20.375,
+      "step": 1360
+    },
+    {
+      "epoch": 0.1750910601316378,
+      "grad_norm": 6.390424788669095,
+      "learning_rate": 2.1613729078331965e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.78125,
+      "logps/chosen": -344.0,
+      "logps/rejected": -374.0,
+      "loss": 0.1837,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.125,
+      "rewards/margins": 5.09375,
+      "rewards/rejected": -19.25,
+      "step": 1370
+    },
+    {
+      "epoch": 0.17636909706690523,
+      "grad_norm": 5.515284743724044,
+      "learning_rate": 2.1535276082326617e-06,
+      "logits/chosen": -1.15625,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -346.0,
+      "logps/rejected": -380.0,
+      "loss": 0.1943,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.625,
+      "rewards/margins": 5.375,
+      "rewards/rejected": -19.0,
+      "step": 1380
+    },
+    {
+      "epoch": 0.17764713400217266,
+      "grad_norm": 8.4570921005926,
+      "learning_rate": 2.14576712314328e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.73046875,
+      "logps/chosen": -344.0,
+      "logps/rejected": -376.0,
+      "loss": 0.1883,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.4375,
+      "rewards/margins": 5.6875,
+      "rewards/rejected": -19.125,
+      "step": 1390
+    },
+    {
+      "epoch": 0.1789251709374401,
+      "grad_norm": 9.549613268445338,
+      "learning_rate": 2.138089935299395e-06,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -376.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1875,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -14.1875,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -20.625,
+      "step": 1400
+    },
+    {
+      "epoch": 0.18020320787270752,
+      "grad_norm": 7.380828136210876,
+      "learning_rate": 2.1304945651652297e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -370.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1704,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 6.09375,
+      "rewards/rejected": -21.0,
+      "step": 1410
+    },
+    {
+      "epoch": 0.18148124480797495,
+      "grad_norm": 6.263374950188119,
+      "learning_rate": 2.122979569737101e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -366.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1829,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 6.09375,
+      "rewards/rejected": -21.25,
+      "step": 1420
+    },
+    {
+      "epoch": 0.18275928174324238,
+      "grad_norm": 5.833976849391102,
+      "learning_rate": 2.11554354139178e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -346.0,
+      "logps/rejected": -392.0,
+      "loss": 0.2099,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.625,
+      "rewards/margins": 6.28125,
+      "rewards/rejected": -20.875,
+      "step": 1430
+    },
+    {
+      "epoch": 0.1840373186785098,
+      "grad_norm": 9.495848476346712,
+      "learning_rate": 2.1081851067789196e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -378.0,
+      "logps/rejected": -418.0,
+      "loss": 0.2122,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 6.25,
+      "rewards/rejected": -21.5,
+      "step": 1440
+    },
+    {
+      "epoch": 0.18531535561377724,
+      "grad_norm": 9.911111591218397,
+      "learning_rate": 2.1009029257555606e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -352.0,
+      "logps/rejected": -392.0,
+      "loss": 0.2132,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -13.5625,
+      "rewards/margins": 5.25,
+      "rewards/rejected": -18.75,
+      "step": 1450
+    },
+    {
+      "epoch": 0.18659339254904467,
+      "grad_norm": 8.722955061254407,
+      "learning_rate": 2.0936956903608545e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -334.0,
+      "logps/rejected": -362.0,
+      "loss": 0.2117,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -12.625,
+      "rewards/margins": 5.3125,
+      "rewards/rejected": -18.0,
+      "step": 1460
+    },
+    {
+      "epoch": 0.1878714294843121,
+      "grad_norm": 11.653053012562472,
+      "learning_rate": 2.0865621238292046e-06,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -358.0,
+      "logps/rejected": -390.0,
+      "loss": 0.2219,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -12.625,
+      "rewards/margins": 5.5,
+      "rewards/rejected": -18.125,
+      "step": 1470
+    },
+    {
+      "epoch": 0.18914946641957953,
+      "grad_norm": 9.702295528815855,
+      "learning_rate": 2.079500979640145e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -362.0,
+      "logps/rejected": -392.0,
+      "loss": 0.1891,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -13.625,
+      "rewards/margins": 5.5625,
+      "rewards/rejected": -19.25,
+      "step": 1480
+    },
+    {
+      "epoch": 0.19042750335484696,
+      "grad_norm": 7.596019203915662,
+      "learning_rate": 2.072511040603359e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -358.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1981,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -13.375,
+      "rewards/margins": 5.90625,
+      "rewards/rejected": -19.25,
+      "step": 1490
+    },
+    {
+      "epoch": 0.19170554029011438,
+      "grad_norm": 17.55835999545242,
+      "learning_rate": 2.065591117977289e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.84765625,
+      "logps/chosen": -346.0,
+      "logps/rejected": -380.0,
+      "loss": 0.1927,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -13.6875,
+      "rewards/margins": 6.09375,
+      "rewards/rejected": -19.75,
+      "step": 1500
+    },
+    {
+      "epoch": 0.1929835772253818,
+      "grad_norm": 10.066204082582184,
+      "learning_rate": 2.058740050619915e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.73046875,
+      "logps/chosen": -352.0,
+      "logps/rejected": -366.0,
+      "loss": 0.1902,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -13.875,
+      "rewards/margins": 6.03125,
+      "rewards/rejected": -19.875,
+      "step": 1510
+    },
+    {
+      "epoch": 0.19426161416064924,
+      "grad_norm": 9.063845876423743,
+      "learning_rate": 2.0519567041703083e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -352.0,
+      "logps/rejected": -400.0,
+      "loss": 0.1805,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -13.5,
+      "rewards/margins": 5.59375,
+      "rewards/rejected": -19.125,
+      "step": 1520
+    },
+    {
+      "epoch": 0.19553965109591667,
+      "grad_norm": 31.381712203748485,
+      "learning_rate": 2.0452399702596544e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.77734375,
+      "logps/chosen": -342.0,
+      "logps/rejected": -378.0,
+      "loss": 0.19,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -13.8125,
+      "rewards/margins": 5.03125,
+      "rewards/rejected": -18.875,
+      "step": 1530
+    },
+    {
+      "epoch": 0.1968176880311841,
+      "grad_norm": 10.403937902501944,
+      "learning_rate": 2.0385887657505017e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.8046875,
+      "logps/chosen": -366.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1993,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -13.8125,
+      "rewards/margins": 5.46875,
+      "rewards/rejected": -19.25,
+      "step": 1540
+    },
+    {
+      "epoch": 0.19809572496645153,
+      "grad_norm": 10.449478748361528,
+      "learning_rate": 2.032002032003048e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -372.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1596,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -14.375,
+      "rewards/margins": 5.65625,
+      "rewards/rejected": -20.0,
+      "step": 1550
+    },
+    {
+      "epoch": 0.19937376190171896,
+      "grad_norm": 10.678174088854613,
+      "learning_rate": 2.025478734167333e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.796875,
+      "logps/chosen": -362.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1941,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.125,
+      "rewards/margins": 6.46875,
+      "rewards/rejected": -20.625,
+      "step": 1560
+    },
+    {
+      "epoch": 0.2006517988369864,
+      "grad_norm": 9.290832142092876,
+      "learning_rate": 2.0190178605002747e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -352.0,
+      "logps/rejected": -394.0,
+      "loss": 0.2298,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.125,
+      "rewards/margins": 6.0,
+      "rewards/rejected": -20.125,
+      "step": 1570
+    },
+    {
+      "epoch": 0.20192983577225382,
+      "grad_norm": 9.006013263613145,
+      "learning_rate": 2.0126184217065104e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.72265625,
+      "logps/chosen": -356.0,
+      "logps/rejected": -384.0,
+      "loss": 0.2089,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -15.5,
+      "rewards/margins": 5.09375,
+      "rewards/rejected": -20.625,
+      "step": 1580
+    },
+    {
+      "epoch": 0.20320787270752125,
+      "grad_norm": 9.545196000147508,
+      "learning_rate": 2.0062794503020765e-06,
+      "logits/chosen": -1.1328125,
+      "logits/rejected": -0.77734375,
+      "logps/chosen": -368.0,
+      "logps/rejected": -394.0,
+      "loss": 0.1906,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -14.875,
+      "rewards/margins": 5.15625,
+      "rewards/rejected": -20.0,
+      "step": 1590
+    },
+    {
+      "epoch": 0.20448590964278868,
+      "grad_norm": 14.017014588015606,
+      "learning_rate": 2e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -358.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1768,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 6.34375,
+      "rewards/rejected": -21.75,
+      "step": 1600
+    },
+    {
+      "epoch": 0.2057639465780561,
+      "grad_norm": 12.143680516585514,
+      "learning_rate": 1.993779145116907e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.7734375,
+      "logps/chosen": -356.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1801,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -15.0625,
+      "rewards/margins": 6.875,
+      "rewards/rejected": -22.0,
+      "step": 1610
+    },
+    {
+      "epoch": 0.20704198351332354,
+      "grad_norm": 17.788019561478457,
+      "learning_rate": 1.987615979999813e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.76171875,
+      "logps/chosen": -368.0,
+      "logps/rejected": -446.0,
+      "loss": 0.2137,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -21.75,
+      "step": 1620
+    },
+    {
+      "epoch": 0.20832002044859096,
+      "grad_norm": 8.638126700789615,
+      "learning_rate": 1.9815096184722797e-06,
+      "logits/chosen": -1.125,
+      "logits/rejected": -0.76171875,
+      "logps/chosen": -376.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1823,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 5.59375,
+      "rewards/rejected": -21.75,
+      "step": 1630
+    },
+    {
+      "epoch": 0.2095980573838584,
+      "grad_norm": 4.332363462943313,
+      "learning_rate": 1.9754591932991793e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.796875,
+      "logps/chosen": -362.0,
+      "logps/rejected": -410.0,
+      "loss": 0.161,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.0625,
+      "rewards/margins": 6.5,
+      "rewards/rejected": -20.5,
+      "step": 1640
+    },
+    {
+      "epoch": 0.21087609431912582,
+      "grad_norm": 15.57112713107406,
+      "learning_rate": 1.9694638556693235e-06,
+      "logits/chosen": -1.1640625,
+      "logits/rejected": -0.78515625,
+      "logps/chosen": -358.0,
+      "logps/rejected": -396.0,
+      "loss": 0.1749,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -14.8125,
+      "rewards/margins": 7.1875,
+      "rewards/rejected": -22.0,
+      "step": 1650
+    },
+    {
+      "epoch": 0.21215413125439325,
+      "grad_norm": 377.7920454927129,
+      "learning_rate": 1.963522774695264e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -380.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1955,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.75,
+      "rewards/margins": 6.46875,
+      "rewards/rejected": -22.25,
+      "step": 1660
+    },
+    {
+      "epoch": 0.21343216818966068,
+      "grad_norm": 8.019294941904924,
+      "learning_rate": 1.9576351369295853e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.8046875,
+      "logps/chosen": -394.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1668,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 5.65625,
+      "rewards/rejected": -21.875,
+      "step": 1670
+    },
+    {
+      "epoch": 0.2147102051249281,
+      "grad_norm": 8.504402521384709,
+      "learning_rate": 1.951800145897066e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.76953125,
+      "logps/chosen": -354.0,
+      "logps/rejected": -376.0,
+      "loss": 0.2052,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 5.21875,
+      "rewards/rejected": -21.25,
+      "step": 1680
+    },
+    {
+      "epoch": 0.21598824206019554,
+      "grad_norm": 9.142850942392126,
+      "learning_rate": 1.9460170216420797e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -386.0,
+      "logps/rejected": -400.0,
+      "loss": 0.1704,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 5.3125,
+      "rewards/rejected": -21.5,
+      "step": 1690
+    },
+    {
+      "epoch": 0.21726627899546297,
+      "grad_norm": 14.28325315069547,
+      "learning_rate": 1.9402850002906637e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.828125,
+      "logps/chosen": -368.0,
+      "logps/rejected": -400.0,
+      "loss": 0.1995,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 5.65625,
+      "rewards/rejected": -21.5,
+      "step": 1700
+    },
+    {
+      "epoch": 0.2185443159307304,
+      "grad_norm": 2.8177020230992915,
+      "learning_rate": 1.9346033336266974e-06,
+      "logits/chosen": -1.15625,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1802,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 5.25,
+      "rewards/rejected": -20.625,
+      "step": 1710
+    },
+    {
+      "epoch": 0.21982235286599783,
+      "grad_norm": 7.067056996819025,
+      "learning_rate": 1.9289712886816486e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -356.0,
+      "logps/rejected": -396.0,
+      "loss": 0.1691,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.0625,
+      "rewards/margins": 6.34375,
+      "rewards/rejected": -20.375,
+      "step": 1720
+    },
+    {
+      "epoch": 0.22110038980126526,
+      "grad_norm": 78.04958948240446,
+      "learning_rate": 1.92338814733738e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -340.0,
+      "logps/rejected": -390.0,
+      "loss": 0.1905,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -13.5625,
+      "rewards/margins": 6.4375,
+      "rewards/rejected": -20.0,
+      "step": 1730
+    },
+    {
+      "epoch": 0.22237842673653269,
+      "grad_norm": 12.184941512990717,
+      "learning_rate": 1.9178532059415367e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -366.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1743,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.5625,
+      "rewards/margins": 6.03125,
+      "rewards/rejected": -20.625,
+      "step": 1740
+    },
+    {
+      "epoch": 0.22365646367180012,
+      "grad_norm": 9.125673745600459,
+      "learning_rate": 1.9123657749350298e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -354.0,
+      "logps/rejected": -378.0,
+      "loss": 0.2051,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.375,
+      "rewards/margins": 6.0625,
+      "rewards/rejected": -20.5,
+      "step": 1750
+    },
+    {
+      "epoch": 0.22493450060706754,
+      "grad_norm": 8.373944216513223,
+      "learning_rate": 1.9069251784911844e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -384.0,
+      "logps/rejected": -404.0,
+      "loss": 0.205,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 5.09375,
+      "rewards/rejected": -21.0,
+      "step": 1760
+    },
+    {
+      "epoch": 0.22621253754233497,
+      "grad_norm": 5.843200952238089,
+      "learning_rate": 1.9015307541661132e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -388.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1761,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 5.15625,
+      "rewards/rejected": -21.125,
+      "step": 1770
+    },
+    {
+      "epoch": 0.2274905744776024,
+      "grad_norm": 5.7890586821904915,
+      "learning_rate": 1.8961818525599089e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -366.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1877,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -15.6875,
+      "rewards/margins": 5.90625,
+      "rewards/rejected": -21.625,
+      "step": 1780
+    },
+    {
+      "epoch": 0.22876861141286983,
+      "grad_norm": 7.842327664831985,
+      "learning_rate": 1.890877836988262e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.859375,
+      "logps/chosen": -354.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1585,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -14.6875,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -22.0,
+      "step": 1790
+    },
+    {
+      "epoch": 0.23004664834813726,
+      "grad_norm": 7.2870112608491455,
+      "learning_rate": 1.8856180831641269e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -370.0,
+      "logps/rejected": -390.0,
+      "loss": 0.1765,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.125,
+      "rewards/margins": 5.78125,
+      "rewards/rejected": -19.875,
+      "step": 1800
+    },
+    {
+      "epoch": 0.2313246852834047,
+      "grad_norm": 4.52644645364935,
+      "learning_rate": 1.8804019788890737e-06,
+      "logits/chosen": -1.1640625,
+      "logits/rejected": -0.78125,
+      "logps/chosen": -356.0,
+      "logps/rejected": -382.0,
+      "loss": 0.1766,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -13.9375,
+      "rewards/margins": 5.75,
+      "rewards/rejected": -19.75,
+      "step": 1810
+    },
+    {
+      "epoch": 0.23260272221867212,
+      "grad_norm": 8.308765595580109,
+      "learning_rate": 1.8752289237539816e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -350.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1697,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.5625,
+      "rewards/margins": 5.875,
+      "rewards/rejected": -20.375,
+      "step": 1820
+    },
+    {
+      "epoch": 0.23388075915393955,
+      "grad_norm": 12.408549898540661,
+      "learning_rate": 1.8700983288487376e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.78515625,
+      "logps/chosen": -362.0,
+      "logps/rejected": -416.0,
+      "loss": 0.2051,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.0,
+      "rewards/margins": 6.1875,
+      "rewards/rejected": -21.25,
+      "step": 1830
+    },
+    {
+      "epoch": 0.23515879608920698,
+      "grad_norm": 9.45902705544609,
+      "learning_rate": 1.8650096164806275e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.78515625,
+      "logps/chosen": -358.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1798,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.625,
+      "rewards/margins": 6.46875,
+      "rewards/rejected": -21.125,
+      "step": 1840
+    },
+    {
+      "epoch": 0.2364368330244744,
+      "grad_norm": 7.21742966060711,
+      "learning_rate": 1.8599622199011084e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.78125,
+      "logps/chosen": -368.0,
+      "logps/rejected": -400.0,
+      "loss": 0.1824,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.8125,
+      "rewards/margins": 6.25,
+      "rewards/rejected": -21.0,
+      "step": 1850
+    },
+    {
+      "epoch": 0.23771486995974184,
+      "grad_norm": 15.986007562763428,
+      "learning_rate": 1.854955583040673e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -376.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1956,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -14.5625,
+      "rewards/margins": 5.53125,
+      "rewards/rejected": -20.125,
+      "step": 1860
+    },
+    {
+      "epoch": 0.23899290689500927,
+      "grad_norm": 9.000142861883766,
+      "learning_rate": 1.849989160251521e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -0.76171875,
+      "logps/chosen": -360.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1738,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.6875,
+      "rewards/margins": 5.5,
+      "rewards/rejected": -20.25,
+      "step": 1870
+    },
+    {
+      "epoch": 0.2402709438302767,
+      "grad_norm": 5.232158513035901,
+      "learning_rate": 1.8450624160577701e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.8359375,
+      "logps/chosen": -378.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1694,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 6.875,
+      "rewards/rejected": -22.75,
+      "step": 1880
+    },
+    {
+      "epoch": 0.24154898076554412,
+      "grad_norm": 6.061017046339365,
+      "learning_rate": 1.8401748249129445e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.6953125,
+      "logps/chosen": -358.0,
+      "logps/rejected": -392.0,
+      "loss": 0.1705,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 6.78125,
+      "rewards/rejected": -22.0,
+      "step": 1890
+    },
+    {
+      "epoch": 0.24282701770081155,
+      "grad_norm": 8.8928126131724,
+      "learning_rate": 1.8353258709644938e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -358.0,
+      "logps/rejected": -388.0,
+      "loss": 0.2131,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.0625,
+      "rewards/margins": 6.34375,
+      "rewards/rejected": -21.375,
+      "step": 1900
+    },
+    {
+      "epoch": 0.24410505463607898,
+      "grad_norm": 8.0790568068676,
+      "learning_rate": 1.830515047825102e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -0.71484375,
+      "logps/chosen": -370.0,
+      "logps/rejected": -404.0,
+      "loss": 0.201,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 5.71875,
+      "rewards/rejected": -20.75,
+      "step": 1910
+    },
+    {
+      "epoch": 0.2453830915713464,
+      "grad_norm": 8.41286507829882,
+      "learning_rate": 1.8257418583505536e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.75,
+      "logps/chosen": -376.0,
+      "logps/rejected": -388.0,
+      "loss": 0.1992,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 4.9375,
+      "rewards/rejected": -19.375,
+      "step": 1920
+    },
+    {
+      "epoch": 0.24666112850661384,
+      "grad_norm": 9.175885823550692,
+      "learning_rate": 1.8210058144239416e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -370.0,
+      "logps/rejected": -386.0,
+      "loss": 0.1927,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -15.0,
+      "rewards/margins": 5.09375,
+      "rewards/rejected": -20.125,
+      "step": 1930
+    },
+    {
+      "epoch": 0.24793916544188127,
+      "grad_norm": 6.397145920826923,
+      "learning_rate": 1.816306436745999e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -350.0,
+      "logps/rejected": -396.0,
+      "loss": 0.2024,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -14.1875,
+      "rewards/margins": 5.6875,
+      "rewards/rejected": -19.875,
+      "step": 1940
+    },
+    {
+      "epoch": 0.2492172023771487,
+      "grad_norm": 5.572438133900194,
+      "learning_rate": 1.8116432546313529e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.8359375,
+      "logps/chosen": -354.0,
+      "logps/rejected": -396.0,
+      "loss": 0.1735,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -13.75,
+      "rewards/margins": 5.8125,
+      "rewards/rejected": -19.5,
+      "step": 1950
+    },
+    {
+      "epoch": 0.25049523931241613,
+      "grad_norm": 6.621464616473529,
+      "learning_rate": 1.8070158058105026e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.76953125,
+      "logps/chosen": -356.0,
+      "logps/rejected": -378.0,
+      "loss": 0.2021,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.8125,
+      "rewards/margins": 4.90625,
+      "rewards/rejected": -19.75,
+      "step": 1960
+    },
+    {
+      "epoch": 0.25177327624768353,
+      "grad_norm": 6.616293991663503,
+      "learning_rate": 1.8024236362373315e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.765625,
+      "logps/chosen": -350.0,
+      "logps/rejected": -366.0,
+      "loss": 0.189,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -14.25,
+      "rewards/margins": 5.0625,
+      "rewards/rejected": -19.25,
+      "step": 1970
+    },
+    {
+      "epoch": 0.253051313182951,
+      "grad_norm": 7.042079111166021,
+      "learning_rate": 1.7978662999019787e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -346.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1701,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -14.6875,
+      "rewards/margins": 5.59375,
+      "rewards/rejected": -20.25,
+      "step": 1980
+    },
+    {
+      "epoch": 0.2543293501182184,
+      "grad_norm": 10.514525806737336,
+      "learning_rate": 1.793343358648881e-06,
+      "logits/chosen": -1.1953125,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -358.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1962,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 5.5,
+      "rewards/rejected": -20.75,
+      "step": 1990
+    },
+    {
+      "epoch": 0.25560738705348585,
+      "grad_norm": 8.633722682779158,
+      "learning_rate": 1.7888543819998317e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.76171875,
+      "logps/chosen": -362.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1764,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -15.0,
+      "rewards/margins": 6.25,
+      "rewards/rejected": -21.25,
+      "step": 2000
+    },
+    {
+      "epoch": 0.25688542398875325,
+      "grad_norm": 18.371332498606307,
+      "learning_rate": 1.7843989469818819e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -364.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1879,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.25,
+      "rewards/margins": 6.59375,
+      "rewards/rejected": -20.875,
+      "step": 2010
+    },
+    {
+      "epoch": 0.2581634609240207,
+      "grad_norm": 9.132239411855826,
+      "learning_rate": 1.779976637959939e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.875,
+      "logps/chosen": -360.0,
+      "logps/rejected": -394.0,
+      "loss": 0.1553,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -14.125,
+      "rewards/margins": 5.65625,
+      "rewards/rejected": -19.75,
+      "step": 2020
+    },
+    {
+      "epoch": 0.2594414978592881,
+      "grad_norm": 8.320021979050132,
+      "learning_rate": 1.7755870464739012e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -348.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1745,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 6.375,
+      "rewards/rejected": -20.875,
+      "step": 2030
+    },
+    {
+      "epoch": 0.26071953479455556,
+      "grad_norm": 6.351620079524678,
+      "learning_rate": 1.7712297710801907e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -0.7421875,
+      "logps/chosen": -356.0,
+      "logps/rejected": -400.0,
+      "loss": 0.1964,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.3125,
+      "rewards/margins": 5.59375,
+      "rewards/rejected": -20.875,
+      "step": 2040
+    },
+    {
+      "epoch": 0.26199757172982296,
+      "grad_norm": 7.980486087399472,
+      "learning_rate": 1.7669044171975444e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -390.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1744,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 5.78125,
+      "rewards/rejected": -21.75,
+      "step": 2050
+    },
+    {
+      "epoch": 0.2632756086650904,
+      "grad_norm": 6.326141505405479,
+      "learning_rate": 1.7626105969569268e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -382.0,
+      "logps/rejected": -414.0,
+      "loss": 0.2061,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 5.90625,
+      "rewards/rejected": -21.25,
+      "step": 2060
+    },
+    {
+      "epoch": 0.2645536456003578,
+      "grad_norm": 5.719037242002649,
+      "learning_rate": 1.758347929055432e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -354.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1886,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -13.9375,
+      "rewards/margins": 6.15625,
+      "rewards/rejected": -20.125,
+      "step": 2070
+    },
+    {
+      "epoch": 0.2658316825356253,
+      "grad_norm": 5.268723738243688,
+      "learning_rate": 1.7541160386140582e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.80859375,
+      "logps/chosen": -372.0,
+      "logps/rejected": -372.0,
+      "loss": 0.1885,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 5.28125,
+      "rewards/rejected": -19.75,
+      "step": 2080
+    },
+    {
+      "epoch": 0.2671097194708927,
+      "grad_norm": 10.735675382088875,
+      "learning_rate": 1.7499145570392284e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.796875,
+      "logps/chosen": -358.0,
+      "logps/rejected": -382.0,
+      "loss": 0.1832,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 5.84375,
+      "rewards/rejected": -20.375,
+      "step": 2090
+    },
+    {
+      "epoch": 0.26838775640616014,
+      "grad_norm": 6.375756114324437,
+      "learning_rate": 1.745743121887939e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.74609375,
+      "logps/chosen": -352.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1685,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.6875,
+      "rewards/margins": 6.09375,
+      "rewards/rejected": -19.875,
+      "step": 2100
+    },
+    {
+      "epoch": 0.26966579334142754,
+      "grad_norm": 7.440944469589943,
+      "learning_rate": 1.7416013767364324e-06,
+      "logits/chosen": -1.0859375,
+      "logits/rejected": -0.76171875,
+      "logps/chosen": -362.0,
+      "logps/rejected": -392.0,
+      "loss": 0.1781,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -14.5625,
+      "rewards/margins": 6.34375,
+      "rewards/rejected": -20.875,
+      "step": 2110
+    },
+    {
+      "epoch": 0.270943830276695,
+      "grad_norm": 9.25032063290401,
+      "learning_rate": 1.7374889710522776e-06,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -0.59765625,
+      "logps/chosen": -348.0,
+      "logps/rejected": -414.0,
+      "loss": 0.2085,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -14.0625,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -20.75,
+      "step": 2120
+    },
+    {
+      "epoch": 0.2722218672119624,
+      "grad_norm": 6.600058600287687,
+      "learning_rate": 1.7334055600697579e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.77734375,
+      "logps/chosen": -382.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1684,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.5625,
+      "rewards/margins": 5.9375,
+      "rewards/rejected": -21.5,
+      "step": 2130
+    },
+    {
+      "epoch": 0.27349990414722986,
+      "grad_norm": 6.863890076493143,
+      "learning_rate": 1.7293508046684678e-06,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -0.66015625,
+      "logps/chosen": -350.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1535,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.5,
+      "rewards/margins": 5.6875,
+      "rewards/rejected": -21.25,
+      "step": 2140
+    },
+    {
+      "epoch": 0.27477794108249726,
+      "grad_norm": 8.419295670801706,
+      "learning_rate": 1.7253243712550145e-06,
+      "logits/chosen": -1.140625,
+      "logits/rejected": -0.73828125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1739,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 6.53125,
+      "rewards/rejected": -22.375,
+      "step": 2150
+    },
+    {
+      "epoch": 0.2760559780177647,
+      "grad_norm": 8.354124484051964,
+      "learning_rate": 1.7213259316477406e-06,
+      "logits/chosen": -1.15625,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -384.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1649,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 6.625,
+      "rewards/rejected": -22.75,
+      "step": 2160
+    },
+    {
+      "epoch": 0.2773340149530321,
+      "grad_norm": 7.851350208522757,
+      "learning_rate": 1.7173551629643674e-06,
+      "logits/chosen": -1.125,
+      "logits/rejected": -0.703125,
+      "logps/chosen": -368.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1918,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 5.59375,
+      "rewards/rejected": -21.75,
+      "step": 2170
+    },
+    {
+      "epoch": 0.27861205188829957,
+      "grad_norm": 8.571544162279022,
+      "learning_rate": 1.713411747512477e-06,
+      "logits/chosen": -1.0859375,
+      "logits/rejected": -0.71484375,
+      "logps/chosen": -384.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1801,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 6.03125,
+      "rewards/rejected": -22.25,
+      "step": 2180
+    },
+    {
+      "epoch": 0.279890088823567,
+      "grad_norm": 10.341956383788414,
+      "learning_rate": 1.709495372682753e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -386.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1764,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 6.125,
+      "rewards/rejected": -23.25,
+      "step": 2190
+    },
+    {
+      "epoch": 0.28116812575883443,
+      "grad_norm": 6.518717291317195,
+      "learning_rate": 1.7056057308448832e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -368.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1416,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 6.375,
+      "rewards/rejected": -21.5,
+      "step": 2200
+    },
+    {
+      "epoch": 0.28244616269410183,
+      "grad_norm": 3.550327685087959,
+      "learning_rate": 1.701742519246068e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.8359375,
+      "logps/chosen": -376.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1389,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -15.625,
+      "rewards/margins": 7.125,
+      "rewards/rejected": -22.75,
+      "step": 2210
+    },
+    {
+      "epoch": 0.2837241996293693,
+      "grad_norm": 4.549564280441093,
+      "learning_rate": 1.6979054399120355e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.875,
+      "logps/chosen": -374.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1731,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.75,
+      "rewards/margins": 6.71875,
+      "rewards/rejected": -22.5,
+      "step": 2220
+    },
+    {
+      "epoch": 0.2850022365646367,
+      "grad_norm": 8.173725909428468,
+      "learning_rate": 1.6940941995505069e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1755,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -15.5625,
+      "rewards/margins": 6.125,
+      "rewards/rejected": -21.625,
+      "step": 2230
+    },
+    {
+      "epoch": 0.28628027349990415,
+      "grad_norm": 6.504685204854815,
+      "learning_rate": 1.6903085094570331e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -378.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1824,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 5.875,
+      "rewards/rejected": -21.75,
+      "step": 2240
+    },
+    {
+      "epoch": 0.28755831043517155,
+      "grad_norm": 8.492689434457757,
+      "learning_rate": 1.6865480854231356e-06,
+      "logits/chosen": -1.1640625,
+      "logits/rejected": -0.7578125,
+      "logps/chosen": -366.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1563,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 6.28125,
+      "rewards/rejected": -21.25,
+      "step": 2250
+    },
+    {
+      "epoch": 0.288836347370439,
+      "grad_norm": 7.00501974809585,
+      "learning_rate": 1.682812647646685e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -350.0,
+      "logps/rejected": -394.0,
+      "loss": 0.1495,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 6.09375,
+      "rewards/rejected": -21.0,
+      "step": 2260
+    },
+    {
+      "epoch": 0.2901143843057064,
+      "grad_norm": 7.441753386795718,
+      "learning_rate": 1.6791019206444541e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -372.0,
+      "logps/rejected": -390.0,
+      "loss": 0.1467,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -21.375,
+      "step": 2270
+    },
+    {
+      "epoch": 0.29139242124097386,
+      "grad_norm": 6.167243864062666,
+      "learning_rate": 1.675415633166782e-06,
+      "logits/chosen": -1.1640625,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -358.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1668,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.6875,
+      "rewards/margins": 6.65625,
+      "rewards/rejected": -21.375,
+      "step": 2280
+    },
+    {
+      "epoch": 0.29267045817624127,
+      "grad_norm": 7.423572020249587,
+      "learning_rate": 1.6717535181142914e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -1.0,
+      "logps/chosen": -380.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1795,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -14.75,
+      "rewards/margins": 6.46875,
+      "rewards/rejected": -21.25,
+      "step": 2290
+    },
+    {
+      "epoch": 0.2939484951115087,
+      "grad_norm": 5.912951258940523,
+      "learning_rate": 1.668115312456598e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -374.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1346,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -21.875,
+      "step": 2300
+    },
+    {
+      "epoch": 0.2952265320467761,
+      "grad_norm": 9.012670015274143,
+      "learning_rate": 1.6645007571529578e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -354.0,
+      "logps/rejected": -386.0,
+      "loss": 0.1772,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -15.3125,
+      "rewards/margins": 6.0625,
+      "rewards/rejected": -21.375,
+      "step": 2310
+    },
+    {
+      "epoch": 0.2965045689820436,
+      "grad_norm": 8.263141240275903,
+      "learning_rate": 1.6609095970747992e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -380.0,
+      "logps/rejected": -400.0,
+      "loss": 0.2187,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.75,
+      "rewards/margins": 5.9375,
+      "rewards/rejected": -21.75,
+      "step": 2320
+    },
+    {
+      "epoch": 0.297782605917311,
+      "grad_norm": 4.279988417561156,
+      "learning_rate": 1.6573415809300833e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.7265625,
+      "logps/chosen": -354.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1567,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 5.65625,
+      "rewards/rejected": -20.875,
+      "step": 2330
+    },
+    {
+      "epoch": 0.29906064285257844,
+      "grad_norm": 6.703162826003085,
+      "learning_rate": 1.6537964611894462e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.78515625,
+      "logps/chosen": -364.0,
+      "logps/rejected": -390.0,
+      "loss": 0.1635,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 5.59375,
+      "rewards/rejected": -20.0,
+      "step": 2340
+    },
+    {
+      "epoch": 0.30033867978784584,
+      "grad_norm": 5.474679115348783,
+      "learning_rate": 1.6502739940140692e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -378.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1907,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 6.4375,
+      "rewards/rejected": -21.625,
+      "step": 2350
+    },
+    {
+      "epoch": 0.3016167167231133,
+      "grad_norm": 8.440300163070988,
+      "learning_rate": 1.6467739391852364e-06,
+      "logits/chosen": -1.1484375,
+      "logits/rejected": -0.765625,
+      "logps/chosen": -368.0,
+      "logps/rejected": -402.0,
+      "loss": 0.2064,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 6.28125,
+      "rewards/rejected": -21.5,
+      "step": 2360
+    },
+    {
+      "epoch": 0.3028947536583807,
+      "grad_norm": 5.479599557746082,
+      "learning_rate": 1.6432960600355221e-06,
+      "logits/chosen": -1.109375,
+      "logits/rejected": -0.73046875,
+      "logps/chosen": -370.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1849,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 6.03125,
+      "rewards/rejected": -20.5,
+      "step": 2370
+    },
+    {
+      "epoch": 0.30417279059364816,
+      "grad_norm": 6.835709879830589,
+      "learning_rate": 1.6398401233815756e-06,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -0.78125,
+      "logps/chosen": -356.0,
+      "logps/rejected": -380.0,
+      "loss": 0.1817,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.5625,
+      "rewards/margins": 6.0625,
+      "rewards/rejected": -20.625,
+      "step": 2380
+    },
+    {
+      "epoch": 0.30545082752891556,
+      "grad_norm": 4.344420911042229,
+      "learning_rate": 1.6364058994584524e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -366.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1866,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 6.15625,
+      "rewards/rejected": -21.5,
+      "step": 2390
+    },
+    {
+      "epoch": 0.306728864464183,
+      "grad_norm": 4.871698301531515,
+      "learning_rate": 1.6329931618554522e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.79296875,
+      "logps/chosen": -350.0,
+      "logps/rejected": -392.0,
+      "loss": 0.1599,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -14.875,
+      "rewards/margins": 6.28125,
+      "rewards/rejected": -21.125,
+      "step": 2400
+    },
+    {
+      "epoch": 0.3080069013994504,
+      "grad_norm": 6.4310150550293015,
+      "learning_rate": 1.6296016874534209e-06,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -0.78125,
+      "logps/chosen": -380.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1785,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -14.8125,
+      "rewards/margins": 6.375,
+      "rewards/rejected": -21.125,
+      "step": 2410
+    },
+    {
+      "epoch": 0.3092849383347179,
+      "grad_norm": 8.091350582560779,
+      "learning_rate": 1.6262312563634835e-06,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -0.79296875,
+      "logps/chosen": -340.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1622,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.0625,
+      "rewards/margins": 7.03125,
+      "rewards/rejected": -20.125,
+      "step": 2420
+    },
+    {
+      "epoch": 0.3105629752699853,
+      "grad_norm": 8.173408715010284,
+      "learning_rate": 1.6228816518671587e-06,
+      "logits/chosen": -1.140625,
+      "logits/rejected": -0.7109375,
+      "logps/chosen": -352.0,
+      "logps/rejected": -380.0,
+      "loss": 0.145,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -13.5625,
+      "rewards/margins": 6.09375,
+      "rewards/rejected": -19.75,
+      "step": 2430
+    },
+    {
+      "epoch": 0.31184101220525273,
+      "grad_norm": 8.307777929289866,
+      "learning_rate": 1.619552660357832e-06,
+      "logits/chosen": -1.140625,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -372.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1786,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.25,
+      "rewards/margins": 5.71875,
+      "rewards/rejected": -20.0,
+      "step": 2440
+    },
+    {
+      "epoch": 0.31311904914052013,
+      "grad_norm": 8.037346416486134,
+      "learning_rate": 1.616244071283537e-06,
+      "logits/chosen": -1.0859375,
+      "logits/rejected": -0.75,
+      "logps/chosen": -348.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1786,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -14.8125,
+      "rewards/margins": 5.53125,
+      "rewards/rejected": -20.375,
+      "step": 2450
+    },
+    {
+      "epoch": 0.3143970860757876,
+      "grad_norm": 4.598772736488354,
+      "learning_rate": 1.6129556770910235e-06,
+      "logits/chosen": -1.1484375,
+      "logits/rejected": -0.734375,
+      "logps/chosen": -340.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1516,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -14.8125,
+      "rewards/margins": 5.90625,
+      "rewards/rejected": -20.75,
+      "step": 2460
+    },
+    {
+      "epoch": 0.315675123011055,
+      "grad_norm": 9.516098388693068,
+      "learning_rate": 1.6096872731710673e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -380.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1676,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.5,
+      "rewards/margins": 6.96875,
+      "rewards/rejected": -22.375,
+      "step": 2470
+    },
+    {
+      "epoch": 0.31695315994632245,
+      "grad_norm": 4.631558058752617,
+      "learning_rate": 1.6064386578049978e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -376.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1652,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -20.875,
+      "step": 2480
+    },
+    {
+      "epoch": 0.31823119688158985,
+      "grad_norm": 8.54424101337341,
+      "learning_rate": 1.6032096321124046e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -352.0,
+      "logps/rejected": -378.0,
+      "loss": 0.1393,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -13.625,
+      "rewards/margins": 6.34375,
+      "rewards/rejected": -20.0,
+      "step": 2490
+    },
+    {
+      "epoch": 0.3195092338168573,
+      "grad_norm": 7.797885991647695,
+      "learning_rate": 1.6e-06,
+      "logits/chosen": -1.1953125,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -372.0,
+      "logps/rejected": -422.0,
+      "loss": 0.2337,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.75,
+      "rewards/margins": 6.53125,
+      "rewards/rejected": -21.25,
+      "step": 2500
+    },
+    {
+      "epoch": 0.3207872707521247,
+      "grad_norm": 7.412229924158176,
+      "learning_rate": 1.5968095681115984e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -362.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1701,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -22.25,
+      "step": 2510
+    },
+    {
+      "epoch": 0.32206530768739217,
+      "grad_norm": 6.782291579224652,
+      "learning_rate": 1.5936381457791914e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -364.0,
+      "logps/rejected": -380.0,
+      "loss": 0.1608,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -13.625,
+      "rewards/margins": 6.4375,
+      "rewards/rejected": -20.0,
+      "step": 2520
+    },
+    {
+      "epoch": 0.32334334462265957,
+      "grad_norm": 4.9692010410188585,
+      "learning_rate": 1.590485544975088e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -350.0,
+      "logps/rejected": -388.0,
+      "loss": 0.1568,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.5625,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -20.0,
+      "step": 2530
+    },
+    {
+      "epoch": 0.324621381557927,
+      "grad_norm": 7.435679923432267,
+      "learning_rate": 1.5873515802650901e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.76953125,
+      "logps/chosen": -352.0,
+      "logps/rejected": -378.0,
+      "loss": 0.1564,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -13.0625,
+      "rewards/margins": 6.46875,
+      "rewards/rejected": -19.5,
+      "step": 2540
+    },
+    {
+      "epoch": 0.3258994184931944,
+      "grad_norm": 7.586791561859869,
+      "learning_rate": 1.584236068762679e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -360.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1374,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -14.8125,
+      "rewards/margins": 6.625,
+      "rewards/rejected": -21.375,
+      "step": 2550
+    },
+    {
+      "epoch": 0.3271774554284619,
+      "grad_norm": 6.8995652015140205,
+      "learning_rate": 1.5811388300841894e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -396.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1763,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.5625,
+      "rewards/margins": 6.90625,
+      "rewards/rejected": -22.5,
+      "step": 2560
+    },
+    {
+      "epoch": 0.3284554923637293,
+      "grad_norm": 5.35312877031937,
+      "learning_rate": 1.5780596863049431e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.8046875,
+      "logps/chosen": -364.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1474,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 6.375,
+      "rewards/rejected": -21.875,
+      "step": 2570
+    },
+    {
+      "epoch": 0.32973352929899674,
+      "grad_norm": 8.38143060871466,
+      "learning_rate": 1.5749984619163156e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -358.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1616,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -21.75,
+      "step": 2580
+    },
+    {
+      "epoch": 0.33101156623426414,
+      "grad_norm": 11.190613106126323,
+      "learning_rate": 1.5719549837837187e-06,
+      "logits/chosen": -1.140625,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -376.0,
+      "logps/rejected": -396.0,
+      "loss": 0.1623,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 6.5,
+      "rewards/rejected": -20.875,
+      "step": 2590
+    },
+    {
+      "epoch": 0.3322896031695316,
+      "grad_norm": 6.903427457268049,
+      "learning_rate": 1.5689290811054722e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -368.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1725,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 6.125,
+      "rewards/rejected": -21.125,
+      "step": 2600
+    },
+    {
+      "epoch": 0.333567640104799,
+      "grad_norm": 7.016850224823338,
+      "learning_rate": 1.5659205853725426e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.92578125,
+      "logps/chosen": -362.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1819,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -14.6875,
+      "rewards/margins": 5.75,
+      "rewards/rejected": -20.375,
+      "step": 2610
+    },
+    {
+      "epoch": 0.33484567704006646,
+      "grad_norm": 3.3592430991404374,
+      "learning_rate": 1.562929330329127e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.80859375,
+      "logps/chosen": -338.0,
+      "logps/rejected": -388.0,
+      "loss": 0.1548,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 5.71875,
+      "rewards/rejected": -20.25,
+      "step": 2620
+    },
+    {
+      "epoch": 0.33612371397533386,
+      "grad_norm": 9.916737864551118,
+      "learning_rate": 1.5599551519340636e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.80078125,
+      "logps/chosen": -352.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1764,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 6.53125,
+      "rewards/rejected": -21.0,
+      "step": 2630
+    },
+    {
+      "epoch": 0.3374017509106013,
+      "grad_norm": 8.784713053781411,
+      "learning_rate": 1.556997888323046e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -384.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1884,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -15.9375,
+      "rewards/margins": 6.09375,
+      "rewards/rejected": -22.0,
+      "step": 2640
+    },
+    {
+      "epoch": 0.3386797878458687,
+      "grad_norm": 5.88442385550547,
+      "learning_rate": 1.5540573797716226e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -354.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1431,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -14.8125,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -21.25,
+      "step": 2650
+    },
+    {
+      "epoch": 0.3399578247811362,
+      "grad_norm": 8.004724254093452,
+      "learning_rate": 1.5511334686589623e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -390.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1676,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 6.1875,
+      "rewards/rejected": -23.25,
+      "step": 2660
+    },
+    {
+      "epoch": 0.3412358617164036,
+      "grad_norm": 10.014756278323372,
+      "learning_rate": 1.548225999432367e-06,
+      "logits/chosen": -1.109375,
+      "logits/rejected": -0.78515625,
+      "logps/chosen": -384.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1595,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 5.5625,
+      "rewards/rejected": -22.5,
+      "step": 2670
+    },
+    {
+      "epoch": 0.34251389865167103,
+      "grad_norm": 5.952696177013581,
+      "learning_rate": 1.5453348185725114e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.875,
+      "logps/chosen": -376.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1712,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -22.875,
+      "step": 2680
+    },
+    {
+      "epoch": 0.34379193558693844,
+      "grad_norm": 9.637261698353962,
+      "learning_rate": 1.542459774559398e-06,
+      "logits/chosen": -1.15625,
+      "logits/rejected": -0.7578125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1655,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.8125,
+      "rewards/margins": 7.125,
+      "rewards/rejected": -23.0,
+      "step": 2690
+    },
+    {
+      "epoch": 0.3450699725222059,
+      "grad_norm": 6.680174552709829,
+      "learning_rate": 1.539600717839002e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -368.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1665,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -22.625,
+      "step": 2700
+    },
+    {
+      "epoch": 0.3463480094574733,
+      "grad_norm": 8.235809092583775,
+      "learning_rate": 1.536757500790597e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.74609375,
+      "logps/chosen": -370.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1805,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 6.15625,
+      "rewards/rejected": -22.25,
+      "step": 2710
+    },
+    {
+      "epoch": 0.34762604639274075,
+      "grad_norm": 7.416167848623114,
+      "learning_rate": 1.5339299776947407e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -408.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1785,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 6.375,
+      "rewards/rejected": -23.5,
+      "step": 2720
+    },
+    {
+      "epoch": 0.34890408332800815,
+      "grad_norm": 4.754198191009482,
+      "learning_rate": 1.5311180047019054e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.77734375,
+      "logps/chosen": -366.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1531,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 6.59375,
+      "rewards/rejected": -23.0,
+      "step": 2730
+    },
+    {
+      "epoch": 0.3501821202632756,
+      "grad_norm": 10.342217584069093,
+      "learning_rate": 1.5283214398017402e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -354.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1538,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 6.15625,
+      "rewards/rejected": -22.25,
+      "step": 2740
+    },
+    {
+      "epoch": 0.351460157198543,
+      "grad_norm": 10.957791790476678,
+      "learning_rate": 1.5255401427929477e-06,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -0.7265625,
+      "logps/chosen": -352.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1647,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -15.0,
+      "rewards/margins": 7.21875,
+      "rewards/rejected": -22.25,
+      "step": 2750
+    },
+    {
+      "epoch": 0.35273819413381047,
+      "grad_norm": 5.412848387495793,
+      "learning_rate": 1.5227739752537617e-06,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -0.67578125,
+      "logps/chosen": -380.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1731,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -15.625,
+      "rewards/margins": 6.59375,
+      "rewards/rejected": -22.25,
+      "step": 2760
+    },
+    {
+      "epoch": 0.35401623106907787,
+      "grad_norm": 5.358366197634935,
+      "learning_rate": 1.5200228005130127e-06,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -0.765625,
+      "logps/chosen": -376.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1427,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.75,
+      "rewards/margins": 5.96875,
+      "rewards/rejected": -21.75,
+      "step": 2770
+    },
+    {
+      "epoch": 0.3552942680043453,
+      "grad_norm": 7.5367918947665675,
+      "learning_rate": 1.5172864836217631e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -382.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1352,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 6.375,
+      "rewards/rejected": -23.25,
+      "step": 2780
+    },
+    {
+      "epoch": 0.3565723049396127,
+      "grad_norm": 8.235192447266364,
+      "learning_rate": 1.514564891325506e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.78125,
+      "logps/chosen": -396.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1554,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 6.09375,
+      "rewards/rejected": -23.375,
+      "step": 2790
+    },
+    {
+      "epoch": 0.3578503418748802,
+      "grad_norm": 9.314267573220476,
+      "learning_rate": 1.5118578920369086e-06,
+      "logits/chosen": -1.1484375,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -406.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1458,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 7.09375,
+      "rewards/rejected": -23.625,
+      "step": 2800
+    },
+    {
+      "epoch": 0.3591283788101476,
+      "grad_norm": 8.334808455115347,
+      "learning_rate": 1.5091653558090898e-06,
+      "logits/chosen": -1.140625,
+      "logits/rejected": -0.73046875,
+      "logps/chosen": -362.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1947,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 6.25,
+      "rewards/rejected": -22.5,
+      "step": 2810
+    },
+    {
+      "epoch": 0.36040641574541504,
+      "grad_norm": 9.017869279372748,
+      "learning_rate": 1.506487154309419e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.671875,
+      "logps/chosen": -406.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1607,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 6.96875,
+      "rewards/rejected": -23.625,
+      "step": 2820
+    },
+    {
+      "epoch": 0.36168445268068244,
+      "grad_norm": 8.398489278507387,
+      "learning_rate": 1.5038231607938247e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -0.70703125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1474,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 6.8125,
+      "rewards/rejected": -23.75,
+      "step": 2830
+    },
+    {
+      "epoch": 0.3629624896159499,
+      "grad_norm": 8.98056329040152,
+      "learning_rate": 1.501173250081603e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.953125,
+      "logps/chosen": -400.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1832,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 7.15625,
+      "rewards/rejected": -24.75,
+      "step": 2840
+    },
+    {
+      "epoch": 0.3642405265512173,
+      "grad_norm": 3.631405415493462,
+      "learning_rate": 1.4985372985307103e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.78125,
+      "logps/chosen": -404.0,
+      "logps/rejected": -458.0,
+      "loss": 0.1503,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.0625,
+      "rewards/rejected": -23.75,
+      "step": 2850
+    },
+    {
+      "epoch": 0.36551856348648476,
+      "grad_norm": 4.819913331659097,
+      "learning_rate": 1.4959151840135313e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -392.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1181,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.34375,
+      "rewards/rejected": -24.0,
+      "step": 2860
+    },
+    {
+      "epoch": 0.36679660042175216,
+      "grad_norm": 6.012224391924082,
+      "learning_rate": 1.4933067858931148e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.73046875,
+      "logps/chosen": -342.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1285,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.9375,
+      "rewards/margins": 7.0625,
+      "rewards/rejected": -23.0,
+      "step": 2870
+    },
+    {
+      "epoch": 0.3680746373570196,
+      "grad_norm": 8.149056814970246,
+      "learning_rate": 1.4907119849998597e-06,
+      "logits/chosen": -1.15625,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -364.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1304,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -15.75,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -22.5,
+      "step": 2880
+    },
+    {
+      "epoch": 0.369352674292287,
+      "grad_norm": 7.399935517875265,
+      "learning_rate": 1.488130663608649e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -356.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1187,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.625,
+      "rewards/margins": 6.875,
+      "rewards/rejected": -22.5,
+      "step": 2890
+    },
+    {
+      "epoch": 0.3706307112275545,
+      "grad_norm": 5.092194237451721,
+      "learning_rate": 1.4855627054164149e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.734375,
+      "logps/chosen": -374.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1471,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 6.65625,
+      "rewards/rejected": -23.75,
+      "step": 2900
+    },
+    {
+      "epoch": 0.3719087481628219,
+      "grad_norm": 7.26327821963915,
+      "learning_rate": 1.4830079955201294e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -0.69140625,
+      "logps/chosen": -390.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1556,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 8.125,
+      "rewards/rejected": -24.25,
+      "step": 2910
+    },
+    {
+      "epoch": 0.37318678509808934,
+      "grad_norm": 14.906869619296195,
+      "learning_rate": 1.4804664203952103e-06,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -0.58203125,
+      "logps/chosen": -368.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1635,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.09375,
+      "rewards/rejected": -24.125,
+      "step": 2920
+    },
+    {
+      "epoch": 0.37446482203335674,
+      "grad_norm": 6.831024418447815,
+      "learning_rate": 1.4779378678743327e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.7734375,
+      "logps/chosen": -400.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1375,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -18.375,
+      "rewards/margins": 6.6875,
+      "rewards/rejected": -25.125,
+      "step": 2930
+    },
+    {
+      "epoch": 0.3757428589686242,
+      "grad_norm": 11.87740582575529,
+      "learning_rate": 1.4754222271266348e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -404.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1505,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 6.78125,
+      "rewards/rejected": -24.375,
+      "step": 2940
+    },
+    {
+      "epoch": 0.3770208959038916,
+      "grad_norm": 4.897986183732004,
+      "learning_rate": 1.4729193886373175e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1251,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 6.78125,
+      "rewards/rejected": -24.25,
+      "step": 2950
+    },
+    {
+      "epoch": 0.37829893283915905,
+      "grad_norm": 9.07537762107968,
+      "learning_rate": 1.4704292441876156e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -406.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1579,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 7.78125,
+      "rewards/rejected": -25.0,
+      "step": 2960
+    },
+    {
+      "epoch": 0.37957696977442645,
+      "grad_norm": 8.605454163095152,
+      "learning_rate": 1.4679516868351474e-06,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -0.75,
+      "logps/chosen": -378.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1902,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 6.3125,
+      "rewards/rejected": -24.375,
+      "step": 2970
+    },
+    {
+      "epoch": 0.3808550067096939,
+      "grad_norm": 7.3354029924134165,
+      "learning_rate": 1.4654866108946234e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.6953125,
+      "logps/chosen": -390.0,
+      "logps/rejected": -442.0,
+      "loss": 0.1608,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 6.875,
+      "rewards/rejected": -24.75,
+      "step": 2980
+    },
+    {
+      "epoch": 0.3821330436449613,
+      "grad_norm": 6.155418860212319,
+      "learning_rate": 1.4630339119189101e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.78515625,
+      "logps/chosen": -392.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1353,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 5.75,
+      "rewards/rejected": -23.875,
+      "step": 2990
+    },
+    {
+      "epoch": 0.38341108058022877,
+      "grad_norm": 5.403666679517565,
+      "learning_rate": 1.4605934866804429e-06,
+      "logits/chosen": -1.1953125,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -390.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1469,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -19.0,
+      "rewards/margins": 5.65625,
+      "rewards/rejected": -24.625,
+      "step": 3000
+    },
+    {
+      "epoch": 0.38468911751549617,
+      "grad_norm": 6.389340594085818,
+      "learning_rate": 1.4581652331529784e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -400.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1595,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 6.71875,
+      "rewards/rejected": -24.625,
+      "step": 3010
+    },
+    {
+      "epoch": 0.3859671544507636,
+      "grad_norm": 6.834943049888752,
+      "learning_rate": 1.4557490504936778e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -398.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1624,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 6.90625,
+      "rewards/rejected": -24.5,
+      "step": 3020
+    },
+    {
+      "epoch": 0.38724519138603103,
+      "grad_norm": 9.868830042691824,
+      "learning_rate": 1.453344839025519e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.8046875,
+      "logps/chosen": -412.0,
+      "logps/rejected": -464.0,
+      "loss": 0.1585,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -18.25,
+      "rewards/margins": 6.71875,
+      "rewards/rejected": -25.0,
+      "step": 3030
+    },
+    {
+      "epoch": 0.3885232283212985,
+      "grad_norm": 4.815120173317151,
+      "learning_rate": 1.4509525002200234e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.99609375,
+      "logps/chosen": -404.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1657,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 6.25,
+      "rewards/rejected": -23.875,
+      "step": 3040
+    },
+    {
+      "epoch": 0.3898012652565659,
+      "grad_norm": 5.908075246079269,
+      "learning_rate": 1.4485719366802965e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -396.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1417,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 6.8125,
+      "rewards/rejected": -24.0,
+      "step": 3050
+    },
+    {
+      "epoch": 0.39107930219183334,
+      "grad_norm": 5.599288022876796,
+      "learning_rate": 1.4462030521243742e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.80078125,
+      "logps/chosen": -402.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1171,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.03125,
+      "rewards/rejected": -23.75,
+      "step": 3060
+    },
+    {
+      "epoch": 0.39235733912710075,
+      "grad_norm": 8.265629169805178,
+      "learning_rate": 1.443845751368867e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -386.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1653,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 6.96875,
+      "rewards/rejected": -23.375,
+      "step": 3070
+    },
+    {
+      "epoch": 0.3936353760623682,
+      "grad_norm": 4.139917186421324,
+      "learning_rate": 1.4414999403128943e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -362.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1625,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 6.90625,
+      "rewards/rejected": -22.25,
+      "step": 3080
+    },
+    {
+      "epoch": 0.3949134129976356,
+      "grad_norm": 5.942930998801207,
+      "learning_rate": 1.439165525922309e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.796875,
+      "logps/chosen": -378.0,
+      "logps/rejected": -422.0,
+      "loss": 0.145,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 6.28125,
+      "rewards/rejected": -22.75,
+      "step": 3090
+    },
+    {
+      "epoch": 0.39619144993290306,
+      "grad_norm": 4.319772742437636,
+      "learning_rate": 1.4368424162141992e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -376.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1813,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 6.375,
+      "rewards/rejected": -22.625,
+      "step": 3100
+    },
+    {
+      "epoch": 0.39746948686817046,
+      "grad_norm": 5.130278794235308,
+      "learning_rate": 1.434530520241665e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -364.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1335,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.9375,
+      "rewards/margins": 6.4375,
+      "rewards/rejected": -22.375,
+      "step": 3110
+    },
+    {
+      "epoch": 0.3987475238034379,
+      "grad_norm": 9.325617973309617,
+      "learning_rate": 1.4322297480788657e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -360.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1545,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -14.875,
+      "rewards/margins": 6.71875,
+      "rewards/rejected": -21.625,
+      "step": 3120
+    },
+    {
+      "epoch": 0.4000255607387053,
+      "grad_norm": 8.780450151065313,
+      "learning_rate": 1.4299400108063247e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -380.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1473,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 7.09375,
+      "rewards/rejected": -23.125,
+      "step": 3130
+    },
+    {
+      "epoch": 0.4013035976739728,
+      "grad_norm": 10.2040261805842,
+      "learning_rate": 1.4276612204964992e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -376.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1199,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.09375,
+      "rewards/rejected": -23.25,
+      "step": 3140
+    },
+    {
+      "epoch": 0.4025816346092402,
+      "grad_norm": 7.057410129328684,
+      "learning_rate": 1.4253932901995967e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.80078125,
+      "logps/chosen": -396.0,
+      "logps/rejected": -454.0,
+      "loss": 0.1645,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -23.875,
+      "step": 3150
+    },
+    {
+      "epoch": 0.40385967154450764,
+      "grad_norm": 9.19615316476253,
+      "learning_rate": 1.42313613392964e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -398.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1507,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 6.28125,
+      "rewards/rejected": -24.25,
+      "step": 3160
+    },
+    {
+      "epoch": 0.40513770847977504,
+      "grad_norm": 5.529158499367323,
+      "learning_rate": 1.4208896666507756e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -390.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1376,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 7.28125,
+      "rewards/rejected": -24.625,
+      "step": 3170
+    },
+    {
+      "epoch": 0.4064157454150425,
+      "grad_norm": 8.839300613924307,
+      "learning_rate": 1.4186538042638173e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -398.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1418,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 6.4375,
+      "rewards/rejected": -24.125,
+      "step": 3180
+    },
+    {
+      "epoch": 0.4076937823503099,
+      "grad_norm": 9.807154546126684,
+      "learning_rate": 1.416428463593022e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -374.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1429,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 6.90625,
+      "rewards/rejected": -24.125,
+      "step": 3190
+    },
+    {
+      "epoch": 0.40897181928557735,
+      "grad_norm": 6.209826291258967,
+      "learning_rate": 1.414213562373095e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -382.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1688,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 7.21875,
+      "rewards/rejected": -24.875,
+      "step": 3200
+    },
+    {
+      "epoch": 0.41024985622084476,
+      "grad_norm": 6.7685423285524,
+      "learning_rate": 1.4120090192364154e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -378.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1296,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -24.25,
+      "step": 3210
+    },
+    {
+      "epoch": 0.4115278931561122,
+      "grad_norm": 8.116545814283745,
+      "learning_rate": 1.4098147537004828e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -384.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1744,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -23.5,
+      "step": 3220
+    },
+    {
+      "epoch": 0.4128059300913796,
+      "grad_norm": 7.781823219134506,
+      "learning_rate": 1.4076306861555735e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -384.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1359,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 6.5,
+      "rewards/rejected": -23.5,
+      "step": 3230
+    },
+    {
+      "epoch": 0.41408396702664707,
+      "grad_norm": 8.946415474640869,
+      "learning_rate": 1.405456737852613e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -382.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1446,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.1875,
+      "rewards/rejected": -23.875,
+      "step": 3240
+    },
+    {
+      "epoch": 0.41536200396191447,
+      "grad_norm": 4.764559880498014,
+      "learning_rate": 1.4032928308912468e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -400.0,
+      "logps/rejected": -450.0,
+      "loss": 0.145,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 6.53125,
+      "rewards/rejected": -23.625,
+      "step": 3250
+    },
+    {
+      "epoch": 0.41664004089718193,
+      "grad_norm": 4.3114908803784315,
+      "learning_rate": 1.4011388882081175e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -366.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1509,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 6.375,
+      "rewards/rejected": -23.125,
+      "step": 3260
+    },
+    {
+      "epoch": 0.41791807783244933,
+      "grad_norm": 3.855559530088154,
+      "learning_rate": 1.3989948335653378e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -386.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1312,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 6.625,
+      "rewards/rejected": -23.5,
+      "step": 3270
+    },
+    {
+      "epoch": 0.4191961147677168,
+      "grad_norm": 7.950890171041907,
+      "learning_rate": 1.3968605915391564e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -380.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1369,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 7.15625,
+      "rewards/rejected": -24.5,
+      "step": 3280
+    },
+    {
+      "epoch": 0.4204741517029842,
+      "grad_norm": 6.158235945972364,
+      "learning_rate": 1.3947360875088132e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.953125,
+      "logps/chosen": -386.0,
+      "logps/rejected": -434.0,
+      "loss": 0.147,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 7.03125,
+      "rewards/rejected": -24.0,
+      "step": 3290
+    },
+    {
+      "epoch": 0.42175218863825165,
+      "grad_norm": 6.627835436865485,
+      "learning_rate": 1.3926212476455828e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -378.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1448,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -24.5,
+      "step": 3300
+    },
+    {
+      "epoch": 0.42303022557351905,
+      "grad_norm": 6.60348599001986,
+      "learning_rate": 1.3905159989019964e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.74609375,
+      "logps/chosen": -396.0,
+      "logps/rejected": -468.0,
+      "loss": 0.1557,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 6.65625,
+      "rewards/rejected": -24.0,
+      "step": 3310
+    },
+    {
+      "epoch": 0.4243082625087865,
+      "grad_norm": 8.9669865493656,
+      "learning_rate": 1.3884202690012465e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.80078125,
+      "logps/chosen": -402.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1629,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 6.53125,
+      "rewards/rejected": -24.25,
+      "step": 3320
+    },
+    {
+      "epoch": 0.4255862994440539,
+      "grad_norm": 7.357437627405348,
+      "learning_rate": 1.3863339864267636e-06,
+      "logits/chosen": -1.1640625,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -370.0,
+      "logps/rejected": -430.0,
+      "loss": 0.152,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 7.0625,
+      "rewards/rejected": -24.0,
+      "step": 3330
+    },
+    {
+      "epoch": 0.42686433637932136,
+      "grad_norm": 7.467529833429661,
+      "learning_rate": 1.3842570804119655e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -380.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1451,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -24.75,
+      "step": 3340
+    },
+    {
+      "epoch": 0.42814237331458876,
+      "grad_norm": 5.796634143094144,
+      "learning_rate": 1.3821894809301763e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.80078125,
+      "logps/chosen": -404.0,
+      "logps/rejected": -436.0,
+      "loss": 0.132,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 7.15625,
+      "rewards/rejected": -24.75,
+      "step": 3350
+    },
+    {
+      "epoch": 0.4294204102498562,
+      "grad_norm": 8.127845689577871,
+      "learning_rate": 1.3801311186847081e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.765625,
+      "logps/chosen": -390.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1421,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.34375,
+      "rewards/rejected": -24.375,
+      "step": 3360
+    },
+    {
+      "epoch": 0.4306984471851236,
+      "grad_norm": 7.959998820521283,
+      "learning_rate": 1.378081925099109e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.76953125,
+      "logps/chosen": -370.0,
+      "logps/rejected": -444.0,
+      "loss": 0.135,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 6.96875,
+      "rewards/rejected": -23.375,
+      "step": 3370
+    },
+    {
+      "epoch": 0.4319764841203911,
+      "grad_norm": 6.2696403620436865,
+      "learning_rate": 1.376041832307563e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.71875,
+      "logps/chosen": -364.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1536,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -23.75,
+      "step": 3380
+    },
+    {
+      "epoch": 0.4332545210556585,
+      "grad_norm": 7.7561440507099295,
+      "learning_rate": 1.3740107731454524e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -374.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1281,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 6.59375,
+      "rewards/rejected": -23.375,
+      "step": 3390
+    },
+    {
+      "epoch": 0.43453255799092594,
+      "grad_norm": 7.179476693659551,
+      "learning_rate": 1.3719886811400705e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -382.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1399,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.0,
+      "rewards/rejected": -23.25,
+      "step": 3400
+    },
+    {
+      "epoch": 0.43581059492619334,
+      "grad_norm": 5.306936289656675,
+      "learning_rate": 1.3699754905014834e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -370.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1237,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 6.8125,
+      "rewards/rejected": -23.75,
+      "step": 3410
+    },
+    {
+      "epoch": 0.4370886318614608,
+      "grad_norm": 7.33511917259012,
+      "learning_rate": 1.3679711361135388e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.765625,
+      "logps/chosen": -372.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1406,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.15625,
+      "rewards/rejected": -24.25,
+      "step": 3420
+    },
+    {
+      "epoch": 0.4383666687967282,
+      "grad_norm": 7.891421525368514,
+      "learning_rate": 1.3659755535250212e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.796875,
+      "logps/chosen": -376.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1446,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 7.125,
+      "rewards/rejected": -23.625,
+      "step": 3430
+    },
+    {
+      "epoch": 0.43964470573199566,
+      "grad_norm": 4.4977245488764455,
+      "learning_rate": 1.3639886789409469e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -372.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1218,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 5.75,
+      "rewards/rejected": -23.0,
+      "step": 3440
+    },
+    {
+      "epoch": 0.44092274266726306,
+      "grad_norm": 6.192133824919038,
+      "learning_rate": 1.3620104492139977e-06,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -0.79296875,
+      "logps/chosen": -370.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1334,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 7.53125,
+      "rewards/rejected": -24.875,
+      "step": 3450
+    },
+    {
+      "epoch": 0.4422007796025305,
+      "grad_norm": 6.748695841419781,
+      "learning_rate": 1.3600408018360918e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -374.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1546,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 6.78125,
+      "rewards/rejected": -23.875,
+      "step": 3460
+    },
+    {
+      "epoch": 0.4434788165377979,
+      "grad_norm": 8.576278709172726,
+      "learning_rate": 1.3580796749300878e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.859375,
+      "logps/chosen": -416.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1281,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -24.375,
+      "step": 3470
+    },
+    {
+      "epoch": 0.44475685347306537,
+      "grad_norm": 6.507460744728536,
+      "learning_rate": 1.3561270072416209e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.859375,
+      "logps/chosen": -406.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1488,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 6.96875,
+      "rewards/rejected": -24.75,
+      "step": 3480
+    },
+    {
+      "epoch": 0.4460348904083328,
+      "grad_norm": 5.859961217813401,
+      "learning_rate": 1.3541827381310652e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -374.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1209,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 7.21875,
+      "rewards/rejected": -24.125,
+      "step": 3490
+    },
+    {
+      "epoch": 0.44731292734360023,
+      "grad_norm": 5.743291788596569,
+      "learning_rate": 1.3522468075656264e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -356.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1454,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.28125,
+      "rewards/rejected": -23.875,
+      "step": 3500
+    },
+    {
+      "epoch": 0.44859096427886763,
+      "grad_norm": 8.567744687752718,
+      "learning_rate": 1.3503191561115553e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -374.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1283,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.1875,
+      "rewards/rejected": -24.125,
+      "step": 3510
+    },
+    {
+      "epoch": 0.4498690012141351,
+      "grad_norm": 7.21394304984864,
+      "learning_rate": 1.348399724926484e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -398.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1499,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.84375,
+      "rewards/rejected": -24.5,
+      "step": 3520
+    },
+    {
+      "epoch": 0.4511470381494025,
+      "grad_norm": 6.8718798225064095,
+      "learning_rate": 1.346488455751882e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.78125,
+      "logps/chosen": -384.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1839,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 6.96875,
+      "rewards/rejected": -24.75,
+      "step": 3530
+    },
+    {
+      "epoch": 0.45242507508466995,
+      "grad_norm": 7.900397417506228,
+      "learning_rate": 1.3445852909056286e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.8359375,
+      "logps/chosen": -388.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1669,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 6.6875,
+      "rewards/rejected": -24.125,
+      "step": 3540
+    },
+    {
+      "epoch": 0.45370311201993735,
+      "grad_norm": 7.580032666610403,
+      "learning_rate": 1.3426901732747024e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -408.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1404,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -24.25,
+      "step": 3550
+    },
+    {
+      "epoch": 0.4549811489552048,
+      "grad_norm": 7.799351090372204,
+      "learning_rate": 1.3408030463079818e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.77734375,
+      "logps/chosen": -378.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1573,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 6.84375,
+      "rewards/rejected": -24.25,
+      "step": 3560
+    },
+    {
+      "epoch": 0.4562591858904722,
+      "grad_norm": 7.816339760338242,
+      "learning_rate": 1.3389238540091568e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.73828125,
+      "logps/chosen": -376.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1267,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 6.84375,
+      "rewards/rejected": -24.0,
+      "step": 3570
+    },
+    {
+      "epoch": 0.45753722282573966,
+      "grad_norm": 9.401171045263284,
+      "learning_rate": 1.337052540929751e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.8046875,
+      "logps/chosen": -374.0,
+      "logps/rejected": -452.0,
+      "loss": 0.1277,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -24.5,
+      "step": 3580
+    },
+    {
+      "epoch": 0.45881525976100707,
+      "grad_norm": 9.870017460070583,
+      "learning_rate": 1.33518905216225e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -392.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1469,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 6.78125,
+      "rewards/rejected": -24.0,
+      "step": 3590
+    },
+    {
+      "epoch": 0.4600932966962745,
+      "grad_norm": 5.500179153225504,
+      "learning_rate": 1.3333333333333332e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -382.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1304,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -24.25,
+      "step": 3600
+    },
+    {
+      "epoch": 0.4613713336315419,
+      "grad_norm": 3.973386142931335,
+      "learning_rate": 1.3314853305972122e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.76953125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1517,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -23.75,
+      "step": 3610
+    },
+    {
+      "epoch": 0.4626493705668094,
+      "grad_norm": 6.111037112873324,
+      "learning_rate": 1.3296449906290671e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -376.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1204,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 6.4375,
+      "rewards/rejected": -22.5,
+      "step": 3620
+    },
+    {
+      "epoch": 0.4639274075020768,
+      "grad_norm": 5.943152067380111,
+      "learning_rate": 1.3278122606185844e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -368.0,
+      "logps/rejected": -416.0,
+      "loss": 0.13,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.21875,
+      "rewards/rejected": -23.75,
+      "step": 3630
+    },
+    {
+      "epoch": 0.46520544443734424,
+      "grad_norm": 6.481672436539191,
+      "learning_rate": 1.3259870882635918e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -404.0,
+      "logps/rejected": -452.0,
+      "loss": 0.1597,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 6.28125,
+      "rewards/rejected": -23.875,
+      "step": 3640
+    },
+    {
+      "epoch": 0.46648348137261164,
+      "grad_norm": 4.085915928721287,
+      "learning_rate": 1.3241694217637886e-06,
+      "logits/chosen": -1.1953125,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -376.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1331,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 6.875,
+      "rewards/rejected": -24.25,
+      "step": 3650
+    },
+    {
+      "epoch": 0.4677615183078791,
+      "grad_norm": 7.836128061976743,
+      "learning_rate": 1.3223592098145723e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -400.0,
+      "logps/rejected": -468.0,
+      "loss": 0.1322,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -24.75,
+      "step": 3660
+    },
+    {
+      "epoch": 0.4690395552431465,
+      "grad_norm": 7.49772140941902,
+      "learning_rate": 1.3205564016009555e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -384.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1144,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -24.5,
+      "step": 3670
+    },
+    {
+      "epoch": 0.47031759217841396,
+      "grad_norm": 5.529251780644596,
+      "learning_rate": 1.318760946791574e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.78515625,
+      "logps/chosen": -360.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1248,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.65625,
+      "rewards/rejected": -24.625,
+      "step": 3680
+    },
+    {
+      "epoch": 0.47159562911368136,
+      "grad_norm": 8.730661176458746,
+      "learning_rate": 1.316972795532786e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.875,
+      "logps/chosen": -396.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1552,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -24.125,
+      "step": 3690
+    },
+    {
+      "epoch": 0.4728736660489488,
+      "grad_norm": 3.2419995762732334,
+      "learning_rate": 1.3151918984428582e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.82421875,
+      "logps/chosen": -372.0,
+      "logps/rejected": -424.0,
+      "loss": 0.0937,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -15.9375,
+      "rewards/margins": 7.65625,
+      "rewards/rejected": -23.625,
+      "step": 3700
+    },
+    {
+      "epoch": 0.4741517029842162,
+      "grad_norm": 6.188841935853007,
+      "learning_rate": 1.313418206606237e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -374.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1252,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 6.46875,
+      "rewards/rejected": -23.125,
+      "step": 3710
+    },
+    {
+      "epoch": 0.4754297399194837,
+      "grad_norm": 7.468751992893234,
+      "learning_rate": 1.3116516715679057e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -376.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1608,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 6.84375,
+      "rewards/rejected": -23.125,
+      "step": 3720
+    },
+    {
+      "epoch": 0.4767077768547511,
+      "grad_norm": 5.808655909227352,
+      "learning_rate": 1.3098922453278258e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -402.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1347,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 7.125,
+      "rewards/rejected": -25.0,
+      "step": 3730
+    },
+    {
+      "epoch": 0.47798581379001853,
+      "grad_norm": 6.003123080875553,
+      "learning_rate": 1.3081398803354573e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.953125,
+      "logps/chosen": -404.0,
+      "logps/rejected": -452.0,
+      "loss": 0.1239,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 7.75,
+      "rewards/rejected": -25.125,
+      "step": 3740
+    },
+    {
+      "epoch": 0.47926385072528593,
+      "grad_norm": 3.8591751975851296,
+      "learning_rate": 1.3063945294843617e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -376.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1389,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.78125,
+      "rewards/rejected": -24.0,
+      "step": 3750
+    },
+    {
+      "epoch": 0.4805418876605534,
+      "grad_norm": 4.019948083029915,
+      "learning_rate": 1.3046561461068843e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -378.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1261,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.0625,
+      "rewards/rejected": -23.25,
+      "step": 3760
+    },
+    {
+      "epoch": 0.4818199245958208,
+      "grad_norm": 7.528197680804647,
+      "learning_rate": 1.3029246839689124e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.95703125,
+      "logps/chosen": -396.0,
+      "logps/rejected": -448.0,
+      "loss": 0.142,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.5625,
+      "rewards/rejected": -24.25,
+      "step": 3770
+    },
+    {
+      "epoch": 0.48309796153108825,
+      "grad_norm": 6.7744718025634105,
+      "learning_rate": 1.3012000972647109e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -382.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1574,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -15.8125,
+      "rewards/margins": 6.5625,
+      "rewards/rejected": -22.375,
+      "step": 3780
+    },
+    {
+      "epoch": 0.48437599846635565,
+      "grad_norm": 5.75171347028417,
+      "learning_rate": 1.299482340611832e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.734375,
+      "logps/chosen": -360.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1749,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 6.28125,
+      "rewards/rejected": -21.75,
+      "step": 3790
+    },
+    {
+      "epoch": 0.4856540354016231,
+      "grad_norm": 8.118201944930425,
+      "learning_rate": 1.2977713690461003e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.8046875,
+      "logps/chosen": -364.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1506,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 6.25,
+      "rewards/rejected": -22.75,
+      "step": 3800
+    },
+    {
+      "epoch": 0.4869320723368905,
+      "grad_norm": 5.781074466455824,
+      "learning_rate": 1.296067138016669e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -386.0,
+      "logps/rejected": -416.0,
+      "loss": 0.137,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 6.78125,
+      "rewards/rejected": -23.75,
+      "step": 3810
+    },
+    {
+      "epoch": 0.48821010927215797,
+      "grad_norm": 8.567547243991521,
+      "learning_rate": 1.294369603381147e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -392.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1392,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -24.5,
+      "step": 3820
+    },
+    {
+      "epoch": 0.48948814620742537,
+      "grad_norm": 6.390349764077649,
+      "learning_rate": 1.2926787214007981e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -404.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1176,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 7.40625,
+      "rewards/rejected": -24.75,
+      "step": 3830
+    },
+    {
+      "epoch": 0.4907661831426928,
+      "grad_norm": 3.172624398276498,
+      "learning_rate": 1.2909944487358056e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -376.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1279,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.71875,
+      "rewards/rejected": -23.875,
+      "step": 3840
+    },
+    {
+      "epoch": 0.4920442200779602,
+      "grad_norm": 6.877494065462421,
+      "learning_rate": 1.2893167424406084e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -392.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1124,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -24.375,
+      "step": 3850
+    },
+    {
+      "epoch": 0.4933222570132277,
+      "grad_norm": 4.7852166106607195,
+      "learning_rate": 1.2876455599593008e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -396.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1298,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 7.9375,
+      "rewards/rejected": -25.75,
+      "step": 3860
+    },
+    {
+      "epoch": 0.4946002939484951,
+      "grad_norm": 6.776250771068472,
+      "learning_rate": 1.285980859121099e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1449,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -25.625,
+      "step": 3870
+    },
+    {
+      "epoch": 0.49587833088376254,
+      "grad_norm": 7.220973098332561,
+      "learning_rate": 1.2843225981358712e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -402.0,
+      "logps/rejected": -460.0,
+      "loss": 0.1508,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -18.25,
+      "rewards/margins": 7.84375,
+      "rewards/rejected": -26.125,
+      "step": 3880
+    },
+    {
+      "epoch": 0.49715636781902994,
+      "grad_norm": 5.84048599845536,
+      "learning_rate": 1.2826707355897317e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.828125,
+      "logps/chosen": -400.0,
+      "logps/rejected": -460.0,
+      "loss": 0.1222,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 8.125,
+      "rewards/rejected": -25.875,
+      "step": 3890
+    },
+    {
+      "epoch": 0.4984344047542974,
+      "grad_norm": 5.849381490106306,
+      "learning_rate": 1.281025230440697e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -398.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1524,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -18.25,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -25.375,
+      "step": 3900
+    },
+    {
+      "epoch": 0.4997124416895648,
+      "grad_norm": 7.420121633210637,
+      "learning_rate": 1.2793860420144025e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -408.0,
+      "logps/rejected": -454.0,
+      "loss": 0.1261,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 7.8125,
+      "rewards/rejected": -25.375,
+      "step": 3910
+    },
+    {
+      "epoch": 0.5009904786248323,
+      "grad_norm": 4.194460822711977,
+      "learning_rate": 1.2777531299998798e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -414.0,
+      "logps/rejected": -454.0,
+      "loss": 0.1085,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.09375,
+      "rewards/rejected": -23.875,
+      "step": 3920
+    },
+    {
+      "epoch": 0.5022685155600997,
+      "grad_norm": 6.44365877864499,
+      "learning_rate": 1.2761264544453928e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.79296875,
+      "logps/chosen": -386.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1137,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 6.21875,
+      "rewards/rejected": -22.5,
+      "step": 3930
+    },
+    {
+      "epoch": 0.5035465524953671,
+      "grad_norm": 9.471471498856264,
+      "learning_rate": 1.2745059757543324e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -394.0,
+      "logps/rejected": -434.0,
+      "loss": 0.132,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -24.375,
+      "step": 3940
+    },
+    {
+      "epoch": 0.5048245894306346,
+      "grad_norm": 6.952525073057976,
+      "learning_rate": 1.272891654681168e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -378.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1468,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 6.71875,
+      "rewards/rejected": -23.875,
+      "step": 3950
+    },
+    {
+      "epoch": 0.506102626365902,
+      "grad_norm": 5.295328329343153,
+      "learning_rate": 1.2712834523274563e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1345,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.53125,
+      "rewards/rejected": -23.875,
+      "step": 3960
+    },
+    {
+      "epoch": 0.5073806633011694,
+      "grad_norm": 10.169118805733405,
+      "learning_rate": 1.2696813301379032e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.828125,
+      "logps/chosen": -378.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1449,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -24.375,
+      "step": 3970
+    },
+    {
+      "epoch": 0.5086587002364368,
+      "grad_norm": 7.809367609321747,
+      "learning_rate": 1.2680852498964829e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.84765625,
+      "logps/chosen": -368.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1564,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.53125,
+      "rewards/rejected": -24.25,
+      "step": 3980
+    },
+    {
+      "epoch": 0.5099367371717043,
+      "grad_norm": 5.337338997488873,
+      "learning_rate": 1.266495173722607e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -360.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1521,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.15625,
+      "rewards/rejected": -23.5,
+      "step": 3990
+    },
+    {
+      "epoch": 0.5112147741069717,
+      "grad_norm": 6.18336735157534,
+      "learning_rate": 1.2649110640673517e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -360.0,
+      "logps/rejected": -394.0,
+      "loss": 0.1266,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.75,
+      "rewards/margins": 6.4375,
+      "rewards/rejected": -22.25,
+      "step": 4000
+    },
+    {
+      "epoch": 0.5124928110422391,
+      "grad_norm": 5.715217290971865,
+      "learning_rate": 1.2633328837097308e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -386.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1193,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 6.875,
+      "rewards/rejected": -23.625,
+      "step": 4010
+    },
+    {
+      "epoch": 0.5137708479775065,
+      "grad_norm": 7.66224001452475,
+      "learning_rate": 1.2617605957530233e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -382.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1279,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 6.84375,
+      "rewards/rejected": -23.875,
+      "step": 4020
+    },
+    {
+      "epoch": 0.515048884912774,
+      "grad_norm": 7.035786768866535,
+      "learning_rate": 1.2601941636211516e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -396.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1265,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 7.40625,
+      "rewards/rejected": -23.875,
+      "step": 4030
+    },
+    {
+      "epoch": 0.5163269218480414,
+      "grad_norm": 8.467120293146044,
+      "learning_rate": 1.2586335510551052e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -376.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1493,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.9375,
+      "rewards/margins": 6.8125,
+      "rewards/rejected": -22.75,
+      "step": 4040
+    },
+    {
+      "epoch": 0.5176049587833088,
+      "grad_norm": 7.225608426572022,
+      "learning_rate": 1.2570787221094177e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -382.0,
+      "logps/rejected": -442.0,
+      "loss": 0.154,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.3125,
+      "rewards/margins": 7.96875,
+      "rewards/rejected": -23.25,
+      "step": 4050
+    },
+    {
+      "epoch": 0.5188829957185762,
+      "grad_norm": 8.960330110133077,
+      "learning_rate": 1.255529641148689e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -374.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1613,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -15.5,
+      "rewards/margins": 6.6875,
+      "rewards/rejected": -22.25,
+      "step": 4060
+    },
+    {
+      "epoch": 0.5201610326538437,
+      "grad_norm": 7.587345693895239,
+      "learning_rate": 1.2539862728441536e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -364.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1587,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.8125,
+      "rewards/margins": 6.625,
+      "rewards/rejected": -22.375,
+      "step": 4070
+    },
+    {
+      "epoch": 0.5214390695891111,
+      "grad_norm": 8.07316893337428,
+      "learning_rate": 1.252448582170299e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1059,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 6.21875,
+      "rewards/rejected": -21.625,
+      "step": 4080
+    },
+    {
+      "epoch": 0.5227171065243785,
+      "grad_norm": 8.777795841233306,
+      "learning_rate": 1.2509165344015243e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -384.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1194,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 7.5625,
+      "rewards/rejected": -22.75,
+      "step": 4090
+    },
+    {
+      "epoch": 0.5239951434596459,
+      "grad_norm": 7.376492220186144,
+      "learning_rate": 1.2493900951088486e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -372.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1373,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 7.28125,
+      "rewards/rejected": -22.5,
+      "step": 4100
+    },
+    {
+      "epoch": 0.5252731803949134,
+      "grad_norm": 4.836175661802613,
+      "learning_rate": 1.2478692301566601e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -376.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1361,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -23.0,
+      "step": 4110
+    },
+    {
+      "epoch": 0.5265512173301808,
+      "grad_norm": 2.1912168019876788,
+      "learning_rate": 1.2463539056995116e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.78515625,
+      "logps/chosen": -362.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1268,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 6.96875,
+      "rewards/rejected": -23.25,
+      "step": 4120
+    },
+    {
+      "epoch": 0.5278292542654482,
+      "grad_norm": 5.33529847776421,
+      "learning_rate": 1.2448440881789541e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -396.0,
+      "logps/rejected": -452.0,
+      "loss": 0.1343,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.9375,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -24.25,
+      "step": 4130
+    },
+    {
+      "epoch": 0.5291072912007156,
+      "grad_norm": 5.91122985473326,
+      "learning_rate": 1.2433397443204184e-06,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -388.0,
+      "logps/rejected": -452.0,
+      "loss": 0.1366,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -24.375,
+      "step": 4140
+    },
+    {
+      "epoch": 0.5303853281359832,
+      "grad_norm": 7.954321209531519,
+      "learning_rate": 1.2418408411301324e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -372.0,
+      "logps/rejected": -428.0,
+      "loss": 0.152,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 6.34375,
+      "rewards/rejected": -23.875,
+      "step": 4150
+    },
+    {
+      "epoch": 0.5316633650712506,
+      "grad_norm": 6.267659350329792,
+      "learning_rate": 1.2403473458920844e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.80078125,
+      "logps/chosen": -358.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1392,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.3125,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -22.75,
+      "step": 4160
+    },
+    {
+      "epoch": 0.532941402006518,
+      "grad_norm": 4.055486021177046,
+      "learning_rate": 1.2388592261650217e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.80859375,
+      "logps/chosen": -362.0,
+      "logps/rejected": -396.0,
+      "loss": 0.1079,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 6.875,
+      "rewards/rejected": -22.75,
+      "step": 4170
+    },
+    {
+      "epoch": 0.5342194389417854,
+      "grad_norm": 7.2619506906943405,
+      "learning_rate": 1.2373764497794918e-06,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -390.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1461,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 6.875,
+      "rewards/rejected": -23.875,
+      "step": 4180
+    },
+    {
+      "epoch": 0.5354974758770529,
+      "grad_norm": 5.364257727612288,
+      "learning_rate": 1.2358989848349217e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -366.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1177,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 6.71875,
+      "rewards/rejected": -23.375,
+      "step": 4190
+    },
+    {
+      "epoch": 0.5367755128123203,
+      "grad_norm": 6.295173920747968,
+      "learning_rate": 1.2344267996967353e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -382.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1309,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 7.5625,
+      "rewards/rejected": -24.875,
+      "step": 4200
+    },
+    {
+      "epoch": 0.5380535497475877,
+      "grad_norm": 5.594792386133589,
+      "learning_rate": 1.2329598629935076e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -382.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1262,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.21875,
+      "rewards/rejected": -23.5,
+      "step": 4210
+    },
+    {
+      "epoch": 0.5393315866828551,
+      "grad_norm": 7.589837756651567,
+      "learning_rate": 1.2314981436141583e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -366.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1175,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -23.625,
+      "step": 4220
+    },
+    {
+      "epoch": 0.5406096236181226,
+      "grad_norm": 8.069257213461738,
+      "learning_rate": 1.2300416107051802e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -362.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1436,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.8125,
+      "rewards/margins": 7.0625,
+      "rewards/rejected": -22.875,
+      "step": 4230
+    },
+    {
+      "epoch": 0.54188766055339,
+      "grad_norm": 8.822401069484748,
+      "learning_rate": 1.2285902336679024e-06,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -368.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1044,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 6.8125,
+      "rewards/rejected": -23.125,
+      "step": 4240
+    },
+    {
+      "epoch": 0.5431656974886574,
+      "grad_norm": 9.687616141577971,
+      "learning_rate": 1.2271439821557926e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -364.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1207,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.8125,
+      "rewards/margins": 7.21875,
+      "rewards/rejected": -23.0,
+      "step": 4250
+    },
+    {
+      "epoch": 0.5444437344239248,
+      "grad_norm": 8.622642326345176,
+      "learning_rate": 1.225702826071791e-06,
+      "logits/chosen": -1.46875,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -374.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1447,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 7.46875,
+      "rewards/rejected": -23.5,
+      "step": 4260
+    },
+    {
+      "epoch": 0.5457217713591923,
+      "grad_norm": 7.855367178541955,
+      "learning_rate": 1.2242667355656797e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -378.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1363,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 7.78125,
+      "rewards/rejected": -22.875,
+      "step": 4270
+    },
+    {
+      "epoch": 0.5469998082944597,
+      "grad_norm": 7.698289553789727,
+      "learning_rate": 1.2228356810314862e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -362.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1492,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 6.71875,
+      "rewards/rejected": -22.125,
+      "step": 4280
+    },
+    {
+      "epoch": 0.5482778452297271,
+      "grad_norm": 8.210918110223535,
+      "learning_rate": 1.2214096331049186e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -356.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1238,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -13.8125,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -21.375,
+      "step": 4290
+    },
+    {
+      "epoch": 0.5495558821649945,
+      "grad_norm": 7.80225558271295,
+      "learning_rate": 1.2199885626608373e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -352.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1355,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.0,
+      "rewards/margins": 7.875,
+      "rewards/rejected": -22.875,
+      "step": 4300
+    },
+    {
+      "epoch": 0.550833919100262,
+      "grad_norm": 7.396685946554367,
+      "learning_rate": 1.2185724408107546e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.95703125,
+      "logps/chosen": -364.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1444,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 7.21875,
+      "rewards/rejected": -22.375,
+      "step": 4310
+    },
+    {
+      "epoch": 0.5521119560355294,
+      "grad_norm": 8.038504027446018,
+      "learning_rate": 1.2171612389003689e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -356.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1315,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -15.5,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -22.75,
+      "step": 4320
+    },
+    {
+      "epoch": 0.5533899929707968,
+      "grad_norm": 8.338061461024187,
+      "learning_rate": 1.2157549285071297e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.7578125,
+      "logps/chosen": -376.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1104,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 6.59375,
+      "rewards/rejected": -22.875,
+      "step": 4330
+    },
+    {
+      "epoch": 0.5546680299060642,
+      "grad_norm": 8.296028800610419,
+      "learning_rate": 1.2143534814378327e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -382.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1246,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 6.53125,
+      "rewards/rejected": -22.75,
+      "step": 4340
+    },
+    {
+      "epoch": 0.5559460668413317,
+      "grad_norm": 5.382077484968552,
+      "learning_rate": 1.2129568697262454e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1153,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.625,
+      "rewards/margins": 7.84375,
+      "rewards/rejected": -23.5,
+      "step": 4350
+    },
+    {
+      "epoch": 0.5572241037765991,
+      "grad_norm": 5.448804011826712,
+      "learning_rate": 1.2115650656307653e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -382.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1119,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 7.46875,
+      "rewards/rejected": -23.375,
+      "step": 4360
+    },
+    {
+      "epoch": 0.5585021407118665,
+      "grad_norm": 7.8554182191636786,
+      "learning_rate": 1.210178041632103e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1383,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -24.25,
+      "step": 4370
+    },
+    {
+      "epoch": 0.559780177647134,
+      "grad_norm": 6.195537947423835,
+      "learning_rate": 1.2087957704309988e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -384.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1211,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 6.90625,
+      "rewards/rejected": -23.375,
+      "step": 4380
+    },
+    {
+      "epoch": 0.5610582145824015,
+      "grad_norm": 9.813903798949418,
+      "learning_rate": 1.2074182249459642e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -374.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1463,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.8125,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -23.25,
+      "step": 4390
+    },
+    {
+      "epoch": 0.5623362515176689,
+      "grad_norm": 6.682128234316747,
+      "learning_rate": 1.2060453783110545e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -372.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1298,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 6.90625,
+      "rewards/rejected": -23.25,
+      "step": 4400
+    },
+    {
+      "epoch": 0.5636142884529363,
+      "grad_norm": 6.79332680051349,
+      "learning_rate": 1.2046772038736682e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -364.0,
+      "logps/rejected": -396.0,
+      "loss": 0.1107,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -15.625,
+      "rewards/margins": 7.125,
+      "rewards/rejected": -22.75,
+      "step": 4410
+    },
+    {
+      "epoch": 0.5648923253882037,
+      "grad_norm": 4.435439224632074,
+      "learning_rate": 1.2033136751923736e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -384.0,
+      "logps/rejected": -450.0,
+      "loss": 0.1369,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.40625,
+      "rewards/rejected": -23.625,
+      "step": 4420
+    },
+    {
+      "epoch": 0.5661703623234712,
+      "grad_norm": 5.842652695966885,
+      "learning_rate": 1.201954766034762e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.875,
+      "logps/chosen": -382.0,
+      "logps/rejected": -430.0,
+      "loss": 0.154,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -23.625,
+      "step": 4430
+    },
+    {
+      "epoch": 0.5674483992587386,
+      "grad_norm": 6.872268677684548,
+      "learning_rate": 1.2006004503753285e-06,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -362.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1266,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 6.6875,
+      "rewards/rejected": -24.0,
+      "step": 4440
+    },
+    {
+      "epoch": 0.568726436194006,
+      "grad_norm": 9.386572790503253,
+      "learning_rate": 1.1992507023933782e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -404.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1381,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 6.8125,
+      "rewards/rejected": -23.625,
+      "step": 4450
+    },
+    {
+      "epoch": 0.5700044731292734,
+      "grad_norm": 6.2635119973685365,
+      "learning_rate": 1.1979054964709597e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -394.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1305,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.75,
+      "rewards/rejected": -24.125,
+      "step": 4460
+    },
+    {
+      "epoch": 0.5712825100645409,
+      "grad_norm": 7.249791670394813,
+      "learning_rate": 1.1965648071908207e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -376.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1144,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 6.625,
+      "rewards/rejected": -23.875,
+      "step": 4470
+    },
+    {
+      "epoch": 0.5725605469998083,
+      "grad_norm": 5.517251771633542,
+      "learning_rate": 1.1952286093343935e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -380.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1166,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -24.0,
+      "step": 4480
+    },
+    {
+      "epoch": 0.5738385839350757,
+      "grad_norm": 8.237839278967805,
+      "learning_rate": 1.1938968778798005e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -396.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1366,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 7.15625,
+      "rewards/rejected": -24.0,
+      "step": 4490
+    },
+    {
+      "epoch": 0.5751166208703431,
+      "grad_norm": 6.961810325293726,
+      "learning_rate": 1.1925695879998878e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -400.0,
+      "logps/rejected": -458.0,
+      "loss": 0.1243,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -25.25,
+      "step": 4500
+    },
+    {
+      "epoch": 0.5763946578056106,
+      "grad_norm": 8.544441333180373,
+      "learning_rate": 1.1912467150602794e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -370.0,
+      "logps/rejected": -452.0,
+      "loss": 0.1221,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 8.5,
+      "rewards/rejected": -25.5,
+      "step": 4510
+    },
+    {
+      "epoch": 0.577672694740878,
+      "grad_norm": 6.737115879916741,
+      "learning_rate": 1.189928234617459e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -376.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1214,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -24.375,
+      "step": 4520
+    },
+    {
+      "epoch": 0.5789507316761454,
+      "grad_norm": 5.55086970039351,
+      "learning_rate": 1.1886141224168716e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -388.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1189,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.8125,
+      "rewards/rejected": -24.75,
+      "step": 4530
+    },
+    {
+      "epoch": 0.5802287686114128,
+      "grad_norm": 6.35245930526078,
+      "learning_rate": 1.1873043543910495e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -380.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1122,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 7.4375,
+      "rewards/rejected": -23.875,
+      "step": 4540
+    },
+    {
+      "epoch": 0.5815068055466803,
+      "grad_norm": 8.02396984404215,
+      "learning_rate": 1.1859989066577617e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.8046875,
+      "logps/chosen": -372.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1272,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 7.875,
+      "rewards/rejected": -23.25,
+      "step": 4550
+    },
+    {
+      "epoch": 0.5827848424819477,
+      "grad_norm": 6.07141478524121,
+      "learning_rate": 1.1846977555181846e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -378.0,
+      "logps/rejected": -432.0,
+      "loss": 0.133,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.46875,
+      "rewards/rejected": -23.625,
+      "step": 4560
+    },
+    {
+      "epoch": 0.5840628794172151,
+      "grad_norm": 6.6295475948572955,
+      "learning_rate": 1.1834008774550946e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.828125,
+      "logps/chosen": -376.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1233,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.6875,
+      "rewards/margins": 6.90625,
+      "rewards/rejected": -22.625,
+      "step": 4570
+    },
+    {
+      "epoch": 0.5853409163524825,
+      "grad_norm": 8.053365421425838,
+      "learning_rate": 1.1821082491310835e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.84765625,
+      "logps/chosen": -370.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1168,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.875,
+      "rewards/rejected": -24.0,
+      "step": 4580
+    },
+    {
+      "epoch": 0.58661895328775,
+      "grad_norm": 5.728951868806657,
+      "learning_rate": 1.1808198473867937e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -384.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1303,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 7.46875,
+      "rewards/rejected": -24.0,
+      "step": 4590
+    },
+    {
+      "epoch": 0.5878969902230174,
+      "grad_norm": 6.227974143571599,
+      "learning_rate": 1.179535649239177e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.875,
+      "logps/chosen": -366.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1122,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -15.0,
+      "rewards/margins": 8.375,
+      "rewards/rejected": -23.375,
+      "step": 4600
+    },
+    {
+      "epoch": 0.5891750271582848,
+      "grad_norm": 7.386140212898927,
+      "learning_rate": 1.178255631879771e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -364.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1071,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 7.4375,
+      "rewards/rejected": -22.75,
+      "step": 4610
+    },
+    {
+      "epoch": 0.5904530640935522,
+      "grad_norm": 4.967226503717122,
+      "learning_rate": 1.1769797726729992e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.875,
+      "logps/chosen": -374.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1057,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -22.5,
+      "step": 4620
+    },
+    {
+      "epoch": 0.5917311010288198,
+      "grad_norm": 6.780674250637313,
+      "learning_rate": 1.1757080491544881e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -376.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1051,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.5625,
+      "rewards/margins": 7.90625,
+      "rewards/rejected": -23.5,
+      "step": 4630
+    },
+    {
+      "epoch": 0.5930091379640872,
+      "grad_norm": 10.081960340436574,
+      "learning_rate": 1.1744404390294068e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -376.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1117,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 6.65625,
+      "rewards/rejected": -22.125,
+      "step": 4640
+    },
+    {
+      "epoch": 0.5942871748993546,
+      "grad_norm": 5.261421488836751,
+      "learning_rate": 1.1731769201708264e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.97265625,
+      "logps/chosen": -352.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1358,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -22.625,
+      "step": 4650
+    },
+    {
+      "epoch": 0.595565211834622,
+      "grad_norm": 7.074169827324963,
+      "learning_rate": 1.1719174706180952e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -382.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1378,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.6875,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -23.25,
+      "step": 4660
+    },
+    {
+      "epoch": 0.5968432487698895,
+      "grad_norm": 6.771768143397207,
+      "learning_rate": 1.1706620685752386e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -364.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1255,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.3125,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -22.875,
+      "step": 4670
+    },
+    {
+      "epoch": 0.5981212857051569,
+      "grad_norm": 4.382119608637236,
+      "learning_rate": 1.1694106924093723e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -360.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1079,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -15.625,
+      "rewards/margins": 8.1875,
+      "rewards/rejected": -23.875,
+      "step": 4680
+    },
+    {
+      "epoch": 0.5993993226404243,
+      "grad_norm": 11.777896364238343,
+      "learning_rate": 1.1681633206491381e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -378.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1312,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 8.1875,
+      "rewards/rejected": -24.625,
+      "step": 4690
+    },
+    {
+      "epoch": 0.6006773595756917,
+      "grad_norm": 5.183927716766896,
+      "learning_rate": 1.1669199319831564e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -380.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1419,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 7.53125,
+      "rewards/rejected": -24.0,
+      "step": 4700
+    },
+    {
+      "epoch": 0.6019553965109592,
+      "grad_norm": 7.652295457074283,
+      "learning_rate": 1.1656805052584958e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.74609375,
+      "logps/chosen": -380.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1177,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 7.4375,
+      "rewards/rejected": -23.5,
+      "step": 4710
+    },
+    {
+      "epoch": 0.6032334334462266,
+      "grad_norm": 3.0566699247987947,
+      "learning_rate": 1.164445019479164e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -360.0,
+      "logps/rejected": -404.0,
+      "loss": 0.0978,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -14.5625,
+      "rewards/margins": 7.75,
+      "rewards/rejected": -22.375,
+      "step": 4720
+    },
+    {
+      "epoch": 0.604511470381494,
+      "grad_norm": 6.947497645801134,
+      "learning_rate": 1.1632134538046105e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.875,
+      "logps/chosen": -340.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1294,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -14.8125,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -21.5,
+      "step": 4730
+    },
+    {
+      "epoch": 0.6057895073167614,
+      "grad_norm": 10.119691553753803,
+      "learning_rate": 1.1619857875482536e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -352.0,
+      "logps/rejected": -392.0,
+      "loss": 0.0962,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 7.46875,
+      "rewards/rejected": -22.375,
+      "step": 4740
+    },
+    {
+      "epoch": 0.6070675442520289,
+      "grad_norm": 6.481295494104017,
+      "learning_rate": 1.1607620001760185e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -356.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1354,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.0,
+      "rewards/rejected": -23.125,
+      "step": 4750
+    },
+    {
+      "epoch": 0.6083455811872963,
+      "grad_norm": 6.8447858042522745,
+      "learning_rate": 1.1595420713048968e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -380.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1327,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.6875,
+      "rewards/margins": 7.46875,
+      "rewards/rejected": -23.125,
+      "step": 4760
+    },
+    {
+      "epoch": 0.6096236181225637,
+      "grad_norm": 3.5043562217998687,
+      "learning_rate": 1.1583259807015182e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.8359375,
+      "logps/chosen": -372.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1081,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -23.625,
+      "step": 4770
+    },
+    {
+      "epoch": 0.6109016550578311,
+      "grad_norm": 5.579227140790423,
+      "learning_rate": 1.1571137082807434e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -380.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1194,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.03125,
+      "rewards/rejected": -23.25,
+      "step": 4780
+    },
+    {
+      "epoch": 0.6121796919930986,
+      "grad_norm": 5.444055418812265,
+      "learning_rate": 1.155905234104269e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.74609375,
+      "logps/chosen": -368.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1486,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 8.0,
+      "rewards/rejected": -23.875,
+      "step": 4790
+    },
+    {
+      "epoch": 0.613457728928366,
+      "grad_norm": 9.264921390379444,
+      "learning_rate": 1.1547005383792516e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.82421875,
+      "logps/chosen": -364.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1062,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 7.53125,
+      "rewards/rejected": -22.625,
+      "step": 4800
+    },
+    {
+      "epoch": 0.6147357658636334,
+      "grad_norm": 8.657680133145305,
+      "learning_rate": 1.1534996014569446e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -354.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1392,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 7.46875,
+      "rewards/rejected": -22.375,
+      "step": 4810
+    },
+    {
+      "epoch": 0.6160138027989008,
+      "grad_norm": 6.488465026619422,
+      "learning_rate": 1.1523024038313547e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.8359375,
+      "logps/chosen": -356.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1034,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.375,
+      "rewards/margins": 8.4375,
+      "rewards/rejected": -22.75,
+      "step": 4820
+    },
+    {
+      "epoch": 0.6172918397341683,
+      "grad_norm": 5.119168492208587,
+      "learning_rate": 1.1511089261379083e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -366.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1369,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -14.3125,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -22.625,
+      "step": 4830
+    },
+    {
+      "epoch": 0.6185698766694357,
+      "grad_norm": 6.227767107773455,
+      "learning_rate": 1.149919149152138e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -366.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1429,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.25,
+      "rewards/margins": 7.125,
+      "rewards/rejected": -21.375,
+      "step": 4840
+    },
+    {
+      "epoch": 0.6198479136047031,
+      "grad_norm": 4.458115277184433,
+      "learning_rate": 1.148733053788381e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.80859375,
+      "logps/chosen": -360.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1287,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 7.4375,
+      "rewards/rejected": -22.375,
+      "step": 4850
+    },
+    {
+      "epoch": 0.6211259505399706,
+      "grad_norm": 8.119513853649673,
+      "learning_rate": 1.1475506210984938e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.80078125,
+      "logps/chosen": -356.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1503,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 7.4375,
+      "rewards/rejected": -22.375,
+      "step": 4860
+    },
+    {
+      "epoch": 0.6224039874752381,
+      "grad_norm": 7.340814621972273,
+      "learning_rate": 1.1463718322705807e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -364.0,
+      "logps/rejected": -420.0,
+      "loss": 0.107,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -14.625,
+      "rewards/margins": 7.75,
+      "rewards/rejected": -22.375,
+      "step": 4870
+    },
+    {
+      "epoch": 0.6236820244105055,
+      "grad_norm": 4.340126429302955,
+      "learning_rate": 1.1451966686277364e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.859375,
+      "logps/chosen": -362.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1257,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.5,
+      "rewards/margins": 6.8125,
+      "rewards/rejected": -22.375,
+      "step": 4880
+    },
+    {
+      "epoch": 0.6249600613457729,
+      "grad_norm": 6.362814431540742,
+      "learning_rate": 1.1440251116268034e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.8046875,
+      "logps/chosen": -392.0,
+      "logps/rejected": -442.0,
+      "loss": 0.1502,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.28125,
+      "rewards/rejected": -23.625,
+      "step": 4890
+    },
+    {
+      "epoch": 0.6262380982810403,
+      "grad_norm": 6.638988083662024,
+      "learning_rate": 1.1428571428571428e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -386.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1016,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -24.375,
+      "step": 4900
+    },
+    {
+      "epoch": 0.6275161352163078,
+      "grad_norm": 5.498202281128867,
+      "learning_rate": 1.14169274403942e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.79296875,
+      "logps/chosen": -360.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1214,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -23.75,
+      "step": 4910
+    },
+    {
+      "epoch": 0.6287941721515752,
+      "grad_norm": 12.825988995751318,
+      "learning_rate": 1.140531897024402e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -372.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1227,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -23.5,
+      "step": 4920
+    },
+    {
+      "epoch": 0.6300722090868426,
+      "grad_norm": 4.449920674717063,
+      "learning_rate": 1.13937458379177e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.92578125,
+      "logps/chosen": -366.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1235,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -22.75,
+      "step": 4930
+    },
+    {
+      "epoch": 0.63135024602211,
+      "grad_norm": 6.865521262296344,
+      "learning_rate": 1.1382207864489444e-06,
+      "logits/chosen": -1.1953125,
+      "logits/rejected": -0.875,
+      "logps/chosen": -386.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1216,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.75,
+      "rewards/margins": 6.84375,
+      "rewards/rejected": -22.625,
+      "step": 4940
+    },
+    {
+      "epoch": 0.6326282829573775,
+      "grad_norm": 9.287739284268005,
+      "learning_rate": 1.1370704872299223e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -372.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1071,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 6.6875,
+      "rewards/rejected": -22.0,
+      "step": 4950
+    },
+    {
+      "epoch": 0.6339063198926449,
+      "grad_norm": 6.534465166619105,
+      "learning_rate": 1.1359236684941295e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -376.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1257,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -23.125,
+      "step": 4960
+    },
+    {
+      "epoch": 0.6351843568279123,
+      "grad_norm": 7.235171623819455,
+      "learning_rate": 1.1347803127252839e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -380.0,
+      "logps/rejected": -426.0,
+      "loss": 0.0894,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -15.5625,
+      "rewards/margins": 7.09375,
+      "rewards/rejected": -22.625,
+      "step": 4970
+    },
+    {
+      "epoch": 0.6364623937631797,
+      "grad_norm": 8.579088053132145,
+      "learning_rate": 1.1336404025302715e-06,
+      "logits/chosen": -1.1953125,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -376.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1403,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.5625,
+      "rewards/rejected": -23.875,
+      "step": 4980
+    },
+    {
+      "epoch": 0.6377404306984472,
+      "grad_norm": 9.911891227367601,
+      "learning_rate": 1.1325039206380352e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -382.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1589,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 6.96875,
+      "rewards/rejected": -23.75,
+      "step": 4990
+    },
+    {
+      "epoch": 0.6390184676337146,
+      "grad_norm": 5.143274254916214,
+      "learning_rate": 1.131370849898476e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -402.0,
+      "logps/rejected": -450.0,
+      "loss": 0.1252,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 7.875,
+      "rewards/rejected": -25.375,
+      "step": 5000
+    },
+    {
+      "epoch": 0.6390184676337146,
+      "eval_logits/chosen": -1.28125,
+      "eval_logits/rejected": -0.87109375,
+      "eval_logps/chosen": -384.0,
+      "eval_logps/rejected": -426.0,
+      "eval_loss": 0.22941964864730835,
+      "eval_rewards/accuracies": 0.9068260192871094,
+      "eval_rewards/chosen": -17.375,
+      "eval_rewards/margins": 6.09375,
+      "eval_rewards/rejected": -23.5,
+      "eval_runtime": 1569.0951,
+      "eval_samples_per_second": 35.375,
+      "eval_steps_per_second": 0.553,
+      "step": 5000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 7824,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 5000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}